shiplog 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +59 -0
  3. data/README.md +140 -5
  4. data/app/controllers/shiplog/admin/area_detections_controller.rb +15 -0
  5. data/app/controllers/shiplog/admin/area_translations_controller.rb +19 -0
  6. data/app/controllers/shiplog/admin/areas/merges_controller.rb +19 -0
  7. data/app/controllers/shiplog/admin/areas/moves_controller.rb +13 -0
  8. data/app/controllers/shiplog/admin/areas_controller.rb +48 -0
  9. data/app/controllers/shiplog/admin/entries/images/moves_controller.rb +20 -0
  10. data/app/controllers/shiplog/admin/entries/images_controller.rb +76 -0
  11. data/app/controllers/shiplog/admin/entries/translations/retranslations_controller.rb +23 -0
  12. data/app/controllers/shiplog/admin/entries/translations_controller.rb +35 -0
  13. data/app/controllers/shiplog/admin/entries_controller.rb +9 -3
  14. data/app/controllers/shiplog/changelog_entries_controller.rb +26 -0
  15. data/app/controllers/shiplog/changelog_feeds_controller.rb +19 -0
  16. data/app/controllers/shiplog/changelogs_controller.rb +21 -5
  17. data/app/controllers/shiplog/public_controller.rb +55 -0
  18. data/app/controllers/shiplog/roadmaps_controller.rb +3 -1
  19. data/app/frontend/shiplog/admin.css +9 -0
  20. data/app/frontend/shiplog/shiplog.css +47 -2
  21. data/app/helpers/shiplog/admin/entries_helper.rb +15 -0
  22. data/app/helpers/shiplog/permalink_helper.rb +113 -0
  23. data/app/helpers/shiplog/roadmap_helper.rb +9 -0
  24. data/app/helpers/shiplog/unseen_helper.rb +9 -0
  25. data/app/jobs/shiplog/detect_areas_job.rb +18 -0
  26. data/app/jobs/shiplog/rewrite_entry_job.rb +4 -1
  27. data/app/jobs/shiplog/translate_area_job.rb +25 -0
  28. data/app/jobs/shiplog/translate_entry_job.rb +25 -0
  29. data/app/models/shiplog/area.rb +156 -0
  30. data/app/models/shiplog/area_detector.rb +146 -0
  31. data/app/models/shiplog/area_translator.rb +77 -0
  32. data/app/models/shiplog/entry/images.rb +127 -0
  33. data/app/models/shiplog/entry.rb +204 -5
  34. data/app/models/shiplog/entry_translation.rb +15 -0
  35. data/app/models/shiplog/image_fetcher/downloader.rb +98 -0
  36. data/app/models/shiplog/image_fetcher/sources.rb +41 -0
  37. data/app/models/shiplog/image_fetcher.rb +72 -0
  38. data/app/models/shiplog/image_type.rb +111 -0
  39. data/app/models/shiplog/rewriter/prompt.rb +31 -7
  40. data/app/models/shiplog/rewriter.rb +41 -23
  41. data/app/models/shiplog/setting.rb +19 -0
  42. data/app/models/shiplog/translator/prompt.rb +40 -0
  43. data/app/models/shiplog/translator.rb +99 -0
  44. data/app/views/layouts/shiplog/application.html.erb +3 -0
  45. data/app/views/shiplog/_empty.html.erb +1 -1
  46. data/app/views/shiplog/_pagination.html.erb +3 -3
  47. data/app/views/shiplog/admin/_nav.html.erb +2 -1
  48. data/app/views/shiplog/admin/areas/index.html.erb +78 -0
  49. data/app/views/shiplog/admin/entries/_entry_row.html.erb +3 -0
  50. data/app/views/shiplog/admin/entries/_images.html.erb +41 -0
  51. data/app/views/shiplog/admin/entries/_translations.html.erb +42 -0
  52. data/app/views/shiplog/admin/entries/edit.html.erb +12 -0
  53. data/app/views/shiplog/changelog_entries/show.html.erb +25 -0
  54. data/app/views/shiplog/changelog_feeds/show.atom.builder +24 -0
  55. data/app/views/shiplog/changelogs/_areas.html.erb +8 -0
  56. data/app/views/shiplog/changelogs/_entry.html.erb +2 -5
  57. data/app/views/shiplog/changelogs/_entry_content.html.erb +10 -0
  58. data/app/views/shiplog/changelogs/_search.html.erb +4 -2
  59. data/app/views/shiplog/changelogs/show.html.erb +11 -4
  60. data/app/views/shiplog/roadmaps/_card.html.erb +2 -2
  61. data/app/views/shiplog/roadmaps/_column.html.erb +3 -3
  62. data/app/views/shiplog/roadmaps/_shipped.html.erb +1 -1
  63. data/app/views/shiplog/roadmaps/show.html.erb +2 -2
  64. data/config/admin_routes.rb +16 -0
  65. data/config/locales/en.yml +85 -0
  66. data/config/locales/es.yml +223 -0
  67. data/config/routes.rb +5 -0
  68. data/db/migrate/20261027000002_shiplog_0_3_0.rb +105 -0
  69. data/lib/generators/shiplog/install/install_generator.rb +10 -1
  70. data/lib/generators/shiplog/install/templates/initializer.rb.tt +45 -1
  71. data/lib/shiplog/classes.rb +10 -1
  72. data/lib/shiplog/configuration.rb +92 -3
  73. data/lib/shiplog/engine.rb +3 -0
  74. data/lib/shiplog/github/queries.rb +13 -1
  75. data/lib/shiplog/version.rb +1 -1
  76. data/lib/shiplog.rb +73 -0
  77. data/lib/tasks/shiplog.rake +29 -0
  78. metadata +39 -1
@@ -2,7 +2,11 @@ module Shiplog
2
2
  class Entry < ApplicationRecord
3
3
  class InvalidTransition < StandardError; end
4
4
 
5
+ include Images
6
+
5
7
  belongs_to :subject, polymorphic: true
8
+ belongs_to :area, class_name: "Shiplog::Area", optional: true
9
+ has_many :translations, class_name: "Shiplog::EntryTranslation", dependent: :destroy, inverse_of: :entry
6
10
 
7
11
  enum :kind, %w[roadmap feature change].index_by(&:itself), validate: true
8
12
  enum :status, %w[draft approved discarded].index_by(&:itself), default: "draft", validate: true
@@ -10,9 +14,14 @@ module Shiplog
10
14
  CHANGELOG_KINDS = %w[feature change].freeze
11
15
 
12
16
  validates :released_on, presence: true, if: -> { approved? && !roadmap? }
17
+ validate :area_exists
13
18
  validate :kind_change_allowed, on: :update
14
19
  validate :kind_free_for_subject, on: :update
15
20
 
21
+ # Approving an entry, or changing the English text of an approved one, translates it into every configured
22
+ # locale (see TranslateEntryJob). After the commit, so the job reads what was saved.
23
+ after_commit :enqueue_translations, on: :update, if: :translation_due?
24
+
16
25
  SEARCH_MAX_LENGTH = 100
17
26
  SEARCH_MAX_WORDS = 5
18
27
 
@@ -22,12 +31,99 @@ module Shiplog
22
31
  end
23
32
 
24
33
  # Entries whose title or body contains every word of the query (case-insensitive, LIKE wildcards escaped).
25
- # A blank query leaves the scope unchanged.
26
- def self.search(query)
34
+ # With a configured `locale` other than the default, a word also matches the current translation of that
35
+ # locale (the text the page shows), not a stale one. A blank query leaves the scope unchanged.
36
+ def self.search(query, locale: nil)
37
+ locale = locale.to_s.presence_in(Shiplog.config.locales.map(&:to_s))
27
38
  search_terms(query).reduce(all) do |scope, term|
28
39
  pattern = "%#{sanitize_sql_like(term)}%"
29
- scope.where("#{table_name}.title ILIKE :pattern OR #{table_name}.body ILIKE :pattern", pattern:)
40
+ english = "#{table_name}.title ILIKE :pattern OR #{table_name}.body ILIKE :pattern"
41
+ next scope.where(english, pattern:) unless locale
42
+
43
+ # One fragment, so the user's value is bound exactly once: the named binds are substituted in a single
44
+ # pass and the quoted text is never scanned again. Everything else in it is a constant.
45
+ translations = EntryTranslation.table_name
46
+ translated = "EXISTS (SELECT 1 FROM #{translations} WHERE #{translations}.entry_id = #{table_name}.id " \
47
+ "AND #{translations}.locale = :locale AND #{translations}.source_digest = #{TRANSLATION_DIGEST_SQL} " \
48
+ "AND (#{translations}.title ILIKE :pattern OR #{translations}.body ILIKE :pattern))"
49
+ scope.where("#{english} OR #{translated}", pattern:, locale:)
50
+ end
51
+ end
52
+
53
+ # Loads the translations of `entries` for the current locale in one query, and none under a locale that is
54
+ # not translated (then the entries show English and nothing is queried). Only that locale's rows are loaded
55
+ # into the `translations` association, so after this call it holds the current locale's rows, not all of them
56
+ # (use Entry#translation, not `translations`, to read them).
57
+ def self.preload_translations(entries, locale: I18n.locale)
58
+ entries = Array(entries).compact
59
+ return unless entries.any? && translated_locale?(locale)
60
+
61
+ ActiveRecord::Associations::Preloader.new(records: entries, associations: :translations,
62
+ scope: EntryTranslation.where(locale: locale.to_s)).call
63
+ end
64
+
65
+ # Whether pages in this locale show translations: it is one of config.locales.
66
+ def self.translated_locale?(locale) = Shiplog.config.locales.any? { |configured| configured.to_s == locale.to_s }
67
+
68
+ # The SQL twin of #translation_source_digest: SHA-256 of "title\nbody" (a missing text is empty), in hex.
69
+ TRANSLATION_DIGEST_SQL = "encode(sha256(convert_to(COALESCE(#{table_name}.title, '') || E'\\n' || COALESCE(#{table_name}.body, ''), 'UTF8')), 'hex')".freeze
70
+
71
+ UNSEEN_COUNT_CAP = 99
72
+ SLUG_MAX_LENGTH = 80
73
+ SLUG_ATTEMPTS = 5
74
+ # What a slug may look like (the permalink route's constraint, /[a-z0-9-]+/, is the unanchored twin).
75
+ SLUG_FORMAT = /\A[a-z0-9-]+\z/
76
+
77
+ # What the public changelog lists: approved features and changes with a release date.
78
+ scope :published_changelog, -> { approved.where(kind: CHANGELOG_KINDS).where.not(released_on: nil) }
79
+ # The changelog's order, newest first. A missing approved_at sorts first within its day, as PostgreSQL
80
+ # sorts NULLs in a descending order (the neighbour queries rely on it).
81
+ scope :newest_first, -> { order(released_on: :desc, approved_at: :desc, id: :desc) }
82
+
83
+ # The number of approved features and changes approved after `since` (all of them while it is nil), capped
84
+ # at 99. One COUNT over the (status, approved_at) index. It filters on approved_at, not on released_on (the
85
+ # list's order): "new since you last looked" is about when an entry appeared, so an entry approved today
86
+ # with an earlier release date still counts as new.
87
+ def self.unseen_count(since: nil)
88
+ scope = approved.where(kind: CHANGELOG_KINDS)
89
+ scope = scope.where("#{table_name}.approved_at > ?", since) if since
90
+ scope.limit(UNSEEN_COUNT_CAP).count
91
+ end
92
+
93
+ # Gives every published changelog entry that has no slug one (oldest first, so the oldest keeps the plain
94
+ # slug on duplicate titles). Returns how many it filled.
95
+ def self.backfill_slugs
96
+ published_changelog.where(slug: nil).order(:released_on, :approved_at, :id).to_a.count { |entry| entry.ensure_slug!.present? }
97
+ end
98
+
99
+ # Approved entries with no current translation into `locale`: none at all, or one made from older English text
100
+ # (compared with the SQL twin of the digest, in one query).
101
+ def self.missing_translation(locale)
102
+ translations = EntryTranslation.table_name
103
+ current = EntryTranslation.where(locale: locale.to_s).where("#{translations}.entry_id = #{table_name}.id")
104
+ .where("#{translations}.source_digest = #{TRANSLATION_DIGEST_SQL}")
105
+ approved.where(current.arel.exists.not)
106
+ end
107
+
108
+ # Enqueues a TranslateEntryJob for every approved entry and configured locale that has no current translation,
109
+ # reading the entries in batches. Repeating it is harmless: the job itself skips a translation that is current.
110
+ # Returns { approved: how many approved entries, enqueued: { locale => how many jobs } }.
111
+ def self.enqueue_missing_translations(batch_size: 100)
112
+ enqueued = Shiplog.config.locales.index_with do |locale|
113
+ count = 0
114
+ missing_translation(locale).find_each(batch_size:) do |entry|
115
+ TranslateEntryJob.perform_later(entry, locale.to_s)
116
+ count += 1
117
+ end
118
+ count
30
119
  end
120
+ { approved: approved.count, enqueued: }
121
+ end
122
+
123
+ # Whether the entry has a slug the permalink route accepts. Entries without one (approved by older code) or
124
+ # with a stray one render without a link rather than raising.
125
+ def permalink?
126
+ slug.is_a?(String) && SLUG_FORMAT.match?(slug)
31
127
  end
32
128
 
33
129
  def changelog_kind = kind
@@ -44,10 +140,30 @@ module Shiplog
44
140
 
45
141
  def approve!(by:)
46
142
  require_status!("approve", from: "draft")
47
- update!(status: "approved", approved_at: Time.current, approved_by: by, discard_reason: nil,
48
- released_on: released_on || Date.current)
143
+ retrying_on_slug_conflict do
144
+ update!(status: "approved", approved_at: Time.current, approved_by: by, discard_reason: nil,
145
+ released_on: released_on || Date.current, slug: slug_in_database || (generate_slug if permalinked?))
146
+ end
147
+ end
148
+
149
+ # The slug of a published changelog entry, generated and stored when it has none (entries approved before
150
+ # 0.3.0). Nil for entries without a permalink: drafts, discarded and roadmap entries. The write only happens
151
+ # while the slug is still null in the database, so a concurrent writer's slug is never overwritten; the
152
+ # entry is reloaded to return whichever slug won. Not used by the public pages (they must not write, a GET
153
+ # may hit a read replica): approve! and `rake shiplog:backfill_slugs` create slugs.
154
+ def ensure_slug!
155
+ return slug if slug.present?
156
+ return unless approved? && permalinked?
157
+
158
+ retrying_on_slug_conflict { self.class.where(id:, slug: nil).update_all(slug: generate_slug, updated_at: Time.current) }
159
+ reload.slug
49
160
  end
50
161
 
162
+ # The entries right after and before this one in the changelog's order, among the published ones.
163
+ def newer_changelog_entry = changelog_neighbour(">", :asc)
164
+
165
+ def older_changelog_entry = changelog_neighbour("<", :desc)
166
+
51
167
  def discard!(reason:)
52
168
  require_status!("discard", from: "draft")
53
169
  update!(status: "discarded", discard_reason: reason)
@@ -58,18 +174,101 @@ module Shiplog
58
174
  update!(status: "draft", discard_reason: nil)
59
175
  end
60
176
 
177
+ # The digest of the English title and body, stored in a translation made from them. A translation whose digest
178
+ # differs from it is stale.
179
+ def translation_source_digest
180
+ Digest::SHA256.hexdigest([ title, body ].join("\n"))
181
+ end
182
+
183
+ # The current translation for a locale (the request's by default): nil under a locale that is not configured,
184
+ # when none exists, and when it is stale. Reads the preloaded translations (see Entry.preload_translations)
185
+ # and asks the database only when they were not loaded.
186
+ def translation(locale = I18n.locale)
187
+ return unless self.class.translated_locale?(locale)
188
+
189
+ found = translations.loaded? ? translations.find { |record| record.locale == locale.to_s } : translations.find_by(locale: locale.to_s)
190
+ found if found && found.source_digest == translation_source_digest
191
+ end
192
+
193
+ # What public pages show: the translation for the request's locale when there is a current one, else English.
194
+ def display_title(locale = I18n.locale) = translation(locale)&.title || title
195
+
196
+ # The translated body, or the English one when the translation's body is blank and the English one is not
197
+ # (a blank body is never published in place of text).
198
+ def display_body(locale = I18n.locale)
199
+ translated = translation(locale)&.body
200
+ translated.blank? && body.present? ? body : (translated || body)
201
+ end
202
+
61
203
  def source_changed?
62
204
  source_digest.present? && source_digest != self.class.digest_for(subject)
63
205
  end
64
206
 
65
207
  private
66
208
 
209
+ def translation_due?
210
+ approved? && Shiplog.config.locales.any? && (saved_change_to_status? || saved_change_to_title? || saved_change_to_body?)
211
+ end
212
+
213
+ def enqueue_translations
214
+ Shiplog.config.locales.each { |locale| TranslateEntryJob.perform_later(self, locale) }
215
+ end
216
+
217
+ def permalinked? = CHANGELOG_KINDS.include?(kind)
218
+
219
+ # The title parameterized and cut at 80 characters, made unique with -2, -3, ...; entry-<id> when the title
220
+ # has nothing left. The uniqueness check can lose a race, which the unique index catches (see
221
+ # retrying_on_slug_conflict).
222
+ def generate_slug
223
+ base = dashed_title.first(SLUG_MAX_LENGTH).gsub(/\A-+|-+\z/, "").presence || "entry-#{id}"
224
+ candidate = base
225
+ suffix = 2
226
+ while self.class.where(slug: candidate).where.not(id:).exists?
227
+ tail = "-#{suffix}"
228
+ candidate = "#{base.first(SLUG_MAX_LENGTH - tail.length).delete_suffix("-")}#{tail}"
229
+ suffix += 1
230
+ end
231
+ candidate
232
+ end
233
+
234
+ # The title as [a-z0-9-] only: parameterized (which keeps underscores), underscores turned into dashes,
235
+ # repeated dashes squeezed, dashes at both ends stripped.
236
+ def dashed_title
237
+ title.to_s.parameterize.tr("_", "-").squeeze("-").gsub(/\A-+|-+\z/, "")
238
+ end
239
+
240
+ # Runs the block in a savepoint (a unique violation aborts the surrounding PostgreSQL transaction otherwise)
241
+ # and runs it again, with a fresh slug, when another entry took the slug first.
242
+ def retrying_on_slug_conflict
243
+ attempts = 0
244
+ begin
245
+ self.class.transaction(requires_new: true) { yield }
246
+ rescue ActiveRecord::RecordNotUnique
247
+ attempts += 1
248
+ retry if attempts < SLUG_ATTEMPTS
249
+ raise
250
+ end
251
+ end
252
+
253
+ # The published entry next to this one: `operator` compares the (released_on, approved_at, id) keys of
254
+ # the changelog order, with a missing approved_at as the largest value, as the descending order puts it.
255
+ def changelog_neighbour(operator, direction)
256
+ key = "(released_on, COALESCE(approved_at, 'infinity'::timestamp), id)"
257
+ self.class.published_changelog
258
+ .where("#{key} #{operator} (SELECT released_on, COALESCE(approved_at, 'infinity'::timestamp), id FROM #{self.class.table_name} WHERE id = ?)", id)
259
+ .reorder(released_on: direction, approved_at: direction, id: direction).first
260
+ end
261
+
67
262
  def kind_free_for_subject
68
263
  return unless will_save_change_to_kind? && self.class.where(subject_type:, subject_id:, kind:).where.not(id:).exists?
69
264
 
70
265
  errors.add(:changelog_kind, :taken)
71
266
  end
72
267
 
268
+ def area_exists
269
+ errors.add(:area, :invalid) if area_id && will_save_change_to_area_id? && area.nil?
270
+ end
271
+
73
272
  def kind_change_allowed
74
273
  return unless will_save_change_to_kind?
75
274
 
@@ -0,0 +1,15 @@
1
+ module Shiplog
2
+ # An entry's title and body in one locale (config.locales), published with the English text. `source_digest`
3
+ # is the digest of the English title and body it was made from (Entry#translation_source_digest); once the
4
+ # English text differs, the row is stale and the public pages show English instead. `edited_by_admin` marks a
5
+ # text an admin changed, which a new translation only replaces when the English text changed.
6
+ class EntryTranslation < ApplicationRecord
7
+ belongs_to :entry, class_name: "Shiplog::Entry", inverse_of: :translations
8
+
9
+ validates :locale, :title, :source_digest, presence: true
10
+ validates :locale, uniqueness: { scope: :entry_id }
11
+
12
+ # Whether the English text has changed since this translation was made.
13
+ def stale? = source_digest != entry.translation_source_digest
14
+ end
15
+ end
@@ -0,0 +1,98 @@
1
+ require "net/http"
2
+
3
+ module Shiplog
4
+ class ImageFetcher
5
+ # Downloads one allowed image URL: five seconds to connect and five to read, 15 seconds in total (checked
6
+ # on a monotonic clock while streaming and between redirect hops), at most three redirects (each hop is
7
+ # checked against the allowlist again), at most 5 MB (raw bytes: the response is requested uncompressed, and
8
+ # counted while streaming, the download stops past it). The body of a redirect is read with the same cap
9
+ # and thrown away. The GitHub token is sent to github.com only, never to another host. Returns a Download,
10
+ # or nil (logged) when anything goes wrong or the bytes aren't a PNG, JPEG, GIF or WebP of at most 10,000
11
+ # pixels a side and 40 megapixels (see ImageType).
12
+ class Downloader
13
+ TIMEOUT = 5
14
+ TOTAL_DEADLINE = 15
15
+ MAX_REDIRECTS = 3
16
+ Download = Data.define(:body, :content_type, :width, :height)
17
+ Rejected = Class.new(StandardError)
18
+
19
+ # Seconds on the monotonic clock, which wall-clock changes don't move.
20
+ def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
21
+
22
+ def initialize(token: nil)
23
+ @token = token
24
+ end
25
+
26
+ # `budget` is the seconds the caller has left; the deadline is the smaller of that and TOTAL_DEADLINE.
27
+ def call(url, budget: nil)
28
+ deadline = self.class.now + [ TOTAL_DEADLINE, budget ].compact.min
29
+ body = fetch(Sources.parse(url), MAX_REDIRECTS, deadline)
30
+ info = ImageType.analyze(body) # type by magic bytes, size from the header, both limits enforced
31
+
32
+ Download.new(body:, content_type: info.content_type, width: info.width, height: info.height)
33
+ rescue StandardError => error
34
+ Rails.logger&.warn("[shiplog] Skipped image #{safe(url)}: #{error.class}: #{error.message}")
35
+ nil
36
+ end
37
+
38
+ private
39
+
40
+ def fetch(uri, redirects_left, deadline)
41
+ raise Rejected, "host or scheme not allowed" unless uri && Sources.allowed?(uri)
42
+
43
+ check_deadline!(deadline)
44
+ body = +"".b
45
+ location = nil
46
+ http = Net::HTTP.new(uri.host, uri.port)
47
+ http.use_ssl = true
48
+ http.open_timeout = http.read_timeout = [ TIMEOUT, deadline - self.class.now ].min
49
+ http.request(build_request(uri)) do |response|
50
+ case response
51
+ when Net::HTTPSuccess then body = read(response, deadline)
52
+ when Net::HTTPRedirection
53
+ location = response["location"].to_s
54
+ read(response, deadline, keep: false)
55
+ else raise Rejected, "HTTP #{response.code}"
56
+ end
57
+ end
58
+ return body unless location
59
+
60
+ raise Rejected, "too many redirects" if redirects_left.zero?
61
+
62
+ fetch(Sources.parse(URI.join(uri.to_s, location).to_s), redirects_left - 1, deadline)
63
+ end
64
+
65
+ # Streams the body under the size cap and the deadline. Without `keep` the bytes are counted and dropped
66
+ # (Net::HTTP would otherwise read what is left of the body into memory, uncapped, when the block returns).
67
+ def read(response, deadline, keep: true)
68
+ raise Rejected, "larger than #{ImageType::MAX_BYTES} bytes" if response["content-length"].to_i > ImageType::MAX_BYTES
69
+
70
+ body = +"".b
71
+ size = 0
72
+ response.read_body do |chunk|
73
+ check_deadline!(deadline)
74
+ size += chunk.bytesize
75
+ raise Rejected, "larger than #{ImageType::MAX_BYTES} bytes" if size > ImageType::MAX_BYTES
76
+
77
+ body << chunk if keep
78
+ end
79
+ body
80
+ end
81
+
82
+ def check_deadline!(deadline)
83
+ raise Rejected, "deadline of #{TOTAL_DEADLINE} seconds exceeded" if self.class.now > deadline
84
+ end
85
+
86
+ def build_request(uri)
87
+ Net::HTTP::Get.new(uri).tap do |request|
88
+ request["User-Agent"] = "shiplog/#{Shiplog::VERSION}"
89
+ request["Accept-Encoding"] = "identity" # raw bytes only: the cap can't be overshot by inflation
90
+ request["Authorization"] = "Bearer #{@token}" if @token.present? && uri.host.to_s.downcase == Sources::ATTACHMENTS_HOST
91
+ end
92
+ end
93
+
94
+ # A URL for the log without its query string (signed URLs carry a credential there).
95
+ def safe(url) = url.to_s.sub(/[?#].*\z/m, "").first(200)
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,41 @@
1
+ require "nokogiri"
2
+
3
+ module Shiplog
4
+ class ImageFetcher
5
+ # Which images of a GitHub body (its rendered HTML) may be downloaded: <img src> URLs on GitHub's own image
6
+ # hosts (raw. and objects.githubusercontent.com too: GitHub serves attachments and redirects from them),
7
+ # https only, matched on the exact host (so user-images.githubusercontent.com.evil.com is out).
8
+ module Sources
9
+ HOSTS = %w[
10
+ private-user-images.githubusercontent.com user-images.githubusercontent.com camo.githubusercontent.com
11
+ raw.githubusercontent.com objects.githubusercontent.com
12
+ ].freeze
13
+ ATTACHMENTS_HOST = "github.com".freeze
14
+ # Exactly /user-attachments/assets/<id>: the id is letters, digits and dashes, so no dot, no slash and no
15
+ # percent-encoding (%2e%2e, %2f) can walk out of the directory.
16
+ ATTACHMENTS_PATH = %r{\A/user-attachments/assets/[A-Za-z0-9-]+\z}
17
+
18
+ # The allowed, distinct image URLs of the HTML, in document order, at most `limit`.
19
+ def self.urls(html, limit:)
20
+ Nokogiri::HTML.fragment(html.to_s).css("img[src]").filter_map { |img| parse(img["src"]) }
21
+ .select { |uri| allowed?(uri) }.map(&:to_s).uniq.first(limit)
22
+ end
23
+
24
+ # Whether a URI may be requested: also checked on every redirect hop.
25
+ def self.allowed?(uri)
26
+ return false unless uri.is_a?(URI::HTTPS) && uri.userinfo.nil? && uri.port == 443
27
+
28
+ host = uri.host.to_s.downcase
29
+ return true if HOSTS.include?(host)
30
+
31
+ host == ATTACHMENTS_HOST && ATTACHMENTS_PATH.match?(uri.path)
32
+ end
33
+
34
+ def self.parse(value)
35
+ URI.parse(value.to_s.strip)
36
+ rescue URI::InvalidURIError
37
+ nil
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,72 @@
1
+ module Shiplog
2
+ # Attaches the screenshots of a draft's GitHub issue or pull request to the entry. Runs when a draft is
3
+ # written (Shiplog::RewriteEntryJob), only for a draft without images: an approved entry's images are never
4
+ # changed automatically. Every failure (GitHub, network, validation) is logged and skipped; it never raises.
5
+ class ImageFetcher
6
+ # Seconds for all the images of one entry together; the rest are skipped (and logged) once it is spent.
7
+ FETCH_BUDGET = 45
8
+
9
+ def self.call(entry) = new(entry).call
10
+
11
+ def initialize(entry)
12
+ @entry = entry
13
+ end
14
+
15
+ # Returns how many images were attached.
16
+ def call
17
+ return 0 unless eligible?
18
+
19
+ html = body_html
20
+ urls = Sources.urls(html, limit: Shiplog.config.max_images_per_entry)
21
+ return 0 if urls.empty?
22
+
23
+ files = download_all(urls)
24
+ files.empty? ? 0 : attach(files)
25
+ rescue StandardError => error
26
+ Rails.logger&.warn("[shiplog] Fetching images for entry #{@entry.id} failed: #{error.class}: #{error.message}")
27
+ 0
28
+ end
29
+
30
+ private
31
+
32
+ # The files of the URLs that downloaded, in order. Once FETCH_BUDGET seconds are spent, the rest are skipped.
33
+ def download_all(urls)
34
+ downloader = Downloader.new(token: Shiplog.config.resolved_github_token)
35
+ started = Downloader.now
36
+ files = []
37
+ urls.each_with_index do |url, index|
38
+ remaining = FETCH_BUDGET - (Downloader.now - started)
39
+ if remaining <= 0
40
+ Rails.logger&.warn("[shiplog] Entry #{@entry.id}: the image budget of #{FETCH_BUDGET}s is spent, skipped #{urls.size - index} remaining image(s)")
41
+ break
42
+ end
43
+
44
+ download = downloader.call(url, budget: remaining)
45
+ files << file_for(download, index) if download
46
+ end
47
+ files
48
+ end
49
+
50
+ def file_for(download, index)
51
+ { io: StringIO.new(download.body), filename: "screenshot-#{index + 1}.#{ImageType::EXTENSIONS.fetch(download.content_type)}",
52
+ content_type: download.content_type, metadata: { width: download.width, height: download.height } }
53
+ end
54
+
55
+ def eligible?
56
+ Shiplog.images? && Shiplog.config.max_images_per_entry.positive? && @entry.draft? &&
57
+ Entry::CHANGELOG_KINDS.include?(@entry.kind) && !@entry.images.attached? && node_id.present?
58
+ end
59
+
60
+ def node_id = @entry.subject.try(:node_id)
61
+
62
+ def body_html
63
+ Github::Client.new.query(Github::Queries::BODY_HTML, id: node_id).dig("node", "bodyHTML")
64
+ end
65
+
66
+ # Uploaded before the entry's row lock is taken; under the lock, so two jobs cannot both attach, the state is
67
+ # checked again (still a draft, still without images), otherwise the uploaded blobs are purged.
68
+ def attach(files)
69
+ @entry.add_images(files) { @entry.draft? && !@entry.images.attached? }.size
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,111 @@
1
+ module Shiplog
2
+ # What an entry image may be: PNG, JPEG, GIF or WebP, recognised by the file's first bytes (never by its
3
+ # extension or a declared content type), at most 5 MB, at most 10,000 pixels on a side and 40 megapixels. The
4
+ # pixel size is read from the header, without decoding anything, before an image processor ever sees the file
5
+ # (a decompression bomb is a tiny file declaring a huge size).
6
+ module ImageType
7
+ MAX_BYTES = 5 * 1024 * 1024
8
+ MAX_SIDE = 10_000
9
+ MAX_PIXELS = 40_000_000
10
+ # Bytes needed to tell the types apart.
11
+ SNIFF_LENGTH = 12
12
+ EXTENSIONS = { "image/png" => "png", "image/jpeg" => "jpg", "image/gif" => "gif", "image/webp" => "webp" }.freeze
13
+
14
+ Invalid = Class.new(StandardError)
15
+ Info = Data.define(:content_type, :width, :height)
16
+
17
+ # JPEG frame headers (SOF0..SOF15) are the markers 0xC0..0xCF except these three, which are not frames.
18
+ JPEG_NOT_FRAME = [ 0xC4, 0xC8, 0xCC ].freeze # DHT, JPG, DAC
19
+
20
+ # The content type of the data starting with `head` (a binary String), or nil for anything else (SVG, HTML...).
21
+ def self.detect(head)
22
+ head = head.to_s.b
23
+ if head.start_with?("\x89PNG\r\n\x1A\n".b) then "image/png"
24
+ elsif head.start_with?("\xFF\xD8\xFF".b) then "image/jpeg"
25
+ elsif head.start_with?("GIF87a".b, "GIF89a".b) then "image/gif"
26
+ elsif head.bytesize >= SNIFF_LENGTH && head.byteslice(0, 4) == "RIFF".b && head.byteslice(8, 4) == "WEBP".b then "image/webp"
27
+ end
28
+ end
29
+
30
+ # The type and pixel size of the image `data` (a String of at most MAX_BYTES), or raises Invalid: not one of
31
+ # the four types, a size that can't be read from the header, or a size over the limits.
32
+ def self.analyze(data)
33
+ data = data.to_s.b
34
+ content_type = detect(data.byteslice(0, SNIFF_LENGTH)) or raise Invalid, "not a PNG, JPEG, GIF or WebP image"
35
+ width, height = dimensions(data, content_type)
36
+ raise Invalid, "the image's dimensions can't be read" unless width&.positive? && height&.positive?
37
+ raise Invalid, "#{width}x#{height} pixels is too large (at most #{MAX_SIDE} on a side and #{MAX_PIXELS / 1_000_000} megapixels)" if width > MAX_SIDE || height > MAX_SIDE || width * height > MAX_PIXELS
38
+
39
+ Info.new(content_type:, width:, height:)
40
+ end
41
+
42
+ # [width, height] read from the header of `data`, or nil when the header is truncated or malformed.
43
+ def self.dimensions(data, content_type)
44
+ case content_type
45
+ when "image/png" then png_dimensions(data)
46
+ when "image/gif" then gif_dimensions(data)
47
+ when "image/webp" then webp_dimensions(data)
48
+ when "image/jpeg" then jpeg_dimensions(data.byteslice(0, MAX_BYTES))
49
+ end
50
+ end
51
+
52
+ # The IHDR chunk comes first: 8 bytes of signature, length, "IHDR", then width and height (big-endian).
53
+ def self.png_dimensions(data)
54
+ data.byteslice(16, 8).unpack("NN") if data.bytesize >= 24 && data.byteslice(12, 4) == "IHDR".b
55
+ end
56
+
57
+ # The logical screen descriptor follows the 6-byte signature: width and height (little-endian).
58
+ def self.gif_dimensions(data)
59
+ data.byteslice(6, 4).unpack("vv") if data.bytesize >= 10
60
+ end
61
+
62
+ # The first chunk, at byte 12, says which kind of WebP it is.
63
+ def self.webp_dimensions(data)
64
+ case data.byteslice(12, 4)
65
+ when "VP8 ".b # lossy: frame tag (3 bytes), start code 9D 01 2A, 14-bit width and height
66
+ data.byteslice(26, 4).unpack("vv").map { |value| value & 0x3FFF } if data.bytesize >= 30 && data.byteslice(23, 3) == "\x9D\x01\x2A".b
67
+ when "VP8L".b # lossless: signature 0x2F, then 14 bits of width - 1 and 14 bits of height - 1
68
+ if data.bytesize >= 25 && data.getbyte(20) == 0x2F
69
+ bits = data.byteslice(21, 4).unpack1("V")
70
+ [ (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1 ]
71
+ end
72
+ when "VP8X".b # extended: flags, then the canvas size - 1 on 24 bits each
73
+ if data.bytesize >= 30
74
+ width, height = (data.byteslice(24, 3) + "\x00".b).unpack1("V"), (data.byteslice(27, 3) + "\x00".b).unpack1("V")
75
+ [ width + 1, height + 1 ]
76
+ end
77
+ end
78
+ end
79
+
80
+ # Walks the segments from the start until a frame header (SOF0..SOF15, but not DHT, DAC and JPG). Stops at the
81
+ # scan (SOS) or the end of the image: a frame header comes before both. Each step moves forward at least two
82
+ # bytes, over data already capped at MAX_BYTES.
83
+ def self.jpeg_dimensions(data)
84
+ position = 2
85
+ size = data.bytesize
86
+ while position < size
87
+ return unless data.getbyte(position) == 0xFF
88
+
89
+ position += 1 while position < size && data.getbyte(position) == 0xFF # fill bytes
90
+ marker = data.getbyte(position)
91
+ position += 1
92
+ return if marker.nil? || marker == 0xD9 || marker == 0xDA # end of image, start of scan
93
+ next if marker.zero? || marker == 0x01 || marker.between?(0xD0, 0xD8) # no length
94
+
95
+ return if position + 2 > size
96
+
97
+ length = data.byteslice(position, 2).unpack1("n")
98
+ return if length < 2
99
+
100
+ if marker.between?(0xC0, 0xCF) && !JPEG_NOT_FRAME.include?(marker)
101
+ return unless position + 7 <= size
102
+
103
+ height, width = data.byteslice(position + 3, 4).unpack("nn")
104
+ return [ width, height ]
105
+ end
106
+ position += length
107
+ end
108
+ nil
109
+ end
110
+ end
111
+ end