shiplog 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +59 -0
- data/README.md +140 -5
- data/app/controllers/shiplog/admin/area_detections_controller.rb +15 -0
- data/app/controllers/shiplog/admin/area_translations_controller.rb +19 -0
- data/app/controllers/shiplog/admin/areas/merges_controller.rb +19 -0
- data/app/controllers/shiplog/admin/areas/moves_controller.rb +13 -0
- data/app/controllers/shiplog/admin/areas_controller.rb +48 -0
- data/app/controllers/shiplog/admin/entries/images/moves_controller.rb +20 -0
- data/app/controllers/shiplog/admin/entries/images_controller.rb +76 -0
- data/app/controllers/shiplog/admin/entries/translations/retranslations_controller.rb +23 -0
- data/app/controllers/shiplog/admin/entries/translations_controller.rb +35 -0
- data/app/controllers/shiplog/admin/entries_controller.rb +9 -3
- data/app/controllers/shiplog/changelog_entries_controller.rb +26 -0
- data/app/controllers/shiplog/changelog_feeds_controller.rb +19 -0
- data/app/controllers/shiplog/changelogs_controller.rb +21 -5
- data/app/controllers/shiplog/public_controller.rb +55 -0
- data/app/controllers/shiplog/roadmaps_controller.rb +3 -1
- data/app/frontend/shiplog/admin.css +9 -0
- data/app/frontend/shiplog/shiplog.css +47 -2
- data/app/helpers/shiplog/admin/entries_helper.rb +15 -0
- data/app/helpers/shiplog/permalink_helper.rb +113 -0
- data/app/helpers/shiplog/roadmap_helper.rb +9 -0
- data/app/helpers/shiplog/unseen_helper.rb +9 -0
- data/app/jobs/shiplog/detect_areas_job.rb +18 -0
- data/app/jobs/shiplog/rewrite_entry_job.rb +4 -1
- data/app/jobs/shiplog/translate_area_job.rb +25 -0
- data/app/jobs/shiplog/translate_entry_job.rb +25 -0
- data/app/models/shiplog/area.rb +156 -0
- data/app/models/shiplog/area_detector.rb +146 -0
- data/app/models/shiplog/area_translator.rb +77 -0
- data/app/models/shiplog/entry/images.rb +127 -0
- data/app/models/shiplog/entry.rb +204 -5
- data/app/models/shiplog/entry_translation.rb +15 -0
- data/app/models/shiplog/image_fetcher/downloader.rb +98 -0
- data/app/models/shiplog/image_fetcher/sources.rb +41 -0
- data/app/models/shiplog/image_fetcher.rb +72 -0
- data/app/models/shiplog/image_type.rb +111 -0
- data/app/models/shiplog/rewriter/prompt.rb +31 -7
- data/app/models/shiplog/rewriter.rb +41 -23
- data/app/models/shiplog/setting.rb +19 -0
- data/app/models/shiplog/translator/prompt.rb +40 -0
- data/app/models/shiplog/translator.rb +99 -0
- data/app/views/layouts/shiplog/application.html.erb +3 -0
- data/app/views/shiplog/_empty.html.erb +1 -1
- data/app/views/shiplog/_pagination.html.erb +3 -3
- data/app/views/shiplog/admin/_nav.html.erb +2 -1
- data/app/views/shiplog/admin/areas/index.html.erb +78 -0
- data/app/views/shiplog/admin/entries/_entry_row.html.erb +3 -0
- data/app/views/shiplog/admin/entries/_images.html.erb +41 -0
- data/app/views/shiplog/admin/entries/_translations.html.erb +42 -0
- data/app/views/shiplog/admin/entries/edit.html.erb +12 -0
- data/app/views/shiplog/changelog_entries/show.html.erb +25 -0
- data/app/views/shiplog/changelog_feeds/show.atom.builder +24 -0
- data/app/views/shiplog/changelogs/_areas.html.erb +8 -0
- data/app/views/shiplog/changelogs/_entry.html.erb +2 -5
- data/app/views/shiplog/changelogs/_entry_content.html.erb +10 -0
- data/app/views/shiplog/changelogs/_search.html.erb +4 -2
- data/app/views/shiplog/changelogs/show.html.erb +11 -4
- data/app/views/shiplog/roadmaps/_card.html.erb +2 -2
- data/app/views/shiplog/roadmaps/_column.html.erb +3 -3
- data/app/views/shiplog/roadmaps/_shipped.html.erb +1 -1
- data/app/views/shiplog/roadmaps/show.html.erb +2 -2
- data/config/admin_routes.rb +16 -0
- data/config/locales/en.yml +85 -0
- data/config/locales/es.yml +223 -0
- data/config/routes.rb +5 -0
- data/db/migrate/20261027000002_shiplog_0_3_0.rb +105 -0
- data/lib/generators/shiplog/install/install_generator.rb +10 -1
- data/lib/generators/shiplog/install/templates/initializer.rb.tt +45 -1
- data/lib/shiplog/classes.rb +10 -1
- data/lib/shiplog/configuration.rb +92 -3
- data/lib/shiplog/engine.rb +3 -0
- data/lib/shiplog/github/queries.rb +13 -1
- data/lib/shiplog/version.rb +1 -1
- data/lib/shiplog.rb +73 -0
- data/lib/tasks/shiplog.rake +29 -0
- metadata +39 -1
data/app/models/shiplog/entry.rb
CHANGED
|
@@ -2,7 +2,11 @@ module Shiplog
|
|
|
2
2
|
class Entry < ApplicationRecord
|
|
3
3
|
class InvalidTransition < StandardError; end
|
|
4
4
|
|
|
5
|
+
include Images
|
|
6
|
+
|
|
5
7
|
belongs_to :subject, polymorphic: true
|
|
8
|
+
belongs_to :area, class_name: "Shiplog::Area", optional: true
|
|
9
|
+
has_many :translations, class_name: "Shiplog::EntryTranslation", dependent: :destroy, inverse_of: :entry
|
|
6
10
|
|
|
7
11
|
enum :kind, %w[roadmap feature change].index_by(&:itself), validate: true
|
|
8
12
|
enum :status, %w[draft approved discarded].index_by(&:itself), default: "draft", validate: true
|
|
@@ -10,9 +14,14 @@ module Shiplog
|
|
|
10
14
|
CHANGELOG_KINDS = %w[feature change].freeze
|
|
11
15
|
|
|
12
16
|
validates :released_on, presence: true, if: -> { approved? && !roadmap? }
|
|
17
|
+
validate :area_exists
|
|
13
18
|
validate :kind_change_allowed, on: :update
|
|
14
19
|
validate :kind_free_for_subject, on: :update
|
|
15
20
|
|
|
21
|
+
# Approving an entry, or changing the English text of an approved one, translates it into every configured
|
|
22
|
+
# locale (see TranslateEntryJob). After the commit, so the job reads what was saved.
|
|
23
|
+
after_commit :enqueue_translations, on: :update, if: :translation_due?
|
|
24
|
+
|
|
16
25
|
SEARCH_MAX_LENGTH = 100
|
|
17
26
|
SEARCH_MAX_WORDS = 5
|
|
18
27
|
|
|
@@ -22,12 +31,99 @@ module Shiplog
|
|
|
22
31
|
end
|
|
23
32
|
|
|
24
33
|
# Entries whose title or body contains every word of the query (case-insensitive, LIKE wildcards escaped).
|
|
25
|
-
#
|
|
26
|
-
|
|
34
|
+
# With a configured `locale` other than the default, a word also matches the current translation of that
|
|
35
|
+
# locale (the text the page shows), not a stale one. A blank query leaves the scope unchanged.
|
|
36
|
+
def self.search(query, locale: nil)
|
|
37
|
+
locale = locale.to_s.presence_in(Shiplog.config.locales.map(&:to_s))
|
|
27
38
|
search_terms(query).reduce(all) do |scope, term|
|
|
28
39
|
pattern = "%#{sanitize_sql_like(term)}%"
|
|
29
|
-
|
|
40
|
+
english = "#{table_name}.title ILIKE :pattern OR #{table_name}.body ILIKE :pattern"
|
|
41
|
+
next scope.where(english, pattern:) unless locale
|
|
42
|
+
|
|
43
|
+
# One fragment, so the user's value is bound exactly once: the named binds are substituted in a single
|
|
44
|
+
# pass and the quoted text is never scanned again. Everything else in it is a constant.
|
|
45
|
+
translations = EntryTranslation.table_name
|
|
46
|
+
translated = "EXISTS (SELECT 1 FROM #{translations} WHERE #{translations}.entry_id = #{table_name}.id " \
|
|
47
|
+
"AND #{translations}.locale = :locale AND #{translations}.source_digest = #{TRANSLATION_DIGEST_SQL} " \
|
|
48
|
+
"AND (#{translations}.title ILIKE :pattern OR #{translations}.body ILIKE :pattern))"
|
|
49
|
+
scope.where("#{english} OR #{translated}", pattern:, locale:)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Loads the translations of `entries` for the current locale in one query, and none under a locale that is
|
|
54
|
+
# not translated (then the entries show English and nothing is queried). Only that locale's rows are loaded
|
|
55
|
+
# into the `translations` association, so after this call it holds the current locale's rows, not all of them
|
|
56
|
+
# (use Entry#translation, not `translations`, to read them).
|
|
57
|
+
def self.preload_translations(entries, locale: I18n.locale)
|
|
58
|
+
entries = Array(entries).compact
|
|
59
|
+
return unless entries.any? && translated_locale?(locale)
|
|
60
|
+
|
|
61
|
+
ActiveRecord::Associations::Preloader.new(records: entries, associations: :translations,
|
|
62
|
+
scope: EntryTranslation.where(locale: locale.to_s)).call
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Whether pages in this locale show translations: it is one of config.locales.
|
|
66
|
+
def self.translated_locale?(locale) = Shiplog.config.locales.any? { |configured| configured.to_s == locale.to_s }
|
|
67
|
+
|
|
68
|
+
# The SQL twin of #translation_source_digest: SHA-256 of "title\nbody" (a missing text is empty), in hex.
|
|
69
|
+
TRANSLATION_DIGEST_SQL = "encode(sha256(convert_to(COALESCE(#{table_name}.title, '') || E'\\n' || COALESCE(#{table_name}.body, ''), 'UTF8')), 'hex')".freeze
|
|
70
|
+
|
|
71
|
+
UNSEEN_COUNT_CAP = 99
|
|
72
|
+
SLUG_MAX_LENGTH = 80
|
|
73
|
+
SLUG_ATTEMPTS = 5
|
|
74
|
+
# What a slug may look like (the permalink route's constraint, /[a-z0-9-]+/, is the unanchored twin).
|
|
75
|
+
SLUG_FORMAT = /\A[a-z0-9-]+\z/
|
|
76
|
+
|
|
77
|
+
# What the public changelog lists: approved features and changes with a release date.
|
|
78
|
+
scope :published_changelog, -> { approved.where(kind: CHANGELOG_KINDS).where.not(released_on: nil) }
|
|
79
|
+
# The changelog's order, newest first. A missing approved_at sorts first within its day, as PostgreSQL
|
|
80
|
+
# sorts NULLs in a descending order (the neighbour queries rely on it).
|
|
81
|
+
scope :newest_first, -> { order(released_on: :desc, approved_at: :desc, id: :desc) }
|
|
82
|
+
|
|
83
|
+
# The number of approved features and changes approved after `since` (all of them while it is nil), capped
|
|
84
|
+
# at 99. One COUNT over the (status, approved_at) index. It filters on approved_at, not on released_on (the
|
|
85
|
+
# list's order): "new since you last looked" is about when an entry appeared, so an entry approved today
|
|
86
|
+
# with an earlier release date still counts as new.
|
|
87
|
+
def self.unseen_count(since: nil)
|
|
88
|
+
scope = approved.where(kind: CHANGELOG_KINDS)
|
|
89
|
+
scope = scope.where("#{table_name}.approved_at > ?", since) if since
|
|
90
|
+
scope.limit(UNSEEN_COUNT_CAP).count
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Gives every published changelog entry that has no slug one (oldest first, so the oldest keeps the plain
|
|
94
|
+
# slug on duplicate titles). Returns how many it filled.
|
|
95
|
+
def self.backfill_slugs
|
|
96
|
+
published_changelog.where(slug: nil).order(:released_on, :approved_at, :id).to_a.count { |entry| entry.ensure_slug!.present? }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Approved entries with no current translation into `locale`: none at all, or one made from older English text
|
|
100
|
+
# (compared with the SQL twin of the digest, in one query).
|
|
101
|
+
def self.missing_translation(locale)
|
|
102
|
+
translations = EntryTranslation.table_name
|
|
103
|
+
current = EntryTranslation.where(locale: locale.to_s).where("#{translations}.entry_id = #{table_name}.id")
|
|
104
|
+
.where("#{translations}.source_digest = #{TRANSLATION_DIGEST_SQL}")
|
|
105
|
+
approved.where(current.arel.exists.not)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Enqueues a TranslateEntryJob for every approved entry and configured locale that has no current translation,
|
|
109
|
+
# reading the entries in batches. Repeating it is harmless: the job itself skips a translation that is current.
|
|
110
|
+
# Returns { approved: how many approved entries, enqueued: { locale => how many jobs } }.
|
|
111
|
+
def self.enqueue_missing_translations(batch_size: 100)
|
|
112
|
+
enqueued = Shiplog.config.locales.index_with do |locale|
|
|
113
|
+
count = 0
|
|
114
|
+
missing_translation(locale).find_each(batch_size:) do |entry|
|
|
115
|
+
TranslateEntryJob.perform_later(entry, locale.to_s)
|
|
116
|
+
count += 1
|
|
117
|
+
end
|
|
118
|
+
count
|
|
30
119
|
end
|
|
120
|
+
{ approved: approved.count, enqueued: }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Whether the entry has a slug the permalink route accepts. Entries without one (approved by older code) or
|
|
124
|
+
# with a stray one render without a link rather than raising.
|
|
125
|
+
def permalink?
|
|
126
|
+
slug.is_a?(String) && SLUG_FORMAT.match?(slug)
|
|
31
127
|
end
|
|
32
128
|
|
|
33
129
|
def changelog_kind = kind
|
|
@@ -44,10 +140,30 @@ module Shiplog
|
|
|
44
140
|
|
|
45
141
|
def approve!(by:)
|
|
46
142
|
require_status!("approve", from: "draft")
|
|
47
|
-
|
|
48
|
-
|
|
143
|
+
retrying_on_slug_conflict do
|
|
144
|
+
update!(status: "approved", approved_at: Time.current, approved_by: by, discard_reason: nil,
|
|
145
|
+
released_on: released_on || Date.current, slug: slug_in_database || (generate_slug if permalinked?))
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The slug of a published changelog entry, generated and stored when it has none (entries approved before
|
|
150
|
+
# 0.3.0). Nil for entries without a permalink: drafts, discarded and roadmap entries. The write only happens
|
|
151
|
+
# while the slug is still null in the database, so a concurrent writer's slug is never overwritten; the
|
|
152
|
+
# entry is reloaded to return whichever slug won. Not used by the public pages (they must not write, a GET
|
|
153
|
+
# may hit a read replica): approve! and `rake shiplog:backfill_slugs` create slugs.
|
|
154
|
+
def ensure_slug!
|
|
155
|
+
return slug if slug.present?
|
|
156
|
+
return unless approved? && permalinked?
|
|
157
|
+
|
|
158
|
+
retrying_on_slug_conflict { self.class.where(id:, slug: nil).update_all(slug: generate_slug, updated_at: Time.current) }
|
|
159
|
+
reload.slug
|
|
49
160
|
end
|
|
50
161
|
|
|
162
|
+
# The entries right after and before this one in the changelog's order, among the published ones.
|
|
163
|
+
def newer_changelog_entry = changelog_neighbour(">", :asc)
|
|
164
|
+
|
|
165
|
+
def older_changelog_entry = changelog_neighbour("<", :desc)
|
|
166
|
+
|
|
51
167
|
def discard!(reason:)
|
|
52
168
|
require_status!("discard", from: "draft")
|
|
53
169
|
update!(status: "discarded", discard_reason: reason)
|
|
@@ -58,18 +174,101 @@ module Shiplog
|
|
|
58
174
|
update!(status: "draft", discard_reason: nil)
|
|
59
175
|
end
|
|
60
176
|
|
|
177
|
+
# The digest of the English title and body, stored in a translation made from them. A translation whose digest
|
|
178
|
+
# differs from it is stale.
|
|
179
|
+
def translation_source_digest
|
|
180
|
+
Digest::SHA256.hexdigest([ title, body ].join("\n"))
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# The current translation for a locale (the request's by default): nil under a locale that is not configured,
|
|
184
|
+
# when none exists, and when it is stale. Reads the preloaded translations (see Entry.preload_translations)
|
|
185
|
+
# and asks the database only when they were not loaded.
|
|
186
|
+
def translation(locale = I18n.locale)
|
|
187
|
+
return unless self.class.translated_locale?(locale)
|
|
188
|
+
|
|
189
|
+
found = translations.loaded? ? translations.find { |record| record.locale == locale.to_s } : translations.find_by(locale: locale.to_s)
|
|
190
|
+
found if found && found.source_digest == translation_source_digest
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# What public pages show: the translation for the request's locale when there is a current one, else English.
|
|
194
|
+
def display_title(locale = I18n.locale) = translation(locale)&.title || title
|
|
195
|
+
|
|
196
|
+
# The translated body, or the English one when the translation's body is blank and the English one is not
|
|
197
|
+
# (a blank body is never published in place of text).
|
|
198
|
+
def display_body(locale = I18n.locale)
|
|
199
|
+
translated = translation(locale)&.body
|
|
200
|
+
translated.blank? && body.present? ? body : (translated || body)
|
|
201
|
+
end
|
|
202
|
+
|
|
61
203
|
def source_changed?
|
|
62
204
|
source_digest.present? && source_digest != self.class.digest_for(subject)
|
|
63
205
|
end
|
|
64
206
|
|
|
65
207
|
private
|
|
66
208
|
|
|
209
|
+
def translation_due?
|
|
210
|
+
approved? && Shiplog.config.locales.any? && (saved_change_to_status? || saved_change_to_title? || saved_change_to_body?)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def enqueue_translations
|
|
214
|
+
Shiplog.config.locales.each { |locale| TranslateEntryJob.perform_later(self, locale) }
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def permalinked? = CHANGELOG_KINDS.include?(kind)
|
|
218
|
+
|
|
219
|
+
# The title parameterized and cut at 80 characters, made unique with -2, -3, ...; entry-<id> when the title
|
|
220
|
+
# has nothing left. The uniqueness check can lose a race, which the unique index catches (see
|
|
221
|
+
# retrying_on_slug_conflict).
|
|
222
|
+
def generate_slug
|
|
223
|
+
base = dashed_title.first(SLUG_MAX_LENGTH).gsub(/\A-+|-+\z/, "").presence || "entry-#{id}"
|
|
224
|
+
candidate = base
|
|
225
|
+
suffix = 2
|
|
226
|
+
while self.class.where(slug: candidate).where.not(id:).exists?
|
|
227
|
+
tail = "-#{suffix}"
|
|
228
|
+
candidate = "#{base.first(SLUG_MAX_LENGTH - tail.length).delete_suffix("-")}#{tail}"
|
|
229
|
+
suffix += 1
|
|
230
|
+
end
|
|
231
|
+
candidate
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# The title as [a-z0-9-] only: parameterized (which keeps underscores), underscores turned into dashes,
|
|
235
|
+
# repeated dashes squeezed, dashes at both ends stripped.
|
|
236
|
+
def dashed_title
|
|
237
|
+
title.to_s.parameterize.tr("_", "-").squeeze("-").gsub(/\A-+|-+\z/, "")
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Runs the block in a savepoint (a unique violation aborts the surrounding PostgreSQL transaction otherwise)
|
|
241
|
+
# and runs it again, with a fresh slug, when another entry took the slug first.
|
|
242
|
+
def retrying_on_slug_conflict
|
|
243
|
+
attempts = 0
|
|
244
|
+
begin
|
|
245
|
+
self.class.transaction(requires_new: true) { yield }
|
|
246
|
+
rescue ActiveRecord::RecordNotUnique
|
|
247
|
+
attempts += 1
|
|
248
|
+
retry if attempts < SLUG_ATTEMPTS
|
|
249
|
+
raise
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# The published entry next to this one: `operator` compares the (released_on, approved_at, id) keys of
|
|
254
|
+
# the changelog order, with a missing approved_at as the largest value, as the descending order puts it.
|
|
255
|
+
def changelog_neighbour(operator, direction)
|
|
256
|
+
key = "(released_on, COALESCE(approved_at, 'infinity'::timestamp), id)"
|
|
257
|
+
self.class.published_changelog
|
|
258
|
+
.where("#{key} #{operator} (SELECT released_on, COALESCE(approved_at, 'infinity'::timestamp), id FROM #{self.class.table_name} WHERE id = ?)", id)
|
|
259
|
+
.reorder(released_on: direction, approved_at: direction, id: direction).first
|
|
260
|
+
end
|
|
261
|
+
|
|
67
262
|
def kind_free_for_subject
|
|
68
263
|
return unless will_save_change_to_kind? && self.class.where(subject_type:, subject_id:, kind:).where.not(id:).exists?
|
|
69
264
|
|
|
70
265
|
errors.add(:changelog_kind, :taken)
|
|
71
266
|
end
|
|
72
267
|
|
|
268
|
+
def area_exists
|
|
269
|
+
errors.add(:area, :invalid) if area_id && will_save_change_to_area_id? && area.nil?
|
|
270
|
+
end
|
|
271
|
+
|
|
73
272
|
def kind_change_allowed
|
|
74
273
|
return unless will_save_change_to_kind?
|
|
75
274
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
module Shiplog
|
|
2
|
+
# An entry's title and body in one locale (config.locales), published with the English text. `source_digest`
|
|
3
|
+
# is the digest of the English title and body it was made from (Entry#translation_source_digest); once the
|
|
4
|
+
# English text differs, the row is stale and the public pages show English instead. `edited_by_admin` marks a
|
|
5
|
+
# text an admin changed, which a new translation only replaces when the English text changed.
|
|
6
|
+
class EntryTranslation < ApplicationRecord
|
|
7
|
+
belongs_to :entry, class_name: "Shiplog::Entry", inverse_of: :translations
|
|
8
|
+
|
|
9
|
+
validates :locale, :title, :source_digest, presence: true
|
|
10
|
+
validates :locale, uniqueness: { scope: :entry_id }
|
|
11
|
+
|
|
12
|
+
# Whether the English text has changed since this translation was made.
|
|
13
|
+
def stale? = source_digest != entry.translation_source_digest
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
require "net/http"
|
|
2
|
+
|
|
3
|
+
module Shiplog
|
|
4
|
+
class ImageFetcher
|
|
5
|
+
# Downloads one allowed image URL: five seconds to connect and five to read, 15 seconds in total (checked
|
|
6
|
+
# on a monotonic clock while streaming and between redirect hops), at most three redirects (each hop is
|
|
7
|
+
# checked against the allowlist again), at most 5 MB (raw bytes: the response is requested uncompressed, and
|
|
8
|
+
# counted while streaming, the download stops past it). The body of a redirect is read with the same cap
|
|
9
|
+
# and thrown away. The GitHub token is sent to github.com only, never to another host. Returns a Download,
|
|
10
|
+
# or nil (logged) when anything goes wrong or the bytes aren't a PNG, JPEG, GIF or WebP of at most 10,000
|
|
11
|
+
# pixels a side and 40 megapixels (see ImageType).
|
|
12
|
+
class Downloader
|
|
13
|
+
TIMEOUT = 5
|
|
14
|
+
TOTAL_DEADLINE = 15
|
|
15
|
+
MAX_REDIRECTS = 3
|
|
16
|
+
Download = Data.define(:body, :content_type, :width, :height)
|
|
17
|
+
Rejected = Class.new(StandardError)
|
|
18
|
+
|
|
19
|
+
# Seconds on the monotonic clock, which wall-clock changes don't move.
|
|
20
|
+
def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
21
|
+
|
|
22
|
+
def initialize(token: nil)
|
|
23
|
+
@token = token
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# `budget` is the seconds the caller has left; the deadline is the smaller of that and TOTAL_DEADLINE.
|
|
27
|
+
def call(url, budget: nil)
|
|
28
|
+
deadline = self.class.now + [ TOTAL_DEADLINE, budget ].compact.min
|
|
29
|
+
body = fetch(Sources.parse(url), MAX_REDIRECTS, deadline)
|
|
30
|
+
info = ImageType.analyze(body) # type by magic bytes, size from the header, both limits enforced
|
|
31
|
+
|
|
32
|
+
Download.new(body:, content_type: info.content_type, width: info.width, height: info.height)
|
|
33
|
+
rescue StandardError => error
|
|
34
|
+
Rails.logger&.warn("[shiplog] Skipped image #{safe(url)}: #{error.class}: #{error.message}")
|
|
35
|
+
nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def fetch(uri, redirects_left, deadline)
|
|
41
|
+
raise Rejected, "host or scheme not allowed" unless uri && Sources.allowed?(uri)
|
|
42
|
+
|
|
43
|
+
check_deadline!(deadline)
|
|
44
|
+
body = +"".b
|
|
45
|
+
location = nil
|
|
46
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
47
|
+
http.use_ssl = true
|
|
48
|
+
http.open_timeout = http.read_timeout = [ TIMEOUT, deadline - self.class.now ].min
|
|
49
|
+
http.request(build_request(uri)) do |response|
|
|
50
|
+
case response
|
|
51
|
+
when Net::HTTPSuccess then body = read(response, deadline)
|
|
52
|
+
when Net::HTTPRedirection
|
|
53
|
+
location = response["location"].to_s
|
|
54
|
+
read(response, deadline, keep: false)
|
|
55
|
+
else raise Rejected, "HTTP #{response.code}"
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
return body unless location
|
|
59
|
+
|
|
60
|
+
raise Rejected, "too many redirects" if redirects_left.zero?
|
|
61
|
+
|
|
62
|
+
fetch(Sources.parse(URI.join(uri.to_s, location).to_s), redirects_left - 1, deadline)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Streams the body under the size cap and the deadline. Without `keep` the bytes are counted and dropped
|
|
66
|
+
# (Net::HTTP would otherwise read what is left of the body into memory, uncapped, when the block returns).
|
|
67
|
+
def read(response, deadline, keep: true)
|
|
68
|
+
raise Rejected, "larger than #{ImageType::MAX_BYTES} bytes" if response["content-length"].to_i > ImageType::MAX_BYTES
|
|
69
|
+
|
|
70
|
+
body = +"".b
|
|
71
|
+
size = 0
|
|
72
|
+
response.read_body do |chunk|
|
|
73
|
+
check_deadline!(deadline)
|
|
74
|
+
size += chunk.bytesize
|
|
75
|
+
raise Rejected, "larger than #{ImageType::MAX_BYTES} bytes" if size > ImageType::MAX_BYTES
|
|
76
|
+
|
|
77
|
+
body << chunk if keep
|
|
78
|
+
end
|
|
79
|
+
body
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def check_deadline!(deadline)
|
|
83
|
+
raise Rejected, "deadline of #{TOTAL_DEADLINE} seconds exceeded" if self.class.now > deadline
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def build_request(uri)
|
|
87
|
+
Net::HTTP::Get.new(uri).tap do |request|
|
|
88
|
+
request["User-Agent"] = "shiplog/#{Shiplog::VERSION}"
|
|
89
|
+
request["Accept-Encoding"] = "identity" # raw bytes only: the cap can't be overshot by inflation
|
|
90
|
+
request["Authorization"] = "Bearer #{@token}" if @token.present? && uri.host.to_s.downcase == Sources::ATTACHMENTS_HOST
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# A URL for the log without its query string (signed URLs carry a credential there).
|
|
95
|
+
def safe(url) = url.to_s.sub(/[?#].*\z/m, "").first(200)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
require "nokogiri"
|
|
2
|
+
|
|
3
|
+
module Shiplog
|
|
4
|
+
class ImageFetcher
|
|
5
|
+
# Which images of a GitHub body (its rendered HTML) may be downloaded: <img src> URLs on GitHub's own image
|
|
6
|
+
# hosts (raw. and objects.githubusercontent.com too: GitHub serves attachments and redirects from them),
|
|
7
|
+
# https only, matched on the exact host (so user-images.githubusercontent.com.evil.com is out).
|
|
8
|
+
module Sources
|
|
9
|
+
HOSTS = %w[
|
|
10
|
+
private-user-images.githubusercontent.com user-images.githubusercontent.com camo.githubusercontent.com
|
|
11
|
+
raw.githubusercontent.com objects.githubusercontent.com
|
|
12
|
+
].freeze
|
|
13
|
+
ATTACHMENTS_HOST = "github.com".freeze
|
|
14
|
+
# Exactly /user-attachments/assets/<id>: the id is letters, digits and dashes, so no dot, no slash and no
|
|
15
|
+
# percent-encoding (%2e%2e, %2f) can walk out of the directory.
|
|
16
|
+
ATTACHMENTS_PATH = %r{\A/user-attachments/assets/[A-Za-z0-9-]+\z}
|
|
17
|
+
|
|
18
|
+
# The allowed, distinct image URLs of the HTML, in document order, at most `limit`.
|
|
19
|
+
def self.urls(html, limit:)
|
|
20
|
+
Nokogiri::HTML.fragment(html.to_s).css("img[src]").filter_map { |img| parse(img["src"]) }
|
|
21
|
+
.select { |uri| allowed?(uri) }.map(&:to_s).uniq.first(limit)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Whether a URI may be requested: also checked on every redirect hop.
|
|
25
|
+
def self.allowed?(uri)
|
|
26
|
+
return false unless uri.is_a?(URI::HTTPS) && uri.userinfo.nil? && uri.port == 443
|
|
27
|
+
|
|
28
|
+
host = uri.host.to_s.downcase
|
|
29
|
+
return true if HOSTS.include?(host)
|
|
30
|
+
|
|
31
|
+
host == ATTACHMENTS_HOST && ATTACHMENTS_PATH.match?(uri.path)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.parse(value)
|
|
35
|
+
URI.parse(value.to_s.strip)
|
|
36
|
+
rescue URI::InvalidURIError
|
|
37
|
+
nil
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
module Shiplog
|
|
2
|
+
# Attaches the screenshots of a draft's GitHub issue or pull request to the entry. Runs when a draft is
|
|
3
|
+
# written (Shiplog::RewriteEntryJob), only for a draft without images: an approved entry's images are never
|
|
4
|
+
# changed automatically. Every failure (GitHub, network, validation) is logged and skipped; it never raises.
|
|
5
|
+
class ImageFetcher
|
|
6
|
+
# Seconds for all the images of one entry together; the rest are skipped (and logged) once it is spent.
|
|
7
|
+
FETCH_BUDGET = 45
|
|
8
|
+
|
|
9
|
+
def self.call(entry) = new(entry).call
|
|
10
|
+
|
|
11
|
+
def initialize(entry)
|
|
12
|
+
@entry = entry
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Returns how many images were attached.
|
|
16
|
+
def call
|
|
17
|
+
return 0 unless eligible?
|
|
18
|
+
|
|
19
|
+
html = body_html
|
|
20
|
+
urls = Sources.urls(html, limit: Shiplog.config.max_images_per_entry)
|
|
21
|
+
return 0 if urls.empty?
|
|
22
|
+
|
|
23
|
+
files = download_all(urls)
|
|
24
|
+
files.empty? ? 0 : attach(files)
|
|
25
|
+
rescue StandardError => error
|
|
26
|
+
Rails.logger&.warn("[shiplog] Fetching images for entry #{@entry.id} failed: #{error.class}: #{error.message}")
|
|
27
|
+
0
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
# The files of the URLs that downloaded, in order. Once FETCH_BUDGET seconds are spent, the rest are skipped.
|
|
33
|
+
def download_all(urls)
|
|
34
|
+
downloader = Downloader.new(token: Shiplog.config.resolved_github_token)
|
|
35
|
+
started = Downloader.now
|
|
36
|
+
files = []
|
|
37
|
+
urls.each_with_index do |url, index|
|
|
38
|
+
remaining = FETCH_BUDGET - (Downloader.now - started)
|
|
39
|
+
if remaining <= 0
|
|
40
|
+
Rails.logger&.warn("[shiplog] Entry #{@entry.id}: the image budget of #{FETCH_BUDGET}s is spent, skipped #{urls.size - index} remaining image(s)")
|
|
41
|
+
break
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
download = downloader.call(url, budget: remaining)
|
|
45
|
+
files << file_for(download, index) if download
|
|
46
|
+
end
|
|
47
|
+
files
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def file_for(download, index)
|
|
51
|
+
{ io: StringIO.new(download.body), filename: "screenshot-#{index + 1}.#{ImageType::EXTENSIONS.fetch(download.content_type)}",
|
|
52
|
+
content_type: download.content_type, metadata: { width: download.width, height: download.height } }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def eligible?
|
|
56
|
+
Shiplog.images? && Shiplog.config.max_images_per_entry.positive? && @entry.draft? &&
|
|
57
|
+
Entry::CHANGELOG_KINDS.include?(@entry.kind) && !@entry.images.attached? && node_id.present?
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def node_id = @entry.subject.try(:node_id)
|
|
61
|
+
|
|
62
|
+
def body_html
|
|
63
|
+
Github::Client.new.query(Github::Queries::BODY_HTML, id: node_id).dig("node", "bodyHTML")
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Uploaded before the entry's row lock is taken; under the lock, so two jobs cannot both attach, the state is
|
|
67
|
+
# checked again (still a draft, still without images), otherwise the uploaded blobs are purged.
|
|
68
|
+
def attach(files)
|
|
69
|
+
@entry.add_images(files) { @entry.draft? && !@entry.images.attached? }.size
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
module Shiplog
|
|
2
|
+
# What an entry image may be: PNG, JPEG, GIF or WebP, recognised by the file's first bytes (never by its
|
|
3
|
+
# extension or a declared content type), at most 5 MB, at most 10,000 pixels on a side and 40 megapixels. The
|
|
4
|
+
# pixel size is read from the header, without decoding anything, before an image processor ever sees the file
|
|
5
|
+
# (a decompression bomb is a tiny file declaring a huge size).
|
|
6
|
+
module ImageType
|
|
7
|
+
MAX_BYTES = 5 * 1024 * 1024
|
|
8
|
+
MAX_SIDE = 10_000
|
|
9
|
+
MAX_PIXELS = 40_000_000
|
|
10
|
+
# Bytes needed to tell the types apart.
|
|
11
|
+
SNIFF_LENGTH = 12
|
|
12
|
+
EXTENSIONS = { "image/png" => "png", "image/jpeg" => "jpg", "image/gif" => "gif", "image/webp" => "webp" }.freeze
|
|
13
|
+
|
|
14
|
+
Invalid = Class.new(StandardError)
|
|
15
|
+
Info = Data.define(:content_type, :width, :height)
|
|
16
|
+
|
|
17
|
+
# JPEG frame headers (SOF0..SOF15) are the markers 0xC0..0xCF except these three, which are not frames.
|
|
18
|
+
JPEG_NOT_FRAME = [ 0xC4, 0xC8, 0xCC ].freeze # DHT, JPG, DAC
|
|
19
|
+
|
|
20
|
+
# The content type of the data starting with `head` (a binary String), or nil for anything else (SVG, HTML...).
|
|
21
|
+
def self.detect(head)
|
|
22
|
+
head = head.to_s.b
|
|
23
|
+
if head.start_with?("\x89PNG\r\n\x1A\n".b) then "image/png"
|
|
24
|
+
elsif head.start_with?("\xFF\xD8\xFF".b) then "image/jpeg"
|
|
25
|
+
elsif head.start_with?("GIF87a".b, "GIF89a".b) then "image/gif"
|
|
26
|
+
elsif head.bytesize >= SNIFF_LENGTH && head.byteslice(0, 4) == "RIFF".b && head.byteslice(8, 4) == "WEBP".b then "image/webp"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The type and pixel size of the image `data` (a String of at most MAX_BYTES), or raises Invalid: not one of
|
|
31
|
+
# the four types, a size that can't be read from the header, or a size over the limits.
|
|
32
|
+
def self.analyze(data)
|
|
33
|
+
data = data.to_s.b
|
|
34
|
+
content_type = detect(data.byteslice(0, SNIFF_LENGTH)) or raise Invalid, "not a PNG, JPEG, GIF or WebP image"
|
|
35
|
+
width, height = dimensions(data, content_type)
|
|
36
|
+
raise Invalid, "the image's dimensions can't be read" unless width&.positive? && height&.positive?
|
|
37
|
+
raise Invalid, "#{width}x#{height} pixels is too large (at most #{MAX_SIDE} on a side and #{MAX_PIXELS / 1_000_000} megapixels)" if width > MAX_SIDE || height > MAX_SIDE || width * height > MAX_PIXELS
|
|
38
|
+
|
|
39
|
+
Info.new(content_type:, width:, height:)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# [width, height] read from the header of `data`, or nil when the header is truncated or malformed.
|
|
43
|
+
def self.dimensions(data, content_type)
|
|
44
|
+
case content_type
|
|
45
|
+
when "image/png" then png_dimensions(data)
|
|
46
|
+
when "image/gif" then gif_dimensions(data)
|
|
47
|
+
when "image/webp" then webp_dimensions(data)
|
|
48
|
+
when "image/jpeg" then jpeg_dimensions(data.byteslice(0, MAX_BYTES))
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The IHDR chunk comes first: 8 bytes of signature, length, "IHDR", then width and height (big-endian).
|
|
53
|
+
def self.png_dimensions(data)
|
|
54
|
+
data.byteslice(16, 8).unpack("NN") if data.bytesize >= 24 && data.byteslice(12, 4) == "IHDR".b
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The logical screen descriptor follows the 6-byte signature: width and height (little-endian).
|
|
58
|
+
def self.gif_dimensions(data)
|
|
59
|
+
data.byteslice(6, 4).unpack("vv") if data.bytesize >= 10
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The first chunk, at byte 12, says which kind of WebP it is.
|
|
63
|
+
def self.webp_dimensions(data)
|
|
64
|
+
case data.byteslice(12, 4)
|
|
65
|
+
when "VP8 ".b # lossy: frame tag (3 bytes), start code 9D 01 2A, 14-bit width and height
|
|
66
|
+
data.byteslice(26, 4).unpack("vv").map { |value| value & 0x3FFF } if data.bytesize >= 30 && data.byteslice(23, 3) == "\x9D\x01\x2A".b
|
|
67
|
+
when "VP8L".b # lossless: signature 0x2F, then 14 bits of width - 1 and 14 bits of height - 1
|
|
68
|
+
if data.bytesize >= 25 && data.getbyte(20) == 0x2F
|
|
69
|
+
bits = data.byteslice(21, 4).unpack1("V")
|
|
70
|
+
[ (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1 ]
|
|
71
|
+
end
|
|
72
|
+
when "VP8X".b # extended: flags, then the canvas size - 1 on 24 bits each
|
|
73
|
+
if data.bytesize >= 30
|
|
74
|
+
width, height = (data.byteslice(24, 3) + "\x00".b).unpack1("V"), (data.byteslice(27, 3) + "\x00".b).unpack1("V")
|
|
75
|
+
[ width + 1, height + 1 ]
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Walks the segments from the start until a frame header (SOF0..SOF15, but not DHT, DAC and JPG). Stops at the
|
|
81
|
+
# scan (SOS) or the end of the image: a frame header comes before both. Each step moves forward at least two
|
|
82
|
+
# bytes, over data already capped at MAX_BYTES.
|
|
83
|
+
def self.jpeg_dimensions(data)
|
|
84
|
+
position = 2
|
|
85
|
+
size = data.bytesize
|
|
86
|
+
while position < size
|
|
87
|
+
return unless data.getbyte(position) == 0xFF
|
|
88
|
+
|
|
89
|
+
position += 1 while position < size && data.getbyte(position) == 0xFF # fill bytes
|
|
90
|
+
marker = data.getbyte(position)
|
|
91
|
+
position += 1
|
|
92
|
+
return if marker.nil? || marker == 0xD9 || marker == 0xDA # end of image, start of scan
|
|
93
|
+
next if marker.zero? || marker == 0x01 || marker.between?(0xD0, 0xD8) # no length
|
|
94
|
+
|
|
95
|
+
return if position + 2 > size
|
|
96
|
+
|
|
97
|
+
length = data.byteslice(position, 2).unpack1("n")
|
|
98
|
+
return if length < 2
|
|
99
|
+
|
|
100
|
+
if marker.between?(0xC0, 0xCF) && !JPEG_NOT_FRAME.include?(marker)
|
|
101
|
+
return unless position + 7 <= size
|
|
102
|
+
|
|
103
|
+
height, width = data.byteslice(position + 3, 4).unpack("nn")
|
|
104
|
+
return [ width, height ]
|
|
105
|
+
end
|
|
106
|
+
position += length
|
|
107
|
+
end
|
|
108
|
+
nil
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|