maptain 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,357 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ class Builder
5
+ # full: whether this build rewrote everything, asked to or because rebuild_every had passed.
6
+ Result = Data.define(:full, :written, :kept, :deleted, :url_count, :submitted, :warnings) do
7
+ def changed?
8
+ written.any? || deleted.any?
9
+ end
10
+ end
11
+
12
+ # https://www.sitemaps.org/protocol.html
13
+ MAX_URL_CHARACTERS = 2_048
14
+ MAX_FILES_PER_INDEX = 50_000
15
+ LAST_TIME_ZONE_OFFSET_SECONDS = 12 * 60 * 60
16
+ DAY_LENGTH = "2026-01-01".length
17
+ # One request's worth: what a refusing endpoint leaves in the manifest stays small.
18
+ MAX_UNSUBMITTED_URLS = IndexNow::MAX_URLS_PER_REQUEST
19
+ # A language, then maybe a script and a region: en, pt-BR, zh-Hant-TW, es-419.
20
+ HREFLANG = /\A[a-z]{2,3}(-[a-z]{4})?(-([a-z]{2}|\d{3}))?\z/i
21
+
22
+ Rendered = Data.define(:source_slug, :shard, :fingerprint, :files, :changed_urls)
23
+
24
+ # others: how many more times its source made the same mistake.
25
+ Warning = Data.define(:source, :url, :message, :others) do
26
+ def to_s
27
+ line = "#{source}: #{url} #{message}"
28
+ others.zero? ? line : "#{line}, and #{others} more like it"
29
+ end
30
+ end
31
+
32
+ def initialize(configuration, storage: configuration.storage, submit: true)
33
+ @configuration = configuration
34
+ @storage = storage
35
+ @submit = submit
36
+ end
37
+
38
+ def build(full: false)
39
+ ActiveSupport::Notifications.instrument("build.maptain", full:) do |payload|
40
+ payload[:result] = Dir.mktmpdir("maptain") { |directory| build_in(directory, full:) }
41
+ payload[:full] = payload[:result].full
42
+ payload[:result]
43
+ end
44
+ end
45
+
46
+ private
47
+
48
+ def build_in(directory, full:)
49
+ @configuration.validate!
50
+ @directory = directory
51
+ @previous = Manifest.read(@storage, manifest_name)
52
+ @previous_file_by_name = @previous.files.to_h { |file| [file["name"], file] }
53
+ built_at = Time.now
54
+ @full = full || rebuild_due?(built_at)
55
+ @reusable = !@full && @previous.digest == @configuration.digest
56
+ @manifest = Manifest.new(digest: @configuration.digest, built_at:, rebuilt_at: @full ? built_at : @previous.rebuilt_at)
57
+ @written = []
58
+ @kept = []
59
+ @warnings = []
60
+ @changed_urls = []
61
+ @checked_hreflangs = Set.new
62
+ @reported = Set.new
63
+ @warning_index_by_mistake = {}
64
+ @x_default = Xml.hreflang(@configuration.x_default)
65
+ @hosts = @configuration.hosts
66
+ @host_prefixes = @hosts.map { |host| "#{host}/" }
67
+ @outside_message = "is outside #{@hosts.first}#{" and the hosts of its other languages" if @hosts.size > 1}"
68
+ @index_now = IndexNow.new(key: @configuration.index_now_key, key_path: @configuration.index_now_key_location)
69
+
70
+ @rendered = []
71
+ # The first build has no "before": submitting a whole site at once is not what IndexNow is for. Nor is a
72
+ # new configuration, which rewrites every file: then only the lastmods tell what changed.
73
+ @tracking = @submit && @configuration.index_now_key && @previous.built_at.present?
74
+ @diffing = @tracking && @previous.digest == @configuration.digest
75
+ @moved_sources = Set.new
76
+
77
+ @configuration.sources.each { |source_class| build_source(source_class.new(@configuration)) }
78
+ publish_rendered
79
+ publish_index
80
+ deleted = (@previous.file_names | @previous.pending) - @manifest.file_names
81
+ @changed_urls.concat(urls_in(deleted.reject { |name| @moved_sources.include?(previous_source_of(name)) })) if @diffing
82
+ deleted.each { |name| @storage.delete(name) }
83
+ publish_index_now_key
84
+ urls = urls_to_submit
85
+ # Written before submitting, with the URLs still to send: a build that dies while submitting loses none.
86
+ @manifest.unsubmitted.replace(urls.last(MAX_UNSUBMITTED_URLS))
87
+ upload(manifest_name, @manifest.to_json)
88
+ submitted = submit(urls)
89
+
90
+ Result.new(full: @full, written: @written, kept: @kept, deleted:, url_count: @manifest.files.sum { |file| file["urls"] },
91
+ submitted:, warnings: @warnings)
92
+ end
93
+
94
+ def build_source(source)
95
+ seen_urls = Set.new
96
+ source.fingerprint_by_shard.sort_by { |shard, _fingerprint| shard.to_i }.each do |shard, fingerprint|
97
+ stored = @previous.shard(source.slug, shard) if @reusable
98
+ if fingerprint && stored && stored["fingerprint"] == fingerprint
99
+ keep(source, shard, stored)
100
+ else
101
+ render(source, shard, fingerprint, seen_urls)
102
+ end
103
+ end
104
+ end
105
+
106
+ def keep(source, shard, stored)
107
+ @manifest.record(source.slug, shard, fingerprint: stored["fingerprint"], files: stored["files"])
108
+ @kept.concat(stored["files"].map { |file| file["name"] })
109
+ end
110
+
111
+ def render(source, shard, fingerprint, seen_urls)
112
+ writer = ShardWriter.new(directory: @directory, basename: source.basename(shard))
113
+ changed_urls = []
114
+ urls = [] if @diffing
115
+ ActiveSupport::Notifications.instrument("shard.maptain", source: source.slug, shard:) do
116
+ source.each_page(shard) do |page|
117
+ inspect_page(source, page, seen_urls)
118
+ writer.add(Xml.url_entries(page, x_default: @x_default), page.lastmod)
119
+ changed_urls.concat(page.urls) if @tracking && changed_since_last_build?(page)
120
+ urls&.concat(page.urls)
121
+ end
122
+ writer.close
123
+ end
124
+
125
+ if writer.files.all? { |file| same_as_before?(file) }
126
+ changed_urls.clear
127
+ elsif urls
128
+ changed_urls.concat(appeared_or_gone(source.slug, shard, fingerprint, urls))
129
+ end
130
+ fingerprint ||= writer.content_digest
131
+ # Recorded now, published once every shard is rendered: the index keeps the order of the sources.
132
+ @manifest.record(source.slug, shard, fingerprint:, files: [])
133
+ @rendered << Rendered.new(source_slug: source.slug, shard:, fingerprint:, files: writer.files, changed_urls:)
134
+ end
135
+
136
+ # Read while the previous files are still in the storage: a page that appeared or went away is a change too,
137
+ # whatever its lastmod says. Not when the ranges moved, which only carries pages from one file to another.
138
+ def appeared_or_gone(source_slug, shard, fingerprint, urls)
139
+ stored = @previous.shard(source_slug, shard)
140
+ if stored && fingerprint && stored["fingerprint"].to_s.split("/").first != fingerprint.split("/").first
141
+ @moved_sources << source_slug
142
+ return []
143
+ end
144
+
145
+ previous_urls = urls_in((stored&.fetch("files") || []).map { |file| file["name"] }).to_set
146
+ urls = urls.to_set
147
+ (urls - previous_urls).to_a + (previous_urls - urls).to_a
148
+ end
149
+
150
+ def publish_rendered
151
+ changed_names = @rendered.flat_map(&:files).reject { |file| same_as_before?(file) }.to_set(&:name)
152
+ announce_new_files(changed_names)
153
+ @rendered.each do |rendered|
154
+ @changed_urls.concat(rendered.changed_urls)
155
+ entries = rendered.files.map do |file|
156
+ changed = changed_names.include?(file.name)
157
+ publish(file, changed)
158
+ manifest_entry(file, changed)
159
+ end
160
+ @manifest.record(rendered.source_slug, rendered.shard, fingerprint: rendered.fingerprint, files: entries)
161
+ end
162
+ end
163
+
164
+ def previous_source_of(file_name)
165
+ @previous_source_by_file_name ||= @previous.shards_by_source.each_with_object({}) do |(source_slug, shards), source_by_name|
166
+ shards.each_value { |stored| stored["files"].each { |file| source_by_name[file["name"]] = source_slug } }
167
+ end
168
+ @previous_source_by_file_name[file_name]
169
+ end
170
+
171
+ def urls_in(names)
172
+ names.flat_map do |name|
173
+ content = @storage.read(name)
174
+ content ? Xml.locations(Zlib.gunzip(content.b)) : []
175
+ rescue Zlib::Error
176
+ []
177
+ end
178
+ end
179
+
180
+ def same_as_before?(file)
181
+ @previous_file_by_name.dig(file.name, "digest") == file.digest
182
+ end
183
+
184
+ # Recorded before the first upload, so that a build dying midway leaves no file the next one cannot remove.
185
+ def announce_new_files(names)
186
+ new_names = names.to_a - @previous_file_by_name.keys - @previous.pending
187
+ return if new_names.empty?
188
+
189
+ @previous.pending.concat(new_names)
190
+ upload(manifest_name, @previous.to_json)
191
+ end
192
+
193
+ # A full build uploads every file, so that it also repairs a storage that lost one.
194
+ def publish(file, changed)
195
+ if changed || @full
196
+ @storage.write(file.name, file.path)
197
+ @written << file.name
198
+ else
199
+ @kept << file.name
200
+ end
201
+ File.delete(file.path)
202
+ end
203
+
204
+ # The index tells crawlers which files to read again: a file that changed gets a later date even when the
205
+ # change is a removal, and one that did not keeps its date.
206
+ def manifest_entry(file, changed)
207
+ previous_lastmod = @previous_file_by_name.dig(file.name, "lastmod")
208
+ lastmod = if !changed
209
+ previous_lastmod
210
+ elsif @previous_file_by_name.key?(file.name) && !later?(file.lastmod, previous_lastmod)
211
+ @manifest.built_at.getutc.iso8601
212
+ else
213
+ file.lastmod
214
+ end
215
+ {"name" => file.name, "urls" => file.url_count, "lastmod" => lastmod, "digest" => file.digest}
216
+ end
217
+
218
+ # A manifest written by an older version may hold a lastmod in another format: compare times, not text.
219
+ def later?(stamp, previous_stamp)
220
+ return false if stamp.nil?
221
+
222
+ previous_stamp = Xml.timestamp(previous_stamp)
223
+ previous_stamp.nil? || Xml.time(stamp) > Xml.time(previous_stamp)
224
+ end
225
+
226
+ def rebuild_due?(now)
227
+ every = @configuration.rebuild_every
228
+ every.present? && (@previous.rebuilt_at.nil? || @previous.rebuilt_at + every <= now)
229
+ end
230
+
231
+ def publish_index
232
+ name = "#{@configuration.filename}.xml"
233
+ entries = @manifest.files.map do |file|
234
+ Xml.sitemap_entry("#{@configuration.sitemaps_url}/#{file["name"]}", file["lastmod"])
235
+ end
236
+ inspect_index("#{@configuration.sitemaps_url}/#{name}", entries.size)
237
+ upload(name, "#{Xml::INDEX_OPEN}#{entries.join}#{Xml::INDEX_CLOSE}")
238
+ end
239
+
240
+ def upload(name, content)
241
+ path = File.join(@directory, name)
242
+ File.write(path, content)
243
+ @storage.write(name, path)
244
+ end
245
+
246
+ def manifest_name
247
+ "#{@configuration.filename}-manifest.json"
248
+ end
249
+
250
+ def inspect_page(source, page, seen_urls)
251
+ page.urls.each do |url|
252
+ warn_counted(source, :outside, url, @outside_message) unless on_host?(url)
253
+ warn_counted(source, :listed_twice, url, "is listed twice") unless seen_urls.add?(url)
254
+ warn_counted(source, :long, url, "is longer than #{MAX_URL_CHARACTERS} characters") if url.length > MAX_URL_CHARACTERS
255
+ end
256
+ page.url_by_locale.each do |hreflang, url|
257
+ warn(source, url, "has hreflang #{hreflang.inspect}, not a language code") if @checked_hreflangs.add?(hreflang) && !hreflang.match?(HREFLANG)
258
+ end
259
+ if page.url_by_locale.size > 1 && page.urls.one?
260
+ warn_once(source, :one_url, page.urls.first, "is the URL of every language of its page: does the url block use the locale?")
261
+ end
262
+ inspect_lastmod(source, page)
263
+ end
264
+
265
+ def inspect_lastmod(source, page)
266
+ if page.rejected_lastmod
267
+ warn_once(source, :rejected_lastmod, page.urls.first, "has a lastmod that is not a W3C date (#{page.rejected_lastmod.inspect}), left out")
268
+ elsif page.modified_at && page.modified_at > @manifest.built_at
269
+ warn_counted(source, :future_lastmod, page.urls.first, "has a lastmod in the future (#{page.lastmod})")
270
+ end
271
+ end
272
+
273
+ def inspect_index(url, file_count)
274
+ return if file_count <= MAX_FILES_PER_INDEX
275
+
276
+ warn("index", url, "lists #{file_count} files, more than #{MAX_FILES_PER_INDEX}: raise shard_size")
277
+ end
278
+
279
+ # For a mistake every page of a source would repeat.
280
+ def warn_once(source, kind, url, message)
281
+ warn(source, url, message) if @reported.add?([source.slug, kind])
282
+ end
283
+
284
+ # For a mistake any number of URLs of a source may share: the first is reported, the others are counted.
285
+ def warn_counted(source, kind, url, message)
286
+ mistake = [source.slug, kind]
287
+ index = @warning_index_by_mistake[mistake]
288
+ if index
289
+ @warnings[index] = @warnings[index].with(others: @warnings[index].others + 1)
290
+ else
291
+ @warning_index_by_mistake[mistake] = @warnings.size
292
+ warn(source, url, message)
293
+ end
294
+ end
295
+
296
+ def warn(source, url, message)
297
+ @warnings << Warning.new(source: source.is_a?(Source) ? source.slug : source, url:, message:, others: 0)
298
+ end
299
+
300
+ def on_host?(url)
301
+ url.start_with?(*@host_prefixes) || @hosts.include?(url)
302
+ end
303
+
304
+ def host_of(url)
305
+ @hosts.find { |host| url == host || url.start_with?("#{host}/") }
306
+ end
307
+
308
+ # A day may still be the day of the last build somewhere: it changed if it is not older than the date at
309
+ # UTC-12, the last time zone to leave a day. A month or a year alone says too little to tell.
310
+ def changed_since_last_build?(page)
311
+ return false unless page.lastmod
312
+ return page.modified_at >= @previous.built_at if page.lastmod.include?("T")
313
+
314
+ page.lastmod.length == DAY_LENGTH && page.lastmod >= (@previous.built_at - LAST_TIME_ZONE_OFFSET_SECONDS).utc.to_date.iso8601
315
+ end
316
+
317
+ def urls_to_submit
318
+ return [] unless @submit && @configuration.index_now_key
319
+
320
+ (@previous.unsubmitted + @changed_urls).uniq.select { |url| on_host?(url) }
321
+ end
322
+
323
+ # Returns how many URLs the endpoint took. The others, refused or unreachable, stay in the manifest for the
324
+ # next build: the sitemaps are published, so a refusal never fails the build.
325
+ def submit(urls)
326
+ return 0 if urls.empty?
327
+
328
+ replies = urls.group_by { |url| host_of(url) }.flat_map { |host, urls_of_host| submit_to(host, urls_of_host) }
329
+ @manifest.unsubmitted.replace(replies.reject(&:accepted?).flat_map(&:urls).last(MAX_UNSUBMITTED_URLS))
330
+ upload(manifest_name, @manifest.to_json)
331
+ replies.select(&:accepted?).sum { |reply| reply.urls.size }
332
+ end
333
+
334
+ def submit_to(host, urls)
335
+ replies = @index_now.submit(host, urls)
336
+ replies.reject(&:accepted?).group_by(&:status).each do |status, refused|
337
+ count = refused.sum { |reply| reply.urls.size }
338
+ answer = status.is_a?(Integer) ? "answered #{status}" : "could not be reached (#{status})"
339
+ warn("indexnow", IndexNow::ENDPOINT, "#{answer} for #{count} URLs of #{host}, sent again with the next build")
340
+ end
341
+ replies
342
+ end
343
+
344
+ # IndexNow reads the key at the root of a host unless told where it is, and only on that host.
345
+ def publish_index_now_key
346
+ key = @configuration.index_now_key
347
+ return if key.nil? || @configuration.index_now_key_location
348
+
349
+ if @configuration.sitemaps_url == @configuration.host!
350
+ upload("#{key}.txt", key)
351
+ else
352
+ warn("indexnow", "#{@configuration.host!}/#{key}.txt", "must hold the key: sitemaps go to #{@configuration.sitemaps_url}, " \
353
+ "so put #{key}.txt in public/ and set index_now_key_location to \"/#{key}.txt\"")
354
+ end
355
+ end
356
+ end
357
+ end
@@ -0,0 +1,100 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ class Configuration
5
+ # A full build now and then catches what no fingerprint sees: update_columns, a change in another table,
6
+ # a new url block, a file removed from the storage.
7
+ DEFAULT_REBUILD_EVERY_SECONDS = 7 * 24 * 60 * 60
8
+
9
+ attr_accessor :filename, :load_sources, :rebuild_every, :storage, :x_default
10
+ attr_reader :host, :host_by_locale, :index_now_key, :index_now_key_location
11
+ attr_writer :locales, :sitemaps_url, :sources
12
+
13
+ def initialize
14
+ @filename = "sitemap"
15
+ @host_by_locale = {}.freeze
16
+ @load_sources = -> {}
17
+ @rebuild_every = DEFAULT_REBUILD_EVERY_SECONDS
18
+ @storage = Storage::Local.new("public")
19
+ end
20
+
21
+ def host=(url)
22
+ @host = absolute_url(url, "host")
23
+ end
24
+
25
+ def host!
26
+ host || raise(ConfigurationError, "Maptain.configuration.host is not set")
27
+ end
28
+
29
+ # Frozen: a host added in place would be neither checked nor found.
30
+ def host_by_locale=(host_by_locale)
31
+ @host_by_locale = host_by_locale.to_h do |locale, url|
32
+ [Xml.hreflang(locale), absolute_url(url, "host_by_locale[#{locale.inspect}]")]
33
+ end.freeze
34
+ end
35
+
36
+ def host_for(locale)
37
+ host_by_locale.fetch(Xml.hreflang(locale)) { host! }
38
+ end
39
+
40
+ def hosts
41
+ [host!, *host_by_locale.values].uniq
42
+ end
43
+
44
+ # Checked when a build starts, not when host_by_locale is set: locales may be set after it.
45
+ def validate!
46
+ hreflangs = locales.map { |locale| Xml.hreflang(locale) }
47
+ unknown = host_by_locale.keys - hreflangs
48
+ return if unknown.empty?
49
+
50
+ raise ConfigurationError, "host_by_locale[#{unknown.first.inspect}] is not one of the locales: #{hreflangs.join(", ")}"
51
+ end
52
+
53
+ # A blank key, as an environment variable left empty gives, is no key.
54
+ def index_now_key=(key)
55
+ @index_now_key = key.presence
56
+ end
57
+
58
+ # A path, not a URL: IndexNow only reads a key on the host of the URLs it is given, so each host serves it.
59
+ def index_now_key_location=(path)
60
+ path = path.to_s.presence
61
+ unless path.nil? || path.match?(%r{\A/[^/\s]\S*\z})
62
+ raise ConfigurationError, "index_now_key_location must be a path such as \"/key.txt\", got #{path.inspect}"
63
+ end
64
+
65
+ @index_now_key_location = path
66
+ end
67
+
68
+ def locales
69
+ Array(@locales || I18n.available_locales)
70
+ end
71
+
72
+ # Where the sitemap files are served from, when that is not the site itself (a bucket, a CDN).
73
+ def sitemaps_url
74
+ @sitemaps_url ? absolute_url(@sitemaps_url, "sitemaps_url") : host!
75
+ end
76
+
77
+ def sources
78
+ return @sources if @sources
79
+
80
+ load_sources.call
81
+ Source.descendants.select(&:scope?).sort_by(&:slug)
82
+ end
83
+
84
+ # Whatever changes every file: a build under another digest starts over.
85
+ def digest
86
+ Digest::SHA256.hexdigest(JSON.generate([VERSION, host!, host_by_locale, sitemaps_url, locales, x_default, filename]))
87
+ end
88
+
89
+ private
90
+
91
+ def absolute_url(url, setting)
92
+ uri = URI.parse(url.to_s)
93
+ raise ConfigurationError, "#{setting} must be an http(s) URL, got #{url.inspect}" unless uri.is_a?(URI::HTTP) && uri.host.present?
94
+
95
+ url.to_s.delete_suffix("/")
96
+ rescue URI::InvalidURIError
97
+ raise ConfigurationError, "#{setting} must be an http(s) URL, got #{url.inspect}"
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ class Error < StandardError; end
5
+
6
+ class ConfigurationError < Error; end
7
+
8
+ class SourceError < Error; end
9
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ # Tells Bing, Yandex, Seznam and Naver which URLs changed: https://www.indexnow.org/documentation
5
+ class IndexNow
6
+ ENDPOINT = URI("https://api.indexnow.org/indexnow")
7
+ MAX_URLS_PER_REQUEST = 10_000
8
+ HEADERS = {"Content-Type" => "application/json; charset=utf-8"}.freeze
9
+
10
+ def initialize(key:, key_path: nil, transport: Net::HTTP.method(:post))
11
+ @key = key
12
+ @key_path = key_path
13
+ @transport = transport
14
+ end
15
+
16
+ # status: the HTTP status, or the class of what the transport raised.
17
+ Reply = Data.define(:urls, :status) do
18
+ def accepted?
19
+ status.is_a?(Integer) && status.between?(200, 299)
20
+ end
21
+ end
22
+
23
+ # One reply per request of up to 10,000 URLs, all of one host as IndexNow asks. Once the endpoint cannot be
24
+ # reached, the requests that follow, of this host or another, do not each wait for a timeout of their own.
25
+ def submit(host, urls)
26
+ urls.each_slice(MAX_URLS_PER_REQUEST).map do |batch|
27
+ Reply.new(urls: batch, status: @network_error || post(host, batch))
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ # Whatever the transport raises is a reply that did not come, from a refused connection to a body that is
34
+ # not HTTP: no list of classes covers them all, and the sitemaps are published, so none may fail the build.
35
+ def post(host, urls)
36
+ body = {host: URI.parse(host).host, key: @key, keyLocation: @key_path && "#{host}#{@key_path}", urlList: urls}.compact
37
+ body = JSON.generate(body)
38
+ begin
39
+ @transport.call(ENDPOINT, body, HEADERS).code.to_i
40
+ rescue => exception
41
+ @network_error = exception.class.name
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ # What the last build wrote: for each shard of each source, its fingerprint and its files. pending lists the
5
+ # files a build started to publish before it could record them, so that the next one can remove them;
6
+ # unsubmitted the URLs IndexNow did not take, for the next one to send again.
7
+ class Manifest
8
+ attr_reader :built_at, :rebuilt_at, :digest, :shards_by_source, :pending, :unsubmitted
9
+
10
+ def self.read(storage, name)
11
+ json = storage.read(name)
12
+ return new unless json
13
+
14
+ stored = JSON.parse(json)
15
+ new(digest: stored["digest"], built_at: time(stored["built_at"]), rebuilt_at: time(stored["rebuilt_at"]),
16
+ shards_by_source: stored["sources"] || {}, pending: stored["pending"] || [], unsubmitted: stored["unsubmitted"] || [])
17
+ rescue JSON::ParserError, ArgumentError, TypeError
18
+ # A manifest we cannot read is a manifest we do not have: the build starts over and replaces it.
19
+ new
20
+ end
21
+
22
+ def self.time(text)
23
+ Time.iso8601(text) if text
24
+ end
25
+
26
+ def initialize(digest: nil, built_at: nil, rebuilt_at: nil, shards_by_source: {}, pending: [], unsubmitted: [])
27
+ @digest = digest
28
+ @built_at = built_at
29
+ @rebuilt_at = rebuilt_at
30
+ @shards_by_source = shards_by_source
31
+ @pending = pending
32
+ @unsubmitted = unsubmitted
33
+ end
34
+
35
+ def shard(source, key)
36
+ shards_by_source.dig(source, key)
37
+ end
38
+
39
+ def record(source, key, fingerprint:, files:)
40
+ (shards_by_source[source] ||= {})[key] = {"fingerprint" => fingerprint, "files" => files}
41
+ end
42
+
43
+ def files
44
+ shards_by_source.each_value.flat_map { |shards| shards.each_value.flat_map { |shard| shard["files"] } }
45
+ end
46
+
47
+ def file_names
48
+ files.map { |file| file["name"] }
49
+ end
50
+
51
+ def to_json(*)
52
+ # Whole seconds, like the lastmods compared to built_at: a change in the second of a build is sent again
53
+ # rather than never.
54
+ JSON.pretty_generate("digest" => digest, "built_at" => built_at&.utc&.iso8601,
55
+ "rebuilt_at" => rebuilt_at&.utc&.iso8601, "pending" => pending, "unsubmitted" => unsubmitted, "sources" => shards_by_source)
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ # One page of the site: the same content at one URL per language, keyed by hreflang. lastmod is a W3C date;
5
+ # rejected_lastmod what a source gave that was not one.
6
+ Page = Data.define(:url_by_locale, :lastmod, :modified_at, :images, :rejected_lastmod) do
7
+ def urls
8
+ url_by_locale.values.uniq
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ class Railtie < Rails::Railtie
5
+ SOURCES_DIRECTORY = "app/sitemaps"
6
+
7
+ rake_tasks { load File.expand_path("tasks.rake", __dir__) }
8
+
9
+ config.after_initialize do |app|
10
+ Maptain::Source.include(app.routes.url_helpers)
11
+ Maptain.configuration.load_sources = lambda do
12
+ directory = app.root.join(SOURCES_DIRECTORY)
13
+ Rails.autoloaders.main.eager_load_dir(directory) if directory.directory?
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maptain
4
+ # Writes the pages of one shard as gzipped sitemap files, opening a second file when the first is full.
5
+ class ShardWriter
6
+ # https://www.sitemaps.org/protocol.html: 50,000 URLs and 50MB (52,428,800 bytes) a file, uncompressed.
7
+ MAX_URLS_PER_FILE = 50_000
8
+ MAX_BYTES_PER_FILE = 52_428_800
9
+
10
+ Written = Data.define(:name, :path, :url_count, :lastmod, :digest)
11
+
12
+ attr_reader :files
13
+
14
+ def initialize(directory:, basename:, max_urls: MAX_URLS_PER_FILE, max_bytes: MAX_BYTES_PER_FILE)
15
+ @directory = directory
16
+ @basename = basename
17
+ @max_urls = max_urls
18
+ @max_bytes = max_bytes
19
+ @files = []
20
+ end
21
+
22
+ # The entries of one page stay in the same file.
23
+ def add(entries, lastmod)
24
+ bytes = entries.sum(&:bytesize)
25
+ roll if @gzip && (@url_count + entries.size > @max_urls || @bytes + bytes > @max_bytes)
26
+ open_file unless @gzip
27
+ entries.each { |entry| write(entry) }
28
+ @url_count += entries.size
29
+ @lastmod = [@lastmod, lastmod].compact.max
30
+ end
31
+
32
+ def close
33
+ roll if @gzip
34
+ self
35
+ end
36
+
37
+ def content_digest
38
+ Digest::SHA256.hexdigest(@files.map(&:digest).join)
39
+ end
40
+
41
+ private
42
+
43
+ def open_file
44
+ suffix = @files.empty? ? "" : "-#{@files.size + 1}"
45
+ @name = "#{@basename}#{suffix}.xml.gz"
46
+ @path = File.join(@directory, @name)
47
+ @gzip = Zlib::GzipWriter.open(@path, Zlib::BEST_COMPRESSION)
48
+ @file_digest = Digest::SHA256.new
49
+ @url_count = 0
50
+ @bytes = Xml::URLSET_CLOSE.bytesize
51
+ @lastmod = nil
52
+ write(Xml::URLSET_OPEN)
53
+ end
54
+
55
+ def roll
56
+ write(Xml::URLSET_CLOSE)
57
+ @gzip.close
58
+ @gzip = nil
59
+ @files << Written.new(name: @name, path: @path, url_count: @url_count, lastmod: @lastmod, digest: @file_digest.hexdigest)
60
+ end
61
+
62
+ def write(text)
63
+ @gzip.write(text)
64
+ @file_digest << text
65
+ @bytes += text.bytesize
66
+ end
67
+ end
68
+ end