maptain 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +17 -0
- data/LICENSE +21 -0
- data/README.md +386 -0
- data/lib/maptain/builder.rb +357 -0
- data/lib/maptain/configuration.rb +100 -0
- data/lib/maptain/errors.rb +9 -0
- data/lib/maptain/index_now.rb +45 -0
- data/lib/maptain/manifest.rb +58 -0
- data/lib/maptain/page.rb +11 -0
- data/lib/maptain/railtie.rb +17 -0
- data/lib/maptain/shard_writer.rb +68 -0
- data/lib/maptain/source.rb +250 -0
- data/lib/maptain/storage/active_storage.rb +42 -0
- data/lib/maptain/storage/local.rb +33 -0
- data/lib/maptain/storage/null.rb +16 -0
- data/lib/maptain/tasks.rake +26 -0
- data/lib/maptain/version.rb +5 -0
- data/lib/maptain/xml.rb +102 -0
- data/lib/maptain.rb +56 -0
- metadata +80 -0
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
class Builder
|
|
5
|
+
# full: whether this build rewrote everything, asked to or because rebuild_every had passed.
|
|
6
|
+
Result = Data.define(:full, :written, :kept, :deleted, :url_count, :submitted, :warnings) do
|
|
7
|
+
def changed?
|
|
8
|
+
written.any? || deleted.any?
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# https://www.sitemaps.org/protocol.html
|
|
13
|
+
MAX_URL_CHARACTERS = 2_048
|
|
14
|
+
MAX_FILES_PER_INDEX = 50_000
|
|
15
|
+
LAST_TIME_ZONE_OFFSET_SECONDS = 12 * 60 * 60
|
|
16
|
+
DAY_LENGTH = "2026-01-01".length
|
|
17
|
+
# One request's worth: what a refusing endpoint leaves in the manifest stays small.
|
|
18
|
+
MAX_UNSUBMITTED_URLS = IndexNow::MAX_URLS_PER_REQUEST
|
|
19
|
+
# A language, then maybe a script and a region: en, pt-BR, zh-Hant-TW, es-419.
|
|
20
|
+
HREFLANG = /\A[a-z]{2,3}(-[a-z]{4})?(-([a-z]{2}|\d{3}))?\z/i
|
|
21
|
+
|
|
22
|
+
Rendered = Data.define(:source_slug, :shard, :fingerprint, :files, :changed_urls)
|
|
23
|
+
|
|
24
|
+
# others: how many more times its source made the same mistake.
|
|
25
|
+
Warning = Data.define(:source, :url, :message, :others) do
|
|
26
|
+
def to_s
|
|
27
|
+
line = "#{source}: #{url} #{message}"
|
|
28
|
+
others.zero? ? line : "#{line}, and #{others} more like it"
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def initialize(configuration, storage: configuration.storage, submit: true)
|
|
33
|
+
@configuration = configuration
|
|
34
|
+
@storage = storage
|
|
35
|
+
@submit = submit
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def build(full: false)
|
|
39
|
+
ActiveSupport::Notifications.instrument("build.maptain", full:) do |payload|
|
|
40
|
+
payload[:result] = Dir.mktmpdir("maptain") { |directory| build_in(directory, full:) }
|
|
41
|
+
payload[:full] = payload[:result].full
|
|
42
|
+
payload[:result]
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def build_in(directory, full:)
|
|
49
|
+
@configuration.validate!
|
|
50
|
+
@directory = directory
|
|
51
|
+
@previous = Manifest.read(@storage, manifest_name)
|
|
52
|
+
@previous_file_by_name = @previous.files.to_h { |file| [file["name"], file] }
|
|
53
|
+
built_at = Time.now
|
|
54
|
+
@full = full || rebuild_due?(built_at)
|
|
55
|
+
@reusable = !@full && @previous.digest == @configuration.digest
|
|
56
|
+
@manifest = Manifest.new(digest: @configuration.digest, built_at:, rebuilt_at: @full ? built_at : @previous.rebuilt_at)
|
|
57
|
+
@written = []
|
|
58
|
+
@kept = []
|
|
59
|
+
@warnings = []
|
|
60
|
+
@changed_urls = []
|
|
61
|
+
@checked_hreflangs = Set.new
|
|
62
|
+
@reported = Set.new
|
|
63
|
+
@warning_index_by_mistake = {}
|
|
64
|
+
@x_default = Xml.hreflang(@configuration.x_default)
|
|
65
|
+
@hosts = @configuration.hosts
|
|
66
|
+
@host_prefixes = @hosts.map { |host| "#{host}/" }
|
|
67
|
+
@outside_message = "is outside #{@hosts.first}#{" and the hosts of its other languages" if @hosts.size > 1}"
|
|
68
|
+
@index_now = IndexNow.new(key: @configuration.index_now_key, key_path: @configuration.index_now_key_location)
|
|
69
|
+
|
|
70
|
+
@rendered = []
|
|
71
|
+
# The first build has no "before": submitting a whole site at once is not what IndexNow is for. Nor is a
|
|
72
|
+
# new configuration, which rewrites every file: then only the lastmods tell what changed.
|
|
73
|
+
@tracking = @submit && @configuration.index_now_key && @previous.built_at.present?
|
|
74
|
+
@diffing = @tracking && @previous.digest == @configuration.digest
|
|
75
|
+
@moved_sources = Set.new
|
|
76
|
+
|
|
77
|
+
@configuration.sources.each { |source_class| build_source(source_class.new(@configuration)) }
|
|
78
|
+
publish_rendered
|
|
79
|
+
publish_index
|
|
80
|
+
deleted = (@previous.file_names | @previous.pending) - @manifest.file_names
|
|
81
|
+
@changed_urls.concat(urls_in(deleted.reject { |name| @moved_sources.include?(previous_source_of(name)) })) if @diffing
|
|
82
|
+
deleted.each { |name| @storage.delete(name) }
|
|
83
|
+
publish_index_now_key
|
|
84
|
+
urls = urls_to_submit
|
|
85
|
+
# Written before submitting, with the URLs still to send: a build that dies while submitting loses none.
|
|
86
|
+
@manifest.unsubmitted.replace(urls.last(MAX_UNSUBMITTED_URLS))
|
|
87
|
+
upload(manifest_name, @manifest.to_json)
|
|
88
|
+
submitted = submit(urls)
|
|
89
|
+
|
|
90
|
+
Result.new(full: @full, written: @written, kept: @kept, deleted:, url_count: @manifest.files.sum { |file| file["urls"] },
|
|
91
|
+
submitted:, warnings: @warnings)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def build_source(source)
|
|
95
|
+
seen_urls = Set.new
|
|
96
|
+
source.fingerprint_by_shard.sort_by { |shard, _fingerprint| shard.to_i }.each do |shard, fingerprint|
|
|
97
|
+
stored = @previous.shard(source.slug, shard) if @reusable
|
|
98
|
+
if fingerprint && stored && stored["fingerprint"] == fingerprint
|
|
99
|
+
keep(source, shard, stored)
|
|
100
|
+
else
|
|
101
|
+
render(source, shard, fingerprint, seen_urls)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def keep(source, shard, stored)
|
|
107
|
+
@manifest.record(source.slug, shard, fingerprint: stored["fingerprint"], files: stored["files"])
|
|
108
|
+
@kept.concat(stored["files"].map { |file| file["name"] })
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def render(source, shard, fingerprint, seen_urls)
|
|
112
|
+
writer = ShardWriter.new(directory: @directory, basename: source.basename(shard))
|
|
113
|
+
changed_urls = []
|
|
114
|
+
urls = [] if @diffing
|
|
115
|
+
ActiveSupport::Notifications.instrument("shard.maptain", source: source.slug, shard:) do
|
|
116
|
+
source.each_page(shard) do |page|
|
|
117
|
+
inspect_page(source, page, seen_urls)
|
|
118
|
+
writer.add(Xml.url_entries(page, x_default: @x_default), page.lastmod)
|
|
119
|
+
changed_urls.concat(page.urls) if @tracking && changed_since_last_build?(page)
|
|
120
|
+
urls&.concat(page.urls)
|
|
121
|
+
end
|
|
122
|
+
writer.close
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
if writer.files.all? { |file| same_as_before?(file) }
|
|
126
|
+
changed_urls.clear
|
|
127
|
+
elsif urls
|
|
128
|
+
changed_urls.concat(appeared_or_gone(source.slug, shard, fingerprint, urls))
|
|
129
|
+
end
|
|
130
|
+
fingerprint ||= writer.content_digest
|
|
131
|
+
# Recorded now, published once every shard is rendered: the index keeps the order of the sources.
|
|
132
|
+
@manifest.record(source.slug, shard, fingerprint:, files: [])
|
|
133
|
+
@rendered << Rendered.new(source_slug: source.slug, shard:, fingerprint:, files: writer.files, changed_urls:)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Read while the previous files are still in the storage: a page that appeared or went away is a change too,
|
|
137
|
+
# whatever its lastmod says. Not when the ranges moved, which only carries pages from one file to another.
|
|
138
|
+
def appeared_or_gone(source_slug, shard, fingerprint, urls)
|
|
139
|
+
stored = @previous.shard(source_slug, shard)
|
|
140
|
+
if stored && fingerprint && stored["fingerprint"].to_s.split("/").first != fingerprint.split("/").first
|
|
141
|
+
@moved_sources << source_slug
|
|
142
|
+
return []
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
previous_urls = urls_in((stored&.fetch("files") || []).map { |file| file["name"] }).to_set
|
|
146
|
+
urls = urls.to_set
|
|
147
|
+
(urls - previous_urls).to_a + (previous_urls - urls).to_a
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def publish_rendered
|
|
151
|
+
changed_names = @rendered.flat_map(&:files).reject { |file| same_as_before?(file) }.to_set(&:name)
|
|
152
|
+
announce_new_files(changed_names)
|
|
153
|
+
@rendered.each do |rendered|
|
|
154
|
+
@changed_urls.concat(rendered.changed_urls)
|
|
155
|
+
entries = rendered.files.map do |file|
|
|
156
|
+
changed = changed_names.include?(file.name)
|
|
157
|
+
publish(file, changed)
|
|
158
|
+
manifest_entry(file, changed)
|
|
159
|
+
end
|
|
160
|
+
@manifest.record(rendered.source_slug, rendered.shard, fingerprint: rendered.fingerprint, files: entries)
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def previous_source_of(file_name)
|
|
165
|
+
@previous_source_by_file_name ||= @previous.shards_by_source.each_with_object({}) do |(source_slug, shards), source_by_name|
|
|
166
|
+
shards.each_value { |stored| stored["files"].each { |file| source_by_name[file["name"]] = source_slug } }
|
|
167
|
+
end
|
|
168
|
+
@previous_source_by_file_name[file_name]
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def urls_in(names)
|
|
172
|
+
names.flat_map do |name|
|
|
173
|
+
content = @storage.read(name)
|
|
174
|
+
content ? Xml.locations(Zlib.gunzip(content.b)) : []
|
|
175
|
+
rescue Zlib::Error
|
|
176
|
+
[]
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def same_as_before?(file)
|
|
181
|
+
@previous_file_by_name.dig(file.name, "digest") == file.digest
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Recorded before the first upload, so that a build dying midway leaves no file the next one cannot remove.
|
|
185
|
+
def announce_new_files(names)
|
|
186
|
+
new_names = names.to_a - @previous_file_by_name.keys - @previous.pending
|
|
187
|
+
return if new_names.empty?
|
|
188
|
+
|
|
189
|
+
@previous.pending.concat(new_names)
|
|
190
|
+
upload(manifest_name, @previous.to_json)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# A full build uploads every file, so that it also repairs a storage that lost one.
|
|
194
|
+
def publish(file, changed)
|
|
195
|
+
if changed || @full
|
|
196
|
+
@storage.write(file.name, file.path)
|
|
197
|
+
@written << file.name
|
|
198
|
+
else
|
|
199
|
+
@kept << file.name
|
|
200
|
+
end
|
|
201
|
+
File.delete(file.path)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# The index tells crawlers which files to read again: a file that changed gets a later date even when the
|
|
205
|
+
# change is a removal, and one that did not keeps its date.
|
|
206
|
+
def manifest_entry(file, changed)
|
|
207
|
+
previous_lastmod = @previous_file_by_name.dig(file.name, "lastmod")
|
|
208
|
+
lastmod = if !changed
|
|
209
|
+
previous_lastmod
|
|
210
|
+
elsif @previous_file_by_name.key?(file.name) && !later?(file.lastmod, previous_lastmod)
|
|
211
|
+
@manifest.built_at.getutc.iso8601
|
|
212
|
+
else
|
|
213
|
+
file.lastmod
|
|
214
|
+
end
|
|
215
|
+
{"name" => file.name, "urls" => file.url_count, "lastmod" => lastmod, "digest" => file.digest}
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# A manifest written by an older version may hold a lastmod in another format: compare times, not text.
|
|
219
|
+
def later?(stamp, previous_stamp)
|
|
220
|
+
return false if stamp.nil?
|
|
221
|
+
|
|
222
|
+
previous_stamp = Xml.timestamp(previous_stamp)
|
|
223
|
+
previous_stamp.nil? || Xml.time(stamp) > Xml.time(previous_stamp)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def rebuild_due?(now)
|
|
227
|
+
every = @configuration.rebuild_every
|
|
228
|
+
every.present? && (@previous.rebuilt_at.nil? || @previous.rebuilt_at + every <= now)
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def publish_index
|
|
232
|
+
name = "#{@configuration.filename}.xml"
|
|
233
|
+
entries = @manifest.files.map do |file|
|
|
234
|
+
Xml.sitemap_entry("#{@configuration.sitemaps_url}/#{file["name"]}", file["lastmod"])
|
|
235
|
+
end
|
|
236
|
+
inspect_index("#{@configuration.sitemaps_url}/#{name}", entries.size)
|
|
237
|
+
upload(name, "#{Xml::INDEX_OPEN}#{entries.join}#{Xml::INDEX_CLOSE}")
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def upload(name, content)
|
|
241
|
+
path = File.join(@directory, name)
|
|
242
|
+
File.write(path, content)
|
|
243
|
+
@storage.write(name, path)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def manifest_name
|
|
247
|
+
"#{@configuration.filename}-manifest.json"
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def inspect_page(source, page, seen_urls)
|
|
251
|
+
page.urls.each do |url|
|
|
252
|
+
warn_counted(source, :outside, url, @outside_message) unless on_host?(url)
|
|
253
|
+
warn_counted(source, :listed_twice, url, "is listed twice") unless seen_urls.add?(url)
|
|
254
|
+
warn_counted(source, :long, url, "is longer than #{MAX_URL_CHARACTERS} characters") if url.length > MAX_URL_CHARACTERS
|
|
255
|
+
end
|
|
256
|
+
page.url_by_locale.each do |hreflang, url|
|
|
257
|
+
warn(source, url, "has hreflang #{hreflang.inspect}, not a language code") if @checked_hreflangs.add?(hreflang) && !hreflang.match?(HREFLANG)
|
|
258
|
+
end
|
|
259
|
+
if page.url_by_locale.size > 1 && page.urls.one?
|
|
260
|
+
warn_once(source, :one_url, page.urls.first, "is the URL of every language of its page: does the url block use the locale?")
|
|
261
|
+
end
|
|
262
|
+
inspect_lastmod(source, page)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def inspect_lastmod(source, page)
|
|
266
|
+
if page.rejected_lastmod
|
|
267
|
+
warn_once(source, :rejected_lastmod, page.urls.first, "has a lastmod that is not a W3C date (#{page.rejected_lastmod.inspect}), left out")
|
|
268
|
+
elsif page.modified_at && page.modified_at > @manifest.built_at
|
|
269
|
+
warn_counted(source, :future_lastmod, page.urls.first, "has a lastmod in the future (#{page.lastmod})")
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def inspect_index(url, file_count)
|
|
274
|
+
return if file_count <= MAX_FILES_PER_INDEX
|
|
275
|
+
|
|
276
|
+
warn("index", url, "lists #{file_count} files, more than #{MAX_FILES_PER_INDEX}: raise shard_size")
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# For a mistake every page of a source would repeat.
|
|
280
|
+
def warn_once(source, kind, url, message)
|
|
281
|
+
warn(source, url, message) if @reported.add?([source.slug, kind])
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# For a mistake any number of URLs of a source may share: the first is reported, the others are counted.
|
|
285
|
+
def warn_counted(source, kind, url, message)
|
|
286
|
+
mistake = [source.slug, kind]
|
|
287
|
+
index = @warning_index_by_mistake[mistake]
|
|
288
|
+
if index
|
|
289
|
+
@warnings[index] = @warnings[index].with(others: @warnings[index].others + 1)
|
|
290
|
+
else
|
|
291
|
+
@warning_index_by_mistake[mistake] = @warnings.size
|
|
292
|
+
warn(source, url, message)
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def warn(source, url, message)
|
|
297
|
+
@warnings << Warning.new(source: source.is_a?(Source) ? source.slug : source, url:, message:, others: 0)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def on_host?(url)
|
|
301
|
+
url.start_with?(*@host_prefixes) || @hosts.include?(url)
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def host_of(url)
|
|
305
|
+
@hosts.find { |host| url == host || url.start_with?("#{host}/") }
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# A day may still be the day of the last build somewhere: it changed if it is not older than the date at
|
|
309
|
+
# UTC-12, the last time zone to leave a day. A month or a year alone says too little to tell.
|
|
310
|
+
def changed_since_last_build?(page)
|
|
311
|
+
return false unless page.lastmod
|
|
312
|
+
return page.modified_at >= @previous.built_at if page.lastmod.include?("T")
|
|
313
|
+
|
|
314
|
+
page.lastmod.length == DAY_LENGTH && page.lastmod >= (@previous.built_at - LAST_TIME_ZONE_OFFSET_SECONDS).utc.to_date.iso8601
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
def urls_to_submit
|
|
318
|
+
return [] unless @submit && @configuration.index_now_key
|
|
319
|
+
|
|
320
|
+
(@previous.unsubmitted + @changed_urls).uniq.select { |url| on_host?(url) }
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# Returns how many URLs the endpoint took. The others, refused or unreachable, stay in the manifest for the
|
|
324
|
+
# next build: the sitemaps are published, so a refusal never fails the build.
|
|
325
|
+
def submit(urls)
|
|
326
|
+
return 0 if urls.empty?
|
|
327
|
+
|
|
328
|
+
replies = urls.group_by { |url| host_of(url) }.flat_map { |host, urls_of_host| submit_to(host, urls_of_host) }
|
|
329
|
+
@manifest.unsubmitted.replace(replies.reject(&:accepted?).flat_map(&:urls).last(MAX_UNSUBMITTED_URLS))
|
|
330
|
+
upload(manifest_name, @manifest.to_json)
|
|
331
|
+
replies.select(&:accepted?).sum { |reply| reply.urls.size }
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
def submit_to(host, urls)
|
|
335
|
+
replies = @index_now.submit(host, urls)
|
|
336
|
+
replies.reject(&:accepted?).group_by(&:status).each do |status, refused|
|
|
337
|
+
count = refused.sum { |reply| reply.urls.size }
|
|
338
|
+
answer = status.is_a?(Integer) ? "answered #{status}" : "could not be reached (#{status})"
|
|
339
|
+
warn("indexnow", IndexNow::ENDPOINT, "#{answer} for #{count} URLs of #{host}, sent again with the next build")
|
|
340
|
+
end
|
|
341
|
+
replies
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# IndexNow reads the key at the root of a host unless told where it is, and only on that host.
|
|
345
|
+
def publish_index_now_key
|
|
346
|
+
key = @configuration.index_now_key
|
|
347
|
+
return if key.nil? || @configuration.index_now_key_location
|
|
348
|
+
|
|
349
|
+
if @configuration.sitemaps_url == @configuration.host!
|
|
350
|
+
upload("#{key}.txt", key)
|
|
351
|
+
else
|
|
352
|
+
warn("indexnow", "#{@configuration.host!}/#{key}.txt", "must hold the key: sitemaps go to #{@configuration.sitemaps_url}, " \
|
|
353
|
+
"so put #{key}.txt in public/ and set index_now_key_location to \"/#{key}.txt\"")
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
end
|
|
357
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
class Configuration
|
|
5
|
+
# A full build now and then catches what no fingerprint sees: update_columns, a change in another table,
|
|
6
|
+
# a new url block, a file removed from the storage.
|
|
7
|
+
DEFAULT_REBUILD_EVERY_SECONDS = 7 * 24 * 60 * 60
|
|
8
|
+
|
|
9
|
+
attr_accessor :filename, :load_sources, :rebuild_every, :storage, :x_default
|
|
10
|
+
attr_reader :host, :host_by_locale, :index_now_key, :index_now_key_location
|
|
11
|
+
attr_writer :locales, :sitemaps_url, :sources
|
|
12
|
+
|
|
13
|
+
def initialize
|
|
14
|
+
@filename = "sitemap"
|
|
15
|
+
@host_by_locale = {}.freeze
|
|
16
|
+
@load_sources = -> {}
|
|
17
|
+
@rebuild_every = DEFAULT_REBUILD_EVERY_SECONDS
|
|
18
|
+
@storage = Storage::Local.new("public")
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def host=(url)
|
|
22
|
+
@host = absolute_url(url, "host")
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def host!
|
|
26
|
+
host || raise(ConfigurationError, "Maptain.configuration.host is not set")
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Frozen: a host added in place would be neither checked nor found.
|
|
30
|
+
def host_by_locale=(host_by_locale)
|
|
31
|
+
@host_by_locale = host_by_locale.to_h do |locale, url|
|
|
32
|
+
[Xml.hreflang(locale), absolute_url(url, "host_by_locale[#{locale.inspect}]")]
|
|
33
|
+
end.freeze
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def host_for(locale)
|
|
37
|
+
host_by_locale.fetch(Xml.hreflang(locale)) { host! }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def hosts
|
|
41
|
+
[host!, *host_by_locale.values].uniq
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Checked when a build starts, not when host_by_locale is set: locales may be set after it.
|
|
45
|
+
def validate!
|
|
46
|
+
hreflangs = locales.map { |locale| Xml.hreflang(locale) }
|
|
47
|
+
unknown = host_by_locale.keys - hreflangs
|
|
48
|
+
return if unknown.empty?
|
|
49
|
+
|
|
50
|
+
raise ConfigurationError, "host_by_locale[#{unknown.first.inspect}] is not one of the locales: #{hreflangs.join(", ")}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# A blank key, as an environment variable left empty gives, is no key.
|
|
54
|
+
def index_now_key=(key)
|
|
55
|
+
@index_now_key = key.presence
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# A path, not a URL: IndexNow only reads a key on the host of the URLs it is given, so each host serves it.
|
|
59
|
+
def index_now_key_location=(path)
|
|
60
|
+
path = path.to_s.presence
|
|
61
|
+
unless path.nil? || path.match?(%r{\A/[^/\s]\S*\z})
|
|
62
|
+
raise ConfigurationError, "index_now_key_location must be a path such as \"/key.txt\", got #{path.inspect}"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
@index_now_key_location = path
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def locales
|
|
69
|
+
Array(@locales || I18n.available_locales)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Where the sitemap files are served from, when that is not the site itself (a bucket, a CDN).
|
|
73
|
+
def sitemaps_url
|
|
74
|
+
@sitemaps_url ? absolute_url(@sitemaps_url, "sitemaps_url") : host!
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def sources
|
|
78
|
+
return @sources if @sources
|
|
79
|
+
|
|
80
|
+
load_sources.call
|
|
81
|
+
Source.descendants.select(&:scope?).sort_by(&:slug)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Whatever changes every file: a build under another digest starts over.
|
|
85
|
+
def digest
|
|
86
|
+
Digest::SHA256.hexdigest(JSON.generate([VERSION, host!, host_by_locale, sitemaps_url, locales, x_default, filename]))
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private
|
|
90
|
+
|
|
91
|
+
def absolute_url(url, setting)
|
|
92
|
+
uri = URI.parse(url.to_s)
|
|
93
|
+
raise ConfigurationError, "#{setting} must be an http(s) URL, got #{url.inspect}" unless uri.is_a?(URI::HTTP) && uri.host.present?
|
|
94
|
+
|
|
95
|
+
url.to_s.delete_suffix("/")
|
|
96
|
+
rescue URI::InvalidURIError
|
|
97
|
+
raise ConfigurationError, "#{setting} must be an http(s) URL, got #{url.inspect}"
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
# Tells Bing, Yandex, Seznam and Naver which URLs changed: https://www.indexnow.org/documentation
|
|
5
|
+
class IndexNow
|
|
6
|
+
ENDPOINT = URI("https://api.indexnow.org/indexnow")
|
|
7
|
+
MAX_URLS_PER_REQUEST = 10_000
|
|
8
|
+
HEADERS = {"Content-Type" => "application/json; charset=utf-8"}.freeze
|
|
9
|
+
|
|
10
|
+
def initialize(key:, key_path: nil, transport: Net::HTTP.method(:post))
|
|
11
|
+
@key = key
|
|
12
|
+
@key_path = key_path
|
|
13
|
+
@transport = transport
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# status: the HTTP status, or the class of what the transport raised.
|
|
17
|
+
Reply = Data.define(:urls, :status) do
|
|
18
|
+
def accepted?
|
|
19
|
+
status.is_a?(Integer) && status.between?(200, 299)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# One reply per request of up to 10,000 URLs, all of one host as IndexNow asks. Once the endpoint cannot be
|
|
24
|
+
# reached, the requests that follow, of this host or another, do not each wait for a timeout of their own.
|
|
25
|
+
def submit(host, urls)
|
|
26
|
+
urls.each_slice(MAX_URLS_PER_REQUEST).map do |batch|
|
|
27
|
+
Reply.new(urls: batch, status: @network_error || post(host, batch))
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# Whatever the transport raises is a reply that did not come, from a refused connection to a body that is
|
|
34
|
+
# not HTTP: no list of classes covers them all, and the sitemaps are published, so none may fail the build.
|
|
35
|
+
def post(host, urls)
|
|
36
|
+
body = {host: URI.parse(host).host, key: @key, keyLocation: @key_path && "#{host}#{@key_path}", urlList: urls}.compact
|
|
37
|
+
body = JSON.generate(body)
|
|
38
|
+
begin
|
|
39
|
+
@transport.call(ENDPOINT, body, HEADERS).code.to_i
|
|
40
|
+
rescue => exception
|
|
41
|
+
@network_error = exception.class.name
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
# What the last build wrote: for each shard of each source, its fingerprint and its files. pending lists the
|
|
5
|
+
# files a build started to publish before it could record them, so that the next one can remove them;
|
|
6
|
+
# unsubmitted the URLs IndexNow did not take, for the next one to send again.
|
|
7
|
+
class Manifest
|
|
8
|
+
attr_reader :built_at, :rebuilt_at, :digest, :shards_by_source, :pending, :unsubmitted
|
|
9
|
+
|
|
10
|
+
def self.read(storage, name)
|
|
11
|
+
json = storage.read(name)
|
|
12
|
+
return new unless json
|
|
13
|
+
|
|
14
|
+
stored = JSON.parse(json)
|
|
15
|
+
new(digest: stored["digest"], built_at: time(stored["built_at"]), rebuilt_at: time(stored["rebuilt_at"]),
|
|
16
|
+
shards_by_source: stored["sources"] || {}, pending: stored["pending"] || [], unsubmitted: stored["unsubmitted"] || [])
|
|
17
|
+
rescue JSON::ParserError, ArgumentError, TypeError
|
|
18
|
+
# A manifest we cannot read is a manifest we do not have: the build starts over and replaces it.
|
|
19
|
+
new
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.time(text)
|
|
23
|
+
Time.iso8601(text) if text
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(digest: nil, built_at: nil, rebuilt_at: nil, shards_by_source: {}, pending: [], unsubmitted: [])
|
|
27
|
+
@digest = digest
|
|
28
|
+
@built_at = built_at
|
|
29
|
+
@rebuilt_at = rebuilt_at
|
|
30
|
+
@shards_by_source = shards_by_source
|
|
31
|
+
@pending = pending
|
|
32
|
+
@unsubmitted = unsubmitted
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def shard(source, key)
|
|
36
|
+
shards_by_source.dig(source, key)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def record(source, key, fingerprint:, files:)
|
|
40
|
+
(shards_by_source[source] ||= {})[key] = {"fingerprint" => fingerprint, "files" => files}
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def files
|
|
44
|
+
shards_by_source.each_value.flat_map { |shards| shards.each_value.flat_map { |shard| shard["files"] } }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def file_names
|
|
48
|
+
files.map { |file| file["name"] }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def to_json(*)
|
|
52
|
+
# Whole seconds, like the lastmods compared to built_at: a change in the second of a build is sent again
|
|
53
|
+
# rather than never.
|
|
54
|
+
JSON.pretty_generate("digest" => digest, "built_at" => built_at&.utc&.iso8601,
|
|
55
|
+
"rebuilt_at" => rebuilt_at&.utc&.iso8601, "pending" => pending, "unsubmitted" => unsubmitted, "sources" => shards_by_source)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
data/lib/maptain/page.rb
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
# One page of the site: the same content at one URL per language, keyed by hreflang. lastmod is a W3C date;
|
|
5
|
+
# rejected_lastmod what a source gave that was not one.
|
|
6
|
+
Page = Data.define(:url_by_locale, :lastmod, :modified_at, :images, :rejected_lastmod) do
|
|
7
|
+
def urls
|
|
8
|
+
url_by_locale.values.uniq
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
class Railtie < Rails::Railtie
|
|
5
|
+
SOURCES_DIRECTORY = "app/sitemaps"
|
|
6
|
+
|
|
7
|
+
rake_tasks { load File.expand_path("tasks.rake", __dir__) }
|
|
8
|
+
|
|
9
|
+
config.after_initialize do |app|
|
|
10
|
+
Maptain::Source.include(app.routes.url_helpers)
|
|
11
|
+
Maptain.configuration.load_sources = lambda do
|
|
12
|
+
directory = app.root.join(SOURCES_DIRECTORY)
|
|
13
|
+
Rails.autoloaders.main.eager_load_dir(directory) if directory.directory?
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
# Writes the pages of one shard as gzipped sitemap files, opening a second file when the first is full.
|
|
5
|
+
class ShardWriter
|
|
6
|
+
# https://www.sitemaps.org/protocol.html: 50,000 URLs and 50MB (52,428,800 bytes) a file, uncompressed.
|
|
7
|
+
MAX_URLS_PER_FILE = 50_000
|
|
8
|
+
MAX_BYTES_PER_FILE = 52_428_800
|
|
9
|
+
|
|
10
|
+
Written = Data.define(:name, :path, :url_count, :lastmod, :digest)
|
|
11
|
+
|
|
12
|
+
attr_reader :files
|
|
13
|
+
|
|
14
|
+
def initialize(directory:, basename:, max_urls: MAX_URLS_PER_FILE, max_bytes: MAX_BYTES_PER_FILE)
|
|
15
|
+
@directory = directory
|
|
16
|
+
@basename = basename
|
|
17
|
+
@max_urls = max_urls
|
|
18
|
+
@max_bytes = max_bytes
|
|
19
|
+
@files = []
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# The entries of one page stay in the same file.
|
|
23
|
+
def add(entries, lastmod)
|
|
24
|
+
bytes = entries.sum(&:bytesize)
|
|
25
|
+
roll if @gzip && (@url_count + entries.size > @max_urls || @bytes + bytes > @max_bytes)
|
|
26
|
+
open_file unless @gzip
|
|
27
|
+
entries.each { |entry| write(entry) }
|
|
28
|
+
@url_count += entries.size
|
|
29
|
+
@lastmod = [@lastmod, lastmod].compact.max
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def close
|
|
33
|
+
roll if @gzip
|
|
34
|
+
self
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def content_digest
|
|
38
|
+
Digest::SHA256.hexdigest(@files.map(&:digest).join)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def open_file
|
|
44
|
+
suffix = @files.empty? ? "" : "-#{@files.size + 1}"
|
|
45
|
+
@name = "#{@basename}#{suffix}.xml.gz"
|
|
46
|
+
@path = File.join(@directory, @name)
|
|
47
|
+
@gzip = Zlib::GzipWriter.open(@path, Zlib::BEST_COMPRESSION)
|
|
48
|
+
@file_digest = Digest::SHA256.new
|
|
49
|
+
@url_count = 0
|
|
50
|
+
@bytes = Xml::URLSET_CLOSE.bytesize
|
|
51
|
+
@lastmod = nil
|
|
52
|
+
write(Xml::URLSET_OPEN)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def roll
|
|
56
|
+
write(Xml::URLSET_CLOSE)
|
|
57
|
+
@gzip.close
|
|
58
|
+
@gzip = nil
|
|
59
|
+
@files << Written.new(name: @name, path: @path, url_count: @url_count, lastmod: @lastmod, digest: @file_digest.hexdigest)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def write(text)
|
|
63
|
+
@gzip.write(text)
|
|
64
|
+
@file_digest << text
|
|
65
|
+
@bytes += text.bytesize
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|