maptain 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +17 -0
- data/LICENSE +21 -0
- data/README.md +386 -0
- data/lib/maptain/builder.rb +357 -0
- data/lib/maptain/configuration.rb +100 -0
- data/lib/maptain/errors.rb +9 -0
- data/lib/maptain/index_now.rb +45 -0
- data/lib/maptain/manifest.rb +58 -0
- data/lib/maptain/page.rb +11 -0
- data/lib/maptain/railtie.rb +17 -0
- data/lib/maptain/shard_writer.rb +68 -0
- data/lib/maptain/source.rb +250 -0
- data/lib/maptain/storage/active_storage.rb +42 -0
- data/lib/maptain/storage/local.rb +33 -0
- data/lib/maptain/storage/null.rb +16 -0
- data/lib/maptain/tasks.rake +26 -0
- data/lib/maptain/version.rb +5 -0
- data/lib/maptain/xml.rb +102 -0
- data/lib/maptain.rb +56 -0
- metadata +80 -0
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
# One kind of page. A subclass names its records, their URLs and their languages:
|
|
5
|
+
#
|
|
6
|
+
# class ProductSitemap < Maptain::Source
|
|
7
|
+
# scope { Product.published }
|
|
8
|
+
# url { |product, locale| product_url(product, locale:) }
|
|
9
|
+
# end
|
|
10
|
+
class Source
|
|
11
|
+
# The protocol allows 50 MiB a file, uncompressed. Shards aim lower: the sizes below are estimates.
|
|
12
|
+
TARGET_BYTES_PER_FILE = 40 * 1024 * 1024
|
|
13
|
+
BYTES_PER_URL_ESTIMATE = 150
|
|
14
|
+
BYTES_PER_ALTERNATE_ESTIMATE = 120
|
|
15
|
+
MYSQL_ADAPTERS = %w[mysql2 trilogy].freeze
|
|
16
|
+
SLUG_FORMAT = /\A[a-z0-9][a-z0-9_-]*\z/
|
|
17
|
+
# A source that is not an Active Record relation is one shard, rendered at each build.
|
|
18
|
+
WHOLE = "all"
|
|
19
|
+
# What RFC 3986 lets a URL hold as is. Anything else is percent-encoded, byte by byte.
|
|
20
|
+
URL_CHARACTERS = "A-Za-z0-9\\-._~:/?#\\[\\]@!$&'()*+,;=%"
|
|
21
|
+
UNSAFE = "[^#{URL_CHARACTERS}]|%(?![0-9A-Fa-f]{2})"
|
|
22
|
+
UNSAFE_CHARACTER = Regexp.new(UNSAFE)
|
|
23
|
+
UNSAFE_BYTE = Regexp.new(UNSAFE, Regexp::NOENCODING)
|
|
24
|
+
|
|
25
|
+
class << self
|
|
26
|
+
def inherited(subclass)
|
|
27
|
+
super
|
|
28
|
+
subclass.instance_variable_set(:@definitions, definitions.dup)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def definitions
|
|
32
|
+
@definitions ||= {lastmod: :updated_at}
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# The records, as an Active Record relation or any enumerable.
|
|
36
|
+
def scope(&block)
|
|
37
|
+
definitions[:scope] = block
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def scope?
|
|
41
|
+
definitions.key?(:scope)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The URL of a record in a language, or nil when it has none. A path is completed with the host.
|
|
45
|
+
def url(&block)
|
|
46
|
+
definitions[:url] = block
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# The languages a record exists in. Without it, every configured language.
|
|
50
|
+
def locales(&block)
|
|
51
|
+
definitions[:locales] = block
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# A column, or a block for anything else. With a block on a relation, name what changes with it in
|
|
55
|
+
# `fingerprint`: the column alone no longer tells a build what moved.
|
|
56
|
+
def lastmod(column = nil, &block)
|
|
57
|
+
definitions[:lastmod] = block || column
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def images(&block)
|
|
61
|
+
definitions[:images] = block
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# The SQL expression whose greatest value changes whenever a record's sitemap entry does.
|
|
65
|
+
def fingerprint(sql)
|
|
66
|
+
definitions[:fingerprint] = sql
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# How many consecutive ids share a file.
|
|
70
|
+
def shard_size(count)
|
|
71
|
+
definitions[:shard_size] = Integer(count)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def slug(name = nil)
|
|
75
|
+
definitions[:slug] = name.to_s if name
|
|
76
|
+
return definitions[:slug] if definitions[:slug]
|
|
77
|
+
|
|
78
|
+
self.name ? self.name.demodulize.underscore.delete_suffix("_sitemap") : superclass.slug
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
attr_reader :configuration, :slug
|
|
83
|
+
|
|
84
|
+
def initialize(configuration)
|
|
85
|
+
@configuration = configuration
|
|
86
|
+
@slug = self.class.slug
|
|
87
|
+
return if slug.match?(SLUG_FORMAT)
|
|
88
|
+
|
|
89
|
+
raise SourceError, "#{self.class.name}'s slug #{slug.inspect} must be lowercase letters, digits, - and _"
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# What `url` blocks read when they call a route helper. Frozen: one hash serves every URL of a host.
|
|
93
|
+
def default_url_options
|
|
94
|
+
@url_options_by_host ||= {}
|
|
95
|
+
@url_options_by_host[current_host] ||= begin
|
|
96
|
+
uri = URI.parse(current_host)
|
|
97
|
+
{host: uri.host, protocol: uri.scheme, port: (uri.port unless uri.port == uri.default_port)}.compact.freeze
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# A nil fingerprint means the shard has to be rendered to know whether it changed.
|
|
102
|
+
def fingerprint_by_shard
|
|
103
|
+
return {WHOLE => nil} unless relation?
|
|
104
|
+
|
|
105
|
+
# The sum of the ids moves when a record leaves a range and another enters it; the shard size, when the
|
|
106
|
+
# ranges themselves move. Summed as offsets within the range: SQLite's SUM overflows on large ids.
|
|
107
|
+
offset_sum = Arel.sql("SUM(#{model.quoted_table_name}.#{model.quoted_primary_key} % #{shard_size})")
|
|
108
|
+
relation.reorder(nil).group(shard_sql).pluck(shard_sql, Arel.sql("COUNT(*)"), offset_sum, fingerprint_sql)
|
|
109
|
+
.to_h { |shard, count, sum, latest| [shard.to_i.to_s, "#{shard_size}/#{count}/#{sum.to_i}/#{precise(latest)}"] }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def each_page(shard)
|
|
113
|
+
return to_enum(:each_page, shard) unless block_given?
|
|
114
|
+
|
|
115
|
+
records_of(shard) do |record|
|
|
116
|
+
page = page_for(record)
|
|
117
|
+
yield page unless page.url_by_locale.empty?
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def basename(shard)
|
|
122
|
+
prefix = "#{configuration.filename}-#{slug}"
|
|
123
|
+
(shard == WHOLE) ? prefix : "#{prefix}-#{shard.rjust(5, "0")}"
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
private
|
|
127
|
+
|
|
128
|
+
def definitions
|
|
129
|
+
self.class.definitions
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def relation
|
|
133
|
+
@relation ||= instance_exec(&definitions.fetch(:scope))
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def relation?
|
|
137
|
+
relation.respond_to?(:find_each) && relation.respond_to?(:group)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def model
|
|
141
|
+
relation.klass
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def records_of(shard, &)
|
|
145
|
+
return relation.each(&) unless relation?
|
|
146
|
+
|
|
147
|
+
first_id = shard.to_i * shard_size
|
|
148
|
+
relation.where(model.primary_key => first_id...(first_id + shard_size)).find_each(&)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def page_for(record)
|
|
152
|
+
locales = definitions[:locales] ? Array(instance_exec(record, &definitions[:locales])) : configuration.locales
|
|
153
|
+
given_lastmod = lastmod_of(record)
|
|
154
|
+
lastmod = Xml.timestamp(given_lastmod)
|
|
155
|
+
Page.new(url_by_locale: url_by_locale_of(record, locales), lastmod:, modified_at: lastmod && Xml.time(lastmod),
|
|
156
|
+
images: images_of(record), rejected_lastmod: (given_lastmod if lastmod.nil? && given_lastmod.present?))
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# The url block runs with the host of its language and with that language as I18n.locale: route helpers,
|
|
160
|
+
# paths and translated slugs get them without naming them. A language outside I18n.available_locales, which
|
|
161
|
+
# I18n may refuse, runs in the app's. Set per URL and restored per page: I18n.with_locale costs twice as much.
|
|
162
|
+
def url_by_locale_of(record, locales)
|
|
163
|
+
app_locale = I18n.locale
|
|
164
|
+
locales.filter_map do |locale|
|
|
165
|
+
hreflang, @language_host, i18n_locale = language_for(locale)
|
|
166
|
+
I18n.locale = i18n_locale || app_locale
|
|
167
|
+
location = instance_exec(record, locale, &definitions.fetch(:url))
|
|
168
|
+
[hreflang, absolute(location)] if location
|
|
169
|
+
end.to_h
|
|
170
|
+
ensure
|
|
171
|
+
@language_host = nil
|
|
172
|
+
I18n.locale = app_locale
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# [hreflang, host, I18n locale or nil when I18n does not have it], worked out once per language.
|
|
176
|
+
# The hreflang is frozen: a hash copies a key that is not, here once per URL.
|
|
177
|
+
def language_for(locale)
|
|
178
|
+
@language_by_locale ||= {}
|
|
179
|
+
@language_by_locale[locale] ||=
|
|
180
|
+
[Xml.hreflang(locale).freeze, configuration.host_for(locale), (locale if I18n.locale_available?(locale))]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def current_host
|
|
184
|
+
@language_host || configuration.host!
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def absolute(location)
|
|
188
|
+
location = location.to_s
|
|
189
|
+
if location.start_with?("//")
|
|
190
|
+
location = "#{current_host.split("//", 2).first}#{location}"
|
|
191
|
+
elsif location.start_with?("/")
|
|
192
|
+
location = "#{current_host}#{location}"
|
|
193
|
+
end
|
|
194
|
+
encoded(location)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Route helpers encode their URLs already; a URL written by hand may hold spaces or accents.
|
|
198
|
+
def encoded(url)
|
|
199
|
+
return url if url.ascii_only? && !url.match?(UNSAFE_CHARACTER)
|
|
200
|
+
|
|
201
|
+
url.b.gsub(UNSAFE_BYTE) { |byte| format("%%%02X", byte.ord) }.force_encoding(Encoding::UTF_8)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def lastmod_of(record)
|
|
205
|
+
lastmod = definitions[:lastmod]
|
|
206
|
+
return instance_exec(record, &lastmod) if lastmod.respond_to?(:call)
|
|
207
|
+
|
|
208
|
+
record.public_send(lastmod) if lastmod && record.respond_to?(lastmod)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def images_of(record)
|
|
212
|
+
definitions[:images] ? Array(instance_exec(record, &definitions[:images])).map { |location| absolute(location) } : []
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def shard_size
|
|
216
|
+
@shard_size ||= definitions[:shard_size] || default_shard_size
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def default_shard_size
|
|
220
|
+
locale_count = configuration.locales.size
|
|
221
|
+
alternates = (locale_count > 1) ? locale_count + 1 : 0
|
|
222
|
+
bytes_per_record = locale_count * (BYTES_PER_URL_ESTIMATE + alternates * BYTES_PER_ALTERNATE_ESTIMATE)
|
|
223
|
+
[ShardWriter::MAX_URLS_PER_FILE / locale_count, TARGET_BYTES_PER_FILE / bytes_per_record].min.clamp(1..)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Two updates in the same second must not look like one.
|
|
227
|
+
def precise(latest)
|
|
228
|
+
latest.respond_to?(:utc) ? latest.utc.iso8601(6) : latest.to_s
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def shard_sql
|
|
232
|
+
@shard_sql ||= begin
|
|
233
|
+
primary_key = model.primary_key
|
|
234
|
+
unless primary_key.is_a?(String) && model.type_for_attribute(primary_key).type == :integer
|
|
235
|
+
raise SourceError, "#{self.class.name} needs a scope whose model has an integer primary key"
|
|
236
|
+
end
|
|
237
|
+
# MySQL's / returns a decimal.
|
|
238
|
+
division = MYSQL_ADAPTERS.include?(model.connection_db_config.adapter) ? "DIV" : "/"
|
|
239
|
+
Arel.sql("#{model.quoted_table_name}.#{model.quoted_primary_key} #{division} #{shard_size}")
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def fingerprint_sql
|
|
244
|
+
column = definitions[:lastmod].is_a?(Symbol) ? definitions[:lastmod] : :updated_at
|
|
245
|
+
expression = definitions[:fingerprint] ||
|
|
246
|
+
"#{model.quoted_table_name}.#{model.adapter_class.quote_column_name(column)}"
|
|
247
|
+
Arel.sql("MAX(#{expression})")
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
module Storage
|
|
5
|
+
# Sends the files through a service of config/storage.yml, without creating blobs: the sitemaps reuse the
|
|
6
|
+
# credentials of the app, in a bucket of their own when the service names one.
|
|
7
|
+
class ActiveStorage
|
|
8
|
+
# A service name of config/storage.yml, or a service object.
|
|
9
|
+
def initialize(service, prefix: nil)
|
|
10
|
+
@service = service
|
|
11
|
+
@prefix = prefix
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def read(name)
|
|
15
|
+
service.download(key(name))
|
|
16
|
+
rescue ::ActiveStorage::FileNotFoundError
|
|
17
|
+
nil
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def write(name, source_path)
|
|
21
|
+
content_type = CONTENT_TYPE_BY_EXTENSION.fetch(File.extname(name))
|
|
22
|
+
File.open(source_path, "rb") { |file| service.upload(key(name), file, content_type:) }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def delete(name)
|
|
26
|
+
service.delete(key(name))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# Looked up at the first call: an initializer runs before Active Storage has read config/storage.yml.
|
|
32
|
+
def service
|
|
33
|
+
@service = ::ActiveStorage::Blob.services.fetch(@service) if @service.is_a?(Symbol) || @service.is_a?(String)
|
|
34
|
+
@service
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def key(name)
|
|
38
|
+
@prefix ? File.join(@prefix, name) : name
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
module Storage
|
|
5
|
+
CONTENT_TYPE_BY_EXTENSION = {
|
|
6
|
+
".gz" => "application/gzip", ".json" => "application/json", ".txt" => "text/plain", ".xml" => "application/xml"
|
|
7
|
+
}.freeze
|
|
8
|
+
|
|
9
|
+
class Local
|
|
10
|
+
def initialize(directory)
|
|
11
|
+
@directory = directory.to_s
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def read(name)
|
|
15
|
+
path = File.join(@directory, name)
|
|
16
|
+
File.binread(path) if File.exist?(path)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Copied beside the target, then renamed: a crawler never downloads half a file.
|
|
20
|
+
def write(name, source_path)
|
|
21
|
+
FileUtils.mkdir_p(@directory)
|
|
22
|
+
target = File.join(@directory, name)
|
|
23
|
+
partial = "#{target}.#{Process.pid}.partial"
|
|
24
|
+
FileUtils.cp(source_path, partial)
|
|
25
|
+
File.rename(partial, target)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def delete(name)
|
|
29
|
+
FileUtils.rm_f(File.join(@directory, name))
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :maptain do
|
|
4
|
+
report = lambda do |result|
|
|
5
|
+
puts "#{result.url_count} URLs#{" (full build)" if result.full}: #{result.written.size} files written, #{result.kept.size} kept, " \
|
|
6
|
+
"#{result.deleted.size} deleted, #{result.submitted} URLs sent to IndexNow"
|
|
7
|
+
result.warnings.each { |warning| warn " #{warning}" }
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
desc "Rewrite the sitemap files whose records changed since the last build"
|
|
11
|
+
task build: :environment do
|
|
12
|
+
report.call(Maptain.build)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
desc "Rewrite every sitemap file"
|
|
16
|
+
task rebuild: :environment do
|
|
17
|
+
report.call(Maptain.build(full: true))
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
desc "Render every sitemap without publishing it, and fail on what looks wrong in it"
|
|
21
|
+
task check: :environment do
|
|
22
|
+
warnings = Maptain.check
|
|
23
|
+
abort warnings.join("\n") if warnings.any?
|
|
24
|
+
puts "Nothing to report"
|
|
25
|
+
end
|
|
26
|
+
end
|
data/lib/maptain/xml.rb
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maptain
|
|
4
|
+
module Xml
|
|
5
|
+
DECLARATION = %(<?xml version="1.0" encoding="UTF-8"?>\n)
|
|
6
|
+
URLSET_OPEN = %(#{DECLARATION}<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" ) +
|
|
7
|
+
%(xmlns:xhtml="http://www.w3.org/1999/xhtml" ) +
|
|
8
|
+
%(xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">\n)
|
|
9
|
+
URLSET_CLOSE = "</urlset>\n"
|
|
10
|
+
INDEX_OPEN = %(#{DECLARATION}<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n)
|
|
11
|
+
INDEX_CLOSE = "</sitemapindex>\n"
|
|
12
|
+
# Google reads no more than 1,000 images per URL.
|
|
13
|
+
MAX_IMAGES_PER_URL = 1_000
|
|
14
|
+
# https://www.w3.org/TR/NOTE-datetime: a year, a month, a day, or a time with its zone.
|
|
15
|
+
W3C_DATE = /\A(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?\z/
|
|
16
|
+
W3C_TIME = /\A(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?(Z|[+-]\d{2}:\d{2})\z/
|
|
17
|
+
|
|
18
|
+
LOCATION = %r{<loc>([^<]*)</loc>}
|
|
19
|
+
UNESCAPED_BY_ENTITY = {"&" => "&", "<" => "<", ">" => ">", """ => '"', "'" => "'"}.freeze
|
|
20
|
+
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
# One <url> per language. Each lists every language of the page, itself included: search engines drop
|
|
24
|
+
# hreflang links that are not returned by the page they point to.
|
|
25
|
+
def url_entries(page, x_default:)
|
|
26
|
+
shared = "#{lastmod_tag(page.lastmod)}#{alternates(page, x_default)}#{images(page.images)}</url>\n"
|
|
27
|
+
page.urls.map { |url| "<url><loc>#{escape(url)}</loc>#{shared}" }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def sitemap_entry(url, lastmod)
|
|
31
|
+
"<sitemap><loc>#{escape(url)}</loc>#{lastmod_tag(lastmod)}</sitemap>\n"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# The W3C date or time of a lastmod, in UTC, or nil when it is not one.
|
|
35
|
+
def timestamp(value)
|
|
36
|
+
case value
|
|
37
|
+
when nil then nil
|
|
38
|
+
when DateTime then value.to_time.getutc.iso8601
|
|
39
|
+
when Date then value.iso8601
|
|
40
|
+
when String then w3c_timestamp(value)
|
|
41
|
+
else value.getutc.iso8601 if value.respond_to?(:getutc)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def w3c_timestamp(text)
|
|
46
|
+
if (date = W3C_DATE.match(text))
|
|
47
|
+
year, month, day = date.captures
|
|
48
|
+
text if Date.valid_date?(year.to_i, (month || 1).to_i, (day || 1).to_i)
|
|
49
|
+
elsif (time = W3C_TIME.match(text))
|
|
50
|
+
year, month, day, hour, minute, second, zone = time.captures
|
|
51
|
+
return unless Date.valid_date?(year.to_i, month.to_i, day.to_i)
|
|
52
|
+
|
|
53
|
+
Time.new(year.to_i, month.to_i, day.to_i, hour.to_i, minute.to_i, second.to_i, (zone == "Z") ? "UTC" : zone).getutc.iso8601
|
|
54
|
+
end
|
|
55
|
+
rescue ArgumentError
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# A date alone is the day as it starts first, at UTC+14: no time zone has it in the future after that.
|
|
60
|
+
def time(stamp)
|
|
61
|
+
return Time.iso8601(stamp) if stamp.include?("T")
|
|
62
|
+
|
|
63
|
+
year, month, day = stamp.split("-").map(&:to_i)
|
|
64
|
+
Time.new(year, month || 1, day || 1, 0, 0, 0, "+14:00")
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Rails locales spell regions with an underscore (pt_BR), hreflang with a hyphen (pt-BR).
|
|
68
|
+
def hreflang(locale)
|
|
69
|
+
locale.to_s.tr("_", "-")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# The <loc> of each <url> of a sitemap file.
|
|
73
|
+
def locations(xml)
|
|
74
|
+
xml.scan(LOCATION).map { |(location)| location.gsub(/&(?:amp|lt|gt|quot|apos);/, UNESCAPED_BY_ENTITY) }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def escape(text)
|
|
78
|
+
text.to_s.encode(xml: :text).gsub('"', """)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def lastmod_tag(stamp)
|
|
82
|
+
stamp ? "<lastmod>#{escape(stamp)}</lastmod>" : ""
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def alternates(page, x_default)
|
|
86
|
+
return "" if page.url_by_locale.size < 2
|
|
87
|
+
|
|
88
|
+
links = page.url_by_locale.map { |locale, url| alternate(locale, url) }
|
|
89
|
+
default_url = page.url_by_locale[x_default]
|
|
90
|
+
links << alternate("x-default", default_url) if default_url
|
|
91
|
+
links.join
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def alternate(hreflang, url)
|
|
95
|
+
%(<xhtml:link rel="alternate" hreflang="#{escape(hreflang)}" href="#{escape(url)}"/>)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def images(urls)
|
|
99
|
+
urls.first(MAX_IMAGES_PER_URL).map { |url| "<image:image><image:loc>#{escape(url)}</image:loc></image:image>" }.join
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
data/lib/maptain.rb
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support"
|
|
4
|
+
require "active_support/core_ext/object/blank"
|
|
5
|
+
require "active_support/core_ext/string/inflections"
|
|
6
|
+
require "active_support/notifications"
|
|
7
|
+
require "date"
|
|
8
|
+
require "digest"
|
|
9
|
+
require "fileutils"
|
|
10
|
+
require "json"
|
|
11
|
+
require "net/http"
|
|
12
|
+
require "time"
|
|
13
|
+
require "tmpdir"
|
|
14
|
+
require "uri"
|
|
15
|
+
require "zlib"
|
|
16
|
+
|
|
17
|
+
require_relative "maptain/version"
|
|
18
|
+
require_relative "maptain/errors"
|
|
19
|
+
require_relative "maptain/storage/local"
|
|
20
|
+
require_relative "maptain/storage/active_storage"
|
|
21
|
+
require_relative "maptain/storage/null"
|
|
22
|
+
require_relative "maptain/configuration"
|
|
23
|
+
require_relative "maptain/page"
|
|
24
|
+
require_relative "maptain/xml"
|
|
25
|
+
require_relative "maptain/source"
|
|
26
|
+
require_relative "maptain/shard_writer"
|
|
27
|
+
require_relative "maptain/manifest"
|
|
28
|
+
require_relative "maptain/index_now"
|
|
29
|
+
require_relative "maptain/builder"
|
|
30
|
+
require_relative "maptain/railtie" if defined?(Rails::Railtie)
|
|
31
|
+
|
|
32
|
+
module Maptain
|
|
33
|
+
class << self
|
|
34
|
+
def configuration
|
|
35
|
+
@configuration ||= Configuration.new
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def configure
|
|
39
|
+
yield configuration
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def reset_configuration
|
|
43
|
+
@configuration = nil
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# full: true rewrites every file, whatever the manifest of the last build says.
|
|
47
|
+
def build(full: false)
|
|
48
|
+
Builder.new(configuration).build(full:)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Renders every sitemap without writing or submitting anything, and returns what looks wrong in them.
|
|
52
|
+
def check
|
|
53
|
+
Builder.new(configuration, storage: Storage::Null.new, submit: false).build(full: true).warnings
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: maptain
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Level UP Solutions
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: activesupport
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
-
|
|
17
|
+
- ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: "8.0"
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
-
|
|
25
|
+
- ">="
|
|
26
|
+
- !ruby/object:Gem::Version
|
|
27
|
+
version: "8.0"
|
|
28
|
+
description: "Each kind of page is a small class that names its records, its URLs and its languages. Maptain writes the hreflang links itself, cuts the sitemap into files that keep the same records from one build to the next, and rewrites only the files whose records changed."
|
|
29
|
+
executables: []
|
|
30
|
+
extensions: []
|
|
31
|
+
extra_rdoc_files: []
|
|
32
|
+
files:
|
|
33
|
+
- CHANGELOG.md
|
|
34
|
+
- LICENSE
|
|
35
|
+
- README.md
|
|
36
|
+
- lib/maptain.rb
|
|
37
|
+
- lib/maptain/builder.rb
|
|
38
|
+
- lib/maptain/configuration.rb
|
|
39
|
+
- lib/maptain/errors.rb
|
|
40
|
+
- lib/maptain/index_now.rb
|
|
41
|
+
- lib/maptain/manifest.rb
|
|
42
|
+
- lib/maptain/page.rb
|
|
43
|
+
- lib/maptain/railtie.rb
|
|
44
|
+
- lib/maptain/shard_writer.rb
|
|
45
|
+
- lib/maptain/source.rb
|
|
46
|
+
- lib/maptain/storage/active_storage.rb
|
|
47
|
+
- lib/maptain/storage/local.rb
|
|
48
|
+
- lib/maptain/storage/null.rb
|
|
49
|
+
- lib/maptain/tasks.rake
|
|
50
|
+
- lib/maptain/version.rb
|
|
51
|
+
- lib/maptain/xml.rb
|
|
52
|
+
homepage: "https://github.com/levups/maptain"
|
|
53
|
+
licenses:
|
|
54
|
+
- MIT
|
|
55
|
+
metadata:
|
|
56
|
+
source_code_uri: "https://github.com/levups/maptain"
|
|
57
|
+
changelog_uri: "https://github.com/levups/maptain/blob/main/CHANGELOG.md"
|
|
58
|
+
documentation_uri: "https://levups.github.io/maptain/"
|
|
59
|
+
bug_tracker_uri: "https://github.com/levups/maptain/issues"
|
|
60
|
+
rubygems_mfa_required: "true"
|
|
61
|
+
rdoc_options: []
|
|
62
|
+
require_paths:
|
|
63
|
+
- lib
|
|
64
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
-
|
|
67
|
+
- ">="
|
|
68
|
+
- !ruby/object:Gem::Version
|
|
69
|
+
version: "3.3"
|
|
70
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
-
|
|
73
|
+
- ">="
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: "0"
|
|
76
|
+
requirements: []
|
|
77
|
+
rubygems_version: 4.1.0
|
|
78
|
+
specification_version: 4
|
|
79
|
+
summary: "XML sitemaps for Rails sites in many languages, rebuilt only where the content changed"
|
|
80
|
+
test_files: []
|