textus 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +163 -0
- data/README.md +200 -0
- data/SPEC.md +720 -0
- data/docs/architecture.md +57 -0
- data/docs/conventions.md +85 -0
- data/exe/textus +4 -0
- data/lib/textus/audit_log.rb +32 -0
- data/lib/textus/builder.rb +191 -0
- data/lib/textus/builtin_fetchers.rb +63 -0
- data/lib/textus/cli.rb +394 -0
- data/lib/textus/dependencies.rb +23 -0
- data/lib/textus/doctor.rb +281 -0
- data/lib/textus/entry/json.rb +41 -0
- data/lib/textus/entry/markdown.rb +39 -0
- data/lib/textus/entry/text.rb +23 -0
- data/lib/textus/entry/yaml.rb +39 -0
- data/lib/textus/entry.rb +30 -0
- data/lib/textus/errors.rb +168 -0
- data/lib/textus/etag.rb +13 -0
- data/lib/textus/extension_registry.rb +48 -0
- data/lib/textus/extensions.rb +29 -0
- data/lib/textus/init.rb +51 -0
- data/lib/textus/intro.rb +104 -0
- data/lib/textus/key_distance.rb +53 -0
- data/lib/textus/manifest.rb +394 -0
- data/lib/textus/migrate_keys.rb +187 -0
- data/lib/textus/mustache.rb +117 -0
- data/lib/textus/projection.rb +80 -0
- data/lib/textus/proposal.rb +27 -0
- data/lib/textus/publisher.rb +71 -0
- data/lib/textus/refresh.rb +75 -0
- data/lib/textus/role.rb +20 -0
- data/lib/textus/schema.rb +90 -0
- data/lib/textus/schema_tools.rb +87 -0
- data/lib/textus/store.rb +607 -0
- data/lib/textus/store_view.rb +18 -0
- data/lib/textus/version.rb +4 -0
- data/lib/textus.rb +31 -0
- metadata +156 -0
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
require "yaml"
|
|
2
|
+
|
|
3
|
+
module Textus
|
|
4
|
+
class Manifest
|
|
5
|
+
# New stricter grammar: lowercase + digits + internal hyphens. No underscores.
|
|
6
|
+
KEY_SEGMENT = /\A[a-z0-9][a-z0-9-]*\z/
|
|
7
|
+
MAX_SEGMENTS = 8
|
|
8
|
+
MAX_SEGMENT_LEN = 64
|
|
9
|
+
|
|
10
|
+
EXT_TO_FORMAT = {
|
|
11
|
+
".md" => "markdown",
|
|
12
|
+
".json" => "json",
|
|
13
|
+
".yaml" => "yaml",
|
|
14
|
+
".yml" => "yaml",
|
|
15
|
+
".txt" => "text",
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
attr_reader :root, :entries, :raw
|
|
19
|
+
|
|
20
|
+
LEGACY_ZONES = {
|
|
21
|
+
"fixed" => ["human"],
|
|
22
|
+
"state" => %w[human ai script],
|
|
23
|
+
"derived" => ["build"],
|
|
24
|
+
}.freeze
|
|
25
|
+
|
|
26
|
+
def zones
|
|
27
|
+
@zones ||= begin
|
|
28
|
+
declared = Array(@raw["zones"])
|
|
29
|
+
if declared.empty?
|
|
30
|
+
LEGACY_ZONES.transform_values(&:dup)
|
|
31
|
+
else
|
|
32
|
+
declared.to_h do |z|
|
|
33
|
+
[z["name"], Array(z["writable_by"])]
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def zone_writers(zone_name)
|
|
40
|
+
zones[zone_name] or raise UsageError.new("undeclared zone '#{zone_name}'")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.load(root)
|
|
44
|
+
manifest_path = File.join(root, "manifest.yaml")
|
|
45
|
+
raise IoError.new("manifest not found: #{manifest_path}") unless File.exist?(manifest_path)
|
|
46
|
+
|
|
47
|
+
raw = YAML.safe_load_file(manifest_path, aliases: false)
|
|
48
|
+
raise BadFrontmatter.new(manifest_path, "unsupported manifest version #{raw["version"].inspect}") unless raw["version"] == PROTOCOL
|
|
49
|
+
|
|
50
|
+
new(root, raw)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def initialize(root, raw)
|
|
54
|
+
@root = root
|
|
55
|
+
@raw = raw
|
|
56
|
+
@entries = Array(raw["entries"]).map { |e| ManifestEntry.new(self, e) }
|
|
57
|
+
validate_declared_keys!
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Returns [ManifestEntry, resolved_path, remaining_segments]
|
|
61
|
+
def resolve(key)
|
|
62
|
+
validate_key!(key)
|
|
63
|
+
segments = key.split(".")
|
|
64
|
+
# longest-prefix match
|
|
65
|
+
candidates = @entries
|
|
66
|
+
.map { |e| [e, e.key.split(".")] }
|
|
67
|
+
.select { |(_, esegs)| esegs == segments[0, esegs.length] }
|
|
68
|
+
.sort_by { |(_, esegs)| -esegs.length }
|
|
69
|
+
raise UnknownKey.new(key, suggestions: suggestions_for(key)) if candidates.empty?
|
|
70
|
+
|
|
71
|
+
entry, esegs = candidates.first
|
|
72
|
+
remaining = segments[esegs.length..]
|
|
73
|
+
if remaining.empty?
|
|
74
|
+
path = resolve_leaf_path(entry)
|
|
75
|
+
[entry, path, []]
|
|
76
|
+
else
|
|
77
|
+
raise UnknownKey.new(key, suggestions: suggestions_for(key)) unless entry.nested
|
|
78
|
+
|
|
79
|
+
primary_ext = Entry.for_format(entry.format).extensions.first
|
|
80
|
+
path = File.join(@root, "zones", entry.path, *remaining) + primary_ext
|
|
81
|
+
[entry, path, remaining]
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Returns up to 5 dotted keys from the manifest that look similar to the
|
|
86
|
+
# requested key, ranked by shared-prefix length then Levenshtein distance.
|
|
87
|
+
def suggestions_for(key)
|
|
88
|
+
candidates = enumerate.map { |r| r[:key] }
|
|
89
|
+
# Include declared (non-nested) entry keys even if file is missing.
|
|
90
|
+
candidates.concat(@entries.reject(&:nested).map(&:key))
|
|
91
|
+
candidates.uniq!
|
|
92
|
+
KeyDistance.suggest(key, candidates, limit: 5)
|
|
93
|
+
rescue StandardError
|
|
94
|
+
[]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Enumerate all entry files reachable through the manifest. Returns
|
|
98
|
+
# [{ key:, path:, manifest_entry: }, ...]
|
|
99
|
+
# rubocop:disable Metrics/AbcSize
|
|
100
|
+
def enumerate(prefix: nil)
|
|
101
|
+
out = []
|
|
102
|
+
@entries.each do |entry|
|
|
103
|
+
if entry.nested
|
|
104
|
+
base = File.join(@root, "zones", entry.path)
|
|
105
|
+
next unless File.directory?(base)
|
|
106
|
+
|
|
107
|
+
glob_pattern = nested_glob(entry.format)
|
|
108
|
+
Dir.glob(File.join(base, glob_pattern)).each do |fp|
|
|
109
|
+
rel = fp.sub(%r{\A#{Regexp.escape(base)}/?}, "")
|
|
110
|
+
stripped = rel.sub(/#{Regexp.escape(File.extname(rel))}\z/, "")
|
|
111
|
+
segs = stripped.split("/").reject(&:empty?)
|
|
112
|
+
next if segs.empty?
|
|
113
|
+
|
|
114
|
+
illegal = segs.find { |s| !valid_segment?(s) }
|
|
115
|
+
if illegal
|
|
116
|
+
warn("textus: skipping illegal key segment '#{illegal}' at #{fp} — run 'textus migrate-keys --dry-run'")
|
|
117
|
+
next
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
full_key = (entry.key.split(".") + segs).join(".")
|
|
121
|
+
out << { key: full_key, path: fp, manifest_entry: entry }
|
|
122
|
+
end
|
|
123
|
+
else
|
|
124
|
+
fp = resolve_leaf_path(entry)
|
|
125
|
+
out << { key: entry.key, path: fp, manifest_entry: entry } if File.exist?(fp)
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
out.select! { |row| row[:key] == prefix || row[:key].start_with?("#{prefix}.") } if prefix
|
|
129
|
+
out.sort_by { |row| row[:key] }
|
|
130
|
+
end
|
|
131
|
+
# rubocop:enable Metrics/AbcSize
|
|
132
|
+
|
|
133
|
+
# Validates all declared entry keys; raises UsageError listing all offenders.
|
|
134
|
+
def validate_keys!
|
|
135
|
+
offenders = []
|
|
136
|
+
@entries.each do |entry|
|
|
137
|
+
validate_key!(entry.key)
|
|
138
|
+
rescue UsageError => e
|
|
139
|
+
offenders << e.message
|
|
140
|
+
end
|
|
141
|
+
raise UsageError.new("invalid manifest keys: #{offenders.join("; ")}") unless offenders.empty?
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def validate_key!(key)
|
|
145
|
+
raise UsageError.new("empty key") if key.nil? || key.empty?
|
|
146
|
+
|
|
147
|
+
segs = key.split(".")
|
|
148
|
+
raise UsageError.new("key '#{key}' has #{segs.length} segments (max #{MAX_SEGMENTS})") if segs.length > MAX_SEGMENTS
|
|
149
|
+
|
|
150
|
+
segs.each do |seg|
|
|
151
|
+
if seg.empty?
|
|
152
|
+
raise UsageError.new("empty segment in key '#{key}'")
|
|
153
|
+
elsif seg.length > MAX_SEGMENT_LEN
|
|
154
|
+
raise UsageError.new("segment '#{seg}' in key '#{key}' exceeds #{MAX_SEGMENT_LEN} chars")
|
|
155
|
+
elsif !seg.match?(KEY_SEGMENT)
|
|
156
|
+
raise UsageError.new(
|
|
157
|
+
"invalid key segment '#{seg}' in '#{key}': must match [a-z0-9][a-z0-9-]* " \
|
|
158
|
+
"(lowercase, digits, hyphens; no underscores or uppercase)",
|
|
159
|
+
)
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
private
|
|
165
|
+
|
|
166
|
+
def valid_segment?(seg)
|
|
167
|
+
return false if seg.nil? || seg.empty?
|
|
168
|
+
return false if seg.length > MAX_SEGMENT_LEN
|
|
169
|
+
|
|
170
|
+
seg.match?(KEY_SEGMENT)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def validate_declared_keys!
|
|
174
|
+
@entries.each { |e| validate_key!(e.key) }
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def resolve_leaf_path(entry)
|
|
178
|
+
primary_ext = Entry.for_format(entry.format).extensions.first
|
|
179
|
+
if File.extname(entry.path) == ""
|
|
180
|
+
File.join(@root, "zones", entry.path + primary_ext)
|
|
181
|
+
else
|
|
182
|
+
File.join(@root, "zones", entry.path)
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def nested_glob(format)
|
|
187
|
+
case format
|
|
188
|
+
when "markdown" then "**/*.md"
|
|
189
|
+
when "json" then "**/*.json"
|
|
190
|
+
when "yaml" then "**/*.{yaml,yml}"
|
|
191
|
+
when "text" then "**/*.txt"
|
|
192
|
+
else raise UsageError.new("unknown format #{format.inspect} for nested glob")
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
class ManifestEntry
|
|
198
|
+
PUBLISH_EACH_VARS = %w[leaf basename key ext].freeze
|
|
199
|
+
PUBLISH_EACH_VAR_RE = /\{([a-z]+)\}/
|
|
200
|
+
|
|
201
|
+
attr_reader :key, :path, :zone, :schema, :owner, :nested, :generator, :raw, :format,
|
|
202
|
+
:projection, :template, :publish_to, :publish_each, :fetcher, :fetcher_config, :ttl, :events,
|
|
203
|
+
:inject_intro
|
|
204
|
+
|
|
205
|
+
def initialize(manifest, raw)
|
|
206
|
+
@manifest = manifest
|
|
207
|
+
@raw = raw
|
|
208
|
+
@key = raw["key"] or raise UsageError.new("manifest entry missing key")
|
|
209
|
+
@path = raw["path"] or raise UsageError.new("manifest entry '#{@key}' missing path")
|
|
210
|
+
@zone = raw["zone"] or raise UsageError.new("manifest entry '#{@key}' missing zone")
|
|
211
|
+
@schema = raw["schema"]
|
|
212
|
+
@owner = raw["owner"]
|
|
213
|
+
@nested = raw["nested"] == true
|
|
214
|
+
@generator = raw["generator"]
|
|
215
|
+
@projection = raw["projection"]
|
|
216
|
+
@template = raw["template"]
|
|
217
|
+
@publish_to = Array(raw["publish_to"])
|
|
218
|
+
@publish_each = raw["publish_each"]
|
|
219
|
+
@events = raw["events"] || {}
|
|
220
|
+
@inject_intro = raw["inject_intro"] == true
|
|
221
|
+
@format = resolve_format!(raw["format"])
|
|
222
|
+
|
|
223
|
+
reject_legacy!(raw)
|
|
224
|
+
parse_source!(raw["source"])
|
|
225
|
+
validate_format_matrix!
|
|
226
|
+
validate_publish_each!
|
|
227
|
+
validate_inject_intro!
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Resolves the per-leaf target path (relative to repo root) for a full
|
|
231
|
+
# dotted key under this entry's prefix. Returns nil if this entry has no
|
|
232
|
+
# publish_each template.
|
|
233
|
+
def publish_target_for(full_key)
|
|
234
|
+
return nil if @publish_each.nil?
|
|
235
|
+
|
|
236
|
+
entry_segs = @key.split(".")
|
|
237
|
+
key_segs = full_key.split(".")
|
|
238
|
+
raise UsageError.new("key '#{full_key}' is not under entry '#{@key}'") unless key_segs[0, entry_segs.length] == entry_segs
|
|
239
|
+
|
|
240
|
+
remaining = key_segs[entry_segs.length..] || []
|
|
241
|
+
leaf = remaining.join("/")
|
|
242
|
+
basename = remaining.last || ""
|
|
243
|
+
ext = Entry.for_format(@format).extensions.first.to_s.sub(/^\./, "")
|
|
244
|
+
|
|
245
|
+
vars = { "leaf" => leaf, "basename" => basename, "key" => full_key, "ext" => ext }
|
|
246
|
+
@publish_each.gsub(PUBLISH_EACH_VAR_RE) { vars.fetch(::Regexp.last_match(1)) }
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def derived?
|
|
250
|
+
writers = @manifest.zone_writers(@zone)
|
|
251
|
+
writers.include?("build")
|
|
252
|
+
rescue UsageError => e
|
|
253
|
+
raise UsageError.new("entry '#{@key}': #{e.message}")
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
private
|
|
257
|
+
|
|
258
|
+
def validate_inject_intro!
|
|
259
|
+
return unless @inject_intro
|
|
260
|
+
|
|
261
|
+
unless derived?
|
|
262
|
+
raise UsageError.new(
|
|
263
|
+
"entry '#{@key}': inject_intro: is only valid on derived entries",
|
|
264
|
+
)
|
|
265
|
+
end
|
|
266
|
+
return unless @template.nil?
|
|
267
|
+
|
|
268
|
+
raise UsageError.new(
|
|
269
|
+
"entry '#{@key}': inject_intro: requires a template:",
|
|
270
|
+
)
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def validate_publish_each!
|
|
274
|
+
return if @publish_each.nil?
|
|
275
|
+
|
|
276
|
+
raise UsageError.new("entry '#{@key}': publish_each requires nested: true") unless @nested
|
|
277
|
+
raise UsageError.new("entry '#{@key}': publish_to and publish_each are mutually exclusive") unless @publish_to.empty?
|
|
278
|
+
raise UsageError.new("entry '#{@key}': publish_each must be a string") unless @publish_each.is_a?(String)
|
|
279
|
+
|
|
280
|
+
used_vars = @publish_each.scan(PUBLISH_EACH_VAR_RE).flatten
|
|
281
|
+
unknown = used_vars - PUBLISH_EACH_VARS
|
|
282
|
+
unless unknown.empty?
|
|
283
|
+
raise UsageError.new(
|
|
284
|
+
"entry '#{@key}': publish_each uses unknown template variable(s) " \
|
|
285
|
+
"#{unknown.map { |v| "{#{v}}" }.join(", ")}. Known: #{PUBLISH_EACH_VARS.map { |v| "{#{v}}" }.join(", ")}.",
|
|
286
|
+
)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
required = %w[leaf basename key]
|
|
290
|
+
return if used_vars.any? { |v| required.include?(v) }
|
|
291
|
+
|
|
292
|
+
raise UsageError.new(
|
|
293
|
+
"entry '#{@key}': publish_each must reference at least one of {leaf}, {basename}, or {key} " \
|
|
294
|
+
"(else every leaf would clobber the same target).",
|
|
295
|
+
)
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
def resolve_format!(declared)
|
|
299
|
+
ext = File.extname(@path)
|
|
300
|
+
inferred = Manifest::EXT_TO_FORMAT[ext]
|
|
301
|
+
|
|
302
|
+
if declared.nil?
|
|
303
|
+
return inferred if inferred
|
|
304
|
+
# No extension: nested defaults to markdown, leaf with no ext also markdown.
|
|
305
|
+
return "markdown" if ext == "" && @nested
|
|
306
|
+
return "markdown" if ext == ""
|
|
307
|
+
else
|
|
308
|
+
raise UsageError.new("entry '#{@key}': unknown format #{declared.inspect}") unless Manifest::EXT_TO_FORMAT.values.include?(declared)
|
|
309
|
+
# If the path has an extension, the declared format must match.
|
|
310
|
+
if ext != "" && inferred && inferred != declared
|
|
311
|
+
raise UsageError.new(
|
|
312
|
+
"entry '#{@key}': path extension #{ext.inspect} does not match declared format #{declared.inspect}",
|
|
313
|
+
)
|
|
314
|
+
end
|
|
315
|
+
return declared
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
"markdown"
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
322
|
+
def validate_format_matrix!
|
|
323
|
+
ext = File.extname(@path)
|
|
324
|
+
|
|
325
|
+
case @format
|
|
326
|
+
when "markdown"
|
|
327
|
+
# .md, or no extension (will be appended). Anything else is a mismatch caught above.
|
|
328
|
+
raise UsageError.new("entry '#{@key}': markdown format requires '.md' path (got #{ext.inspect})") if ext != "" && ext != ".md"
|
|
329
|
+
when "json"
|
|
330
|
+
if @nested
|
|
331
|
+
# nested json: path is a directory; ext must be empty.
|
|
332
|
+
raise UsageError.new("entry '#{@key}': nested json path must not have an extension") if ext != ""
|
|
333
|
+
elsif ext != ".json"
|
|
334
|
+
raise UsageError.new("entry '#{@key}': json format requires '.json' path (got #{ext.inspect})")
|
|
335
|
+
end
|
|
336
|
+
when "yaml"
|
|
337
|
+
if @nested
|
|
338
|
+
raise UsageError.new("entry '#{@key}': nested yaml path must not have an extension") if ext != ""
|
|
339
|
+
elsif ext != ".yaml" && ext != ".yml"
|
|
340
|
+
raise UsageError.new("entry '#{@key}': yaml format requires '.yaml' or '.yml' path (got #{ext.inspect})")
|
|
341
|
+
end
|
|
342
|
+
when "text"
|
|
343
|
+
if @nested
|
|
344
|
+
raise UsageError.new("entry '#{@key}': nested text path must not have an extension") if ext != ""
|
|
345
|
+
elsif ext != ".txt" && ext != ""
|
|
346
|
+
raise UsageError.new("entry '#{@key}': text format requires '.txt' or no extension (got #{ext.inspect})")
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# Schema rules.
|
|
351
|
+
raise UsageError.new("entry '#{@key}': text format must not declare a schema") if @format == "text" && !@schema.nil?
|
|
352
|
+
|
|
353
|
+
# Template-required-for-derived rules. Skipped for entries materialized by an
|
|
354
|
+
# external generator: command (those produce the bytes themselves).
|
|
355
|
+
if derived? && @template.nil? && @generator.nil? &&
|
|
356
|
+
(@format == "markdown" || @format == "text") && !@nested
|
|
357
|
+
raise UsageError.new("entry '#{@key}': derived #{@format} entries require a template")
|
|
358
|
+
end
|
|
359
|
+
end
|
|
360
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
361
|
+
|
|
362
|
+
def parse_source!(src)
|
|
363
|
+
src ||= {}
|
|
364
|
+
@fetcher = src["fetcher"]
|
|
365
|
+
@fetcher_config = src["config"] || {}
|
|
366
|
+
@ttl = src["ttl"]
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
def reject_legacy!(raw)
|
|
370
|
+
src = raw["source"] || {}
|
|
371
|
+
if src.key?("parse") || src.key?("from")
|
|
372
|
+
raise UsageError.new(
|
|
373
|
+
"entry '#{@key}': source.parse/source.from removed in 0.2; " \
|
|
374
|
+
"use source.fetcher (+ source.config). See SPEC §5.4.",
|
|
375
|
+
)
|
|
376
|
+
end
|
|
377
|
+
if raw.key?("hooks")
|
|
378
|
+
raise UsageError.new(
|
|
379
|
+
"entry '#{@key}': 'hooks:' renamed to 'events:' in 0.2; " \
|
|
380
|
+
"remove on_ prefix from event names. See SPEC §5.10.",
|
|
381
|
+
)
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
@events.each_key do |evt|
|
|
385
|
+
next if ExtensionRegistry::EVENTS.include?(evt.to_sym)
|
|
386
|
+
|
|
387
|
+
raise UsageError.new(
|
|
388
|
+
"entry '#{@key}': unknown event '#{evt}' in events: block. " \
|
|
389
|
+
"Known events: #{ExtensionRegistry::EVENTS.join(", ")}.",
|
|
390
|
+
)
|
|
391
|
+
end
|
|
392
|
+
end
|
|
393
|
+
end
|
|
394
|
+
end
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
module Textus
|
|
2
|
+
# Run-once helper that renames files/directories whose basenames don't
|
|
3
|
+
# conform to the strict key grammar (§3 of plan-1.2). Only walks
|
|
4
|
+
# nested: true manifest entries — leaf entries with illegal declared
|
|
5
|
+
# keys are caught by Manifest load and must be fixed by hand.
|
|
6
|
+
module MigrateKeys
|
|
7
|
+
SEGMENT = /\A[a-z0-9][a-z0-9-]*\z/
|
|
8
|
+
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# Returns the envelope hash described in plan-1.2 §3.
|
|
12
|
+
def run(store, write: false)
|
|
13
|
+
plan = build_plan(store)
|
|
14
|
+
collisions = plan[:collisions]
|
|
15
|
+
renames = plan[:renames]
|
|
16
|
+
|
|
17
|
+
ok = collisions.empty?
|
|
18
|
+
apply!(store, renames) if write && ok
|
|
19
|
+
|
|
20
|
+
{
|
|
21
|
+
"protocol" => Textus::PROTOCOL,
|
|
22
|
+
"mode" => write ? "write" : "dry-run",
|
|
23
|
+
"renames" => renames.map { |r| envelope_rename(r) },
|
|
24
|
+
"collisions" => collisions.map { |c| envelope_collision(c) },
|
|
25
|
+
"ok" => ok,
|
|
26
|
+
}
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# ------------------------------------------------------------------
|
|
30
|
+
# Plan construction
|
|
31
|
+
# ------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
# Returns { renames: [...], collisions: [...] }
|
|
34
|
+
# Each rename: { from:, to:, old_key:, new_key:, kind: :file|:dir }
|
|
35
|
+
# Each collision: { target:, sources: [...] }
|
|
36
|
+
def build_plan(store) # rubocop:disable Metrics/AbcSize
|
|
37
|
+
renames = []
|
|
38
|
+
target_buckets = Hash.new { |h, k| h[k] = [] } # target_path => [source_path, ...]
|
|
39
|
+
|
|
40
|
+
store.manifest.entries.each do |entry|
|
|
41
|
+
next unless entry.nested
|
|
42
|
+
|
|
43
|
+
base = File.join(store.root, "zones", entry.path)
|
|
44
|
+
next unless File.directory?(base)
|
|
45
|
+
|
|
46
|
+
# Walk depth-first. Order matters when computing the "new key"
|
|
47
|
+
# for files inside a renamed directory: we record renames bottom-up,
|
|
48
|
+
# so children are renamed before their parents on apply.
|
|
49
|
+
walk(base) do |abs_path, is_dir|
|
|
50
|
+
next if abs_path == base
|
|
51
|
+
|
|
52
|
+
basename = File.basename(abs_path)
|
|
53
|
+
stem = is_dir ? basename : basename.sub(/#{Regexp.escape(File.extname(basename))}\z/, "")
|
|
54
|
+
next if stem.match?(SEGMENT)
|
|
55
|
+
|
|
56
|
+
new_stem = normalize(stem)
|
|
57
|
+
# Skip if normalization yields the same stem (e.g. already-legal
|
|
58
|
+
# under a different lens). In practice match?(SEGMENT) catches that
|
|
59
|
+
# above; this is a safety net.
|
|
60
|
+
next if new_stem == stem
|
|
61
|
+
|
|
62
|
+
new_basename = is_dir ? new_stem : new_stem + File.extname(basename)
|
|
63
|
+
target = File.join(File.dirname(abs_path), new_basename)
|
|
64
|
+
target_buckets[target] << abs_path
|
|
65
|
+
|
|
66
|
+
renames << {
|
|
67
|
+
from: abs_path,
|
|
68
|
+
to: target,
|
|
69
|
+
kind: is_dir ? :dir : :file,
|
|
70
|
+
entry: entry,
|
|
71
|
+
base: base,
|
|
72
|
+
}
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
collisions = target_buckets.select { |_, srcs| srcs.length > 1 }
|
|
77
|
+
.map { |t, srcs| { target: t, sources: srcs.sort } }
|
|
78
|
+
|
|
79
|
+
# Drop colliding entries from renames (we won't apply any of them)
|
|
80
|
+
colliding_targets = collisions.to_set { |c| c[:target] }
|
|
81
|
+
renames.reject! { |r| colliding_targets.include?(r[:to]) }
|
|
82
|
+
|
|
83
|
+
# Sort renames bottom-up (deepest path first) so children move before parents.
|
|
84
|
+
renames.sort_by! { |r| -r[:from].count("/") }
|
|
85
|
+
|
|
86
|
+
{ renames: renames, collisions: collisions }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Yields [absolute_path, is_dir] for every entry under root. Depth-first.
|
|
90
|
+
def walk(root, &block)
|
|
91
|
+
Dir.each_child(root) do |name|
|
|
92
|
+
abs = File.join(root, name)
|
|
93
|
+
if File.directory?(abs)
|
|
94
|
+
walk(abs, &block)
|
|
95
|
+
yield abs, true
|
|
96
|
+
else
|
|
97
|
+
yield abs, false
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Deterministic transform per plan §3.
|
|
103
|
+
def normalize(s)
|
|
104
|
+
s = s.downcase
|
|
105
|
+
s = s.gsub(/[^a-z0-9-]/, "-") # ., _, and anything else become -
|
|
106
|
+
s = s.gsub(/-+/, "-")
|
|
107
|
+
s.sub(/\A-+/, "").sub(/-+\z/, "")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# ------------------------------------------------------------------
|
|
111
|
+
# Apply
|
|
112
|
+
# ------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
def apply!(store, renames)
|
|
115
|
+
audit = AuditLog.new(store.root)
|
|
116
|
+
renames.each do |r|
|
|
117
|
+
# Bottom-up order means a child's ancestors haven't moved yet, so
|
|
118
|
+
# `from`/`to` are valid as-recorded. The audit `key` reflects the
|
|
119
|
+
# eventual full key once every rename in this batch has applied.
|
|
120
|
+
from = r[:from]
|
|
121
|
+
to = r[:to]
|
|
122
|
+
File.rename(from, to)
|
|
123
|
+
new_key = compute_new_key(r, renames)
|
|
124
|
+
audit.append(
|
|
125
|
+
role: "script",
|
|
126
|
+
verb: "migrate-keys",
|
|
127
|
+
key: new_key,
|
|
128
|
+
etag_before: nil,
|
|
129
|
+
etag_after: nil,
|
|
130
|
+
extras: { "from" => from, "to" => to },
|
|
131
|
+
)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# If an ancestor of `path` was renamed earlier in this batch, rewrite the path.
|
|
136
|
+
def resolve_current_path(path, renames)
|
|
137
|
+
out = path
|
|
138
|
+
renames.each do |r|
|
|
139
|
+
prefix = r[:from] + "/"
|
|
140
|
+
out = r[:to] + out[r[:from].length..] if out.start_with?(prefix)
|
|
141
|
+
end
|
|
142
|
+
out
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# New full key after applying all renames up through this one.
|
|
146
|
+
def compute_new_key(rename, renames)
|
|
147
|
+
base = rename[:base]
|
|
148
|
+
entry = rename[:entry]
|
|
149
|
+
new_to = resolve_current_path(rename[:to], renames)
|
|
150
|
+
|
|
151
|
+
rel = new_to.sub(%r{\A#{Regexp.escape(base)}/?}, "")
|
|
152
|
+
stripped = rel.sub(/#{Regexp.escape(File.extname(rel))}\z/, "") unless rename[:kind] == :dir
|
|
153
|
+
stripped ||= rel
|
|
154
|
+
segs = stripped.split("/").reject(&:empty?)
|
|
155
|
+
(entry.key.split(".") + segs).join(".")
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# ------------------------------------------------------------------
|
|
159
|
+
# Envelope helpers
|
|
160
|
+
# ------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
def envelope_rename(r)
|
|
163
|
+
{
|
|
164
|
+
"from" => r[:from],
|
|
165
|
+
"to" => r[:to],
|
|
166
|
+
"old_key" => path_to_key(r[:from], r[:base], r[:entry], r[:kind]),
|
|
167
|
+
"new_key" => path_to_key(r[:to], r[:base], r[:entry], r[:kind]),
|
|
168
|
+
}
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def envelope_collision(col)
|
|
172
|
+
{ "target" => col[:target], "sources" => col[:sources] }
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def path_to_key(path, base, entry, kind)
|
|
176
|
+
rel = path.sub(%r{\A#{Regexp.escape(base)}/?}, "")
|
|
177
|
+
stripped =
|
|
178
|
+
if kind == :dir
|
|
179
|
+
rel
|
|
180
|
+
else
|
|
181
|
+
rel.sub(/#{Regexp.escape(File.extname(rel))}\z/, "")
|
|
182
|
+
end
|
|
183
|
+
segs = stripped.split("/").reject(&:empty?)
|
|
184
|
+
(entry.key.split(".") + segs).join(".")
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
module Textus
|
|
2
|
+
module Mustache
|
|
3
|
+
MAX_DEPTH = 8
|
|
4
|
+
TAG = %r{\{\{(?<sigil>[#^/!&]?)\s*(?<name>[\w.-]+)\s*\}\}}
|
|
5
|
+
|
|
6
|
+
def self.render(template, context, strict: false, depth: 0) # rubocop:disable Metrics/AbcSize
|
|
7
|
+
raise TemplateError.new("template recursion depth #{depth} exceeded #{MAX_DEPTH}") if depth > MAX_DEPTH
|
|
8
|
+
|
|
9
|
+
out = +""
|
|
10
|
+
pos = 0
|
|
11
|
+
while (m = template.match(TAG, pos))
|
|
12
|
+
out << template[pos...m.begin(0)]
|
|
13
|
+
case m[:sigil]
|
|
14
|
+
when "!"
|
|
15
|
+
# comment, skip
|
|
16
|
+
when "#"
|
|
17
|
+
section, new_pos = parse_section(template, m, m[:name])
|
|
18
|
+
value = lookup(context, m[:name])
|
|
19
|
+
out << render_section(section, value, context, strict, depth)
|
|
20
|
+
pos = new_pos
|
|
21
|
+
next
|
|
22
|
+
when "^"
|
|
23
|
+
section, new_pos = parse_section(template, m, m[:name])
|
|
24
|
+
value = lookup(context, m[:name])
|
|
25
|
+
if falsy?(value)
|
|
26
|
+
raise TemplateError.new("template recursion depth #{depth + 1} exceeded #{MAX_DEPTH}") if depth + 1 > MAX_DEPTH
|
|
27
|
+
|
|
28
|
+
out << render(section, context, strict: strict, depth: depth + 1)
|
|
29
|
+
end
|
|
30
|
+
pos = new_pos
|
|
31
|
+
next
|
|
32
|
+
when "/"
|
|
33
|
+
raise TemplateError.new("unexpected closing tag #{m[:name]}")
|
|
34
|
+
else
|
|
35
|
+
val = lookup(context, m[:name])
|
|
36
|
+
if val.nil?
|
|
37
|
+
raise TemplateError.new("missing variable: #{m[:name]}") if strict
|
|
38
|
+
else
|
|
39
|
+
out << val.to_s
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
pos = m.end(0)
|
|
43
|
+
end
|
|
44
|
+
out << template[pos..]
|
|
45
|
+
out
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def self.parse_section(template, open_match, name)
|
|
49
|
+
open_re = /\{\{#\s*#{Regexp.escape(name)}\s*\}\}|\{\{\^\s*#{Regexp.escape(name)}\s*\}\}/
|
|
50
|
+
close_re = %r{\{\{/\s*#{Regexp.escape(name)}\s*\}\}}
|
|
51
|
+
both = Regexp.union(open_re, close_re)
|
|
52
|
+
depth = 1
|
|
53
|
+
cursor = open_match.end(0)
|
|
54
|
+
while (m = template.match(both, cursor))
|
|
55
|
+
if m[0].start_with?("{{/")
|
|
56
|
+
depth -= 1
|
|
57
|
+
return [template[open_match.end(0)...m.begin(0)], m.end(0)] if depth.zero?
|
|
58
|
+
else
|
|
59
|
+
depth += 1
|
|
60
|
+
end
|
|
61
|
+
cursor = m.end(0)
|
|
62
|
+
end
|
|
63
|
+
raise TemplateError.new("unclosed section: #{name}")
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.render_section(section, value, context, strict, depth)
|
|
67
|
+
raise TemplateError.new("template recursion depth #{depth + 1} exceeded #{MAX_DEPTH}") if depth + 1 > MAX_DEPTH
|
|
68
|
+
|
|
69
|
+
case value
|
|
70
|
+
when Array
|
|
71
|
+
value.map { |v| render(section, scope_for(context, v), strict: strict, depth: depth + 1) }.join
|
|
72
|
+
when Hash
|
|
73
|
+
render(section, merge(context, value), strict: strict, depth: depth + 1)
|
|
74
|
+
when true
|
|
75
|
+
render(section, context, strict: strict, depth: depth + 1)
|
|
76
|
+
when false, nil
|
|
77
|
+
# falsy in regular section: render nothing.
|
|
78
|
+
# render_section is only called for inverted sections when falsy? is true at the call site,
|
|
79
|
+
# so this branch is only hit for normal sections with falsy values.
|
|
80
|
+
""
|
|
81
|
+
else
|
|
82
|
+
render(section, context, strict: strict, depth: depth + 1)
|
|
83
|
+
end || ""
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def self.lookup(context, name)
|
|
87
|
+
# Implicit iterator: {{.}} refers to the current scope itself (used when
|
|
88
|
+
# iterating arrays of primitive values).
|
|
89
|
+
return context["."] if name == "." && context.is_a?(Hash) && context.key?(".")
|
|
90
|
+
return context[name] if context.is_a?(Hash) && context.key?(name)
|
|
91
|
+
|
|
92
|
+
name.split(".").reduce(context) do |acc, seg|
|
|
93
|
+
return nil unless acc.is_a?(Hash)
|
|
94
|
+
|
|
95
|
+
acc[seg]
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Build the rendering scope for one iteration of a section. Hash items
|
|
100
|
+
# merge into the outer context; primitive items (strings, numbers) bind
|
|
101
|
+
# to the implicit iterator under key ".".
|
|
102
|
+
def self.scope_for(context, item)
|
|
103
|
+
return merge(context, item) if item.is_a?(Hash)
|
|
104
|
+
|
|
105
|
+
base = context.is_a?(Hash) ? context : {}
|
|
106
|
+
base.merge("." => item)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def self.merge(base, override)
|
|
110
|
+
return base unless override.is_a?(Hash)
|
|
111
|
+
|
|
112
|
+
base.merge(override)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def self.falsy?(v) = v.nil? || v == false || v == [] || v == ""
|
|
116
|
+
end
|
|
117
|
+
end
|