textus 0.10.4 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +128 -3
- data/README.md +45 -86
- data/SPEC.md +266 -138
- data/docs/conventions.md +47 -15
- data/lib/textus/application/reads/freshness.rb +2 -2
- data/lib/textus/application/reads/get.rb +1 -1
- data/lib/textus/application/reads/policy_explain.rb +2 -2
- data/lib/textus/application/refresh/orchestrator.rb +1 -1
- data/lib/textus/application/refresh/worker.rb +5 -5
- data/lib/textus/application/writes/accept.rb +19 -1
- data/lib/textus/application/writes/build.rb +5 -5
- data/lib/textus/application/writes/delete.rb +2 -3
- data/lib/textus/application/writes/publish.rb +1 -1
- data/lib/textus/application/writes/put.rb +3 -6
- data/lib/textus/builder/pipeline.rb +1 -1
- data/lib/textus/builder/renderer/json.rb +1 -1
- data/lib/textus/builder/renderer/yaml.rb +1 -1
- data/lib/textus/cli/group/key.rb +1 -1
- data/lib/textus/cli/group/refresh.rb +21 -0
- data/lib/textus/cli/group/rule.rb +11 -0
- data/lib/textus/cli/verb/build.rb +1 -1
- data/lib/textus/cli/verb/hook_run.rb +3 -2
- data/lib/textus/cli/verb/hooks.rb +1 -1
- data/lib/textus/cli/verb/{migrate_keys.rb → key_normalize.rb} +1 -1
- data/lib/textus/cli/verb/put.rb +1 -1
- data/lib/textus/cli/verb/{policy_explain.rb → rule_explain.rb} +1 -1
- data/lib/textus/cli/verb/{policy_list.rb → rule_list.rb} +3 -3
- data/lib/textus/cli/verb.rb +3 -2
- data/lib/textus/cli.rb +6 -6
- data/lib/textus/doctor/check/handler_allowlist.rb +1 -1
- data/lib/textus/doctor/check/illegal_keys.rb +39 -16
- data/lib/textus/doctor/check/intake_registration.rb +4 -4
- data/lib/textus/doctor/check/protocol_version.rb +47 -0
- data/lib/textus/doctor/check/{policy_ambiguity.rb → rule_ambiguity.rb} +6 -6
- data/lib/textus/doctor.rb +5 -4
- data/lib/textus/domain/permission.rb +4 -4
- data/lib/textus/domain/policy/predicates/human_accept.rb +31 -0
- data/lib/textus/domain/policy/predicates/schema_valid.rb +50 -0
- data/lib/textus/domain/policy/promotion.rb +45 -0
- data/lib/textus/errors.rb +24 -5
- data/lib/textus/hooks/builtin.rb +5 -5
- data/lib/textus/hooks/dispatcher.rb +1 -1
- data/lib/textus/hooks/dsl.rb +3 -10
- data/lib/textus/hooks/loader.rb +1 -2
- data/lib/textus/hooks/registry.rb +22 -21
- data/lib/textus/infra/refresh/detached.rb +1 -1
- data/lib/textus/init.rb +25 -34
- data/lib/textus/intro.rb +9 -9
- data/lib/textus/manifest/entry.rb +66 -6
- data/lib/textus/manifest/{policies.rb → rules.rb} +12 -10
- data/lib/textus/manifest/schema.rb +49 -0
- data/lib/textus/manifest.rb +79 -39
- data/lib/textus/migrate_keys.rb +1 -1
- data/lib/textus/projection.rb +4 -4
- data/lib/textus/refresh.rb +1 -1
- data/lib/textus/store/mover.rb +91 -50
- data/lib/textus/store/staleness/generator_check.rb +88 -0
- data/lib/textus/store/staleness/intake_check.rb +46 -0
- data/lib/textus/store/staleness.rb +9 -104
- data/lib/textus/store/writer.rb +14 -12
- data/lib/textus/store.rb +1 -1
- data/lib/textus/version.rb +2 -2
- data/lib/textus.rb +1 -0
- metadata +15 -7
- data/lib/textus/cli/group/policy.rb +0 -11
|
@@ -4,10 +4,12 @@ module Textus
|
|
|
4
4
|
PUBLISH_EACH_VARS = %w[leaf basename key ext].freeze
|
|
5
5
|
PUBLISH_EACH_VAR_RE = /\{([a-z]+)\}/
|
|
6
6
|
|
|
7
|
+
COMPUTE_KINDS = %w[projection external].freeze
|
|
8
|
+
|
|
7
9
|
attr_reader :key, :path, :zone, :schema, :owner, :nested, :generator, :raw, :format,
|
|
8
10
|
:projection, :template, :publish_to, :publish_each,
|
|
9
11
|
:intake_handler, :intake_config,
|
|
10
|
-
:events, :inject_intro
|
|
12
|
+
:events, :inject_intro, :index_filename, :compute
|
|
11
13
|
|
|
12
14
|
def initialize(manifest, raw)
|
|
13
15
|
@manifest = manifest
|
|
@@ -18,13 +20,13 @@ module Textus
|
|
|
18
20
|
@schema = raw["schema"]
|
|
19
21
|
@owner = raw["owner"]
|
|
20
22
|
@nested = raw["nested"] == true
|
|
21
|
-
|
|
22
|
-
@projection = raw["projection"]
|
|
23
|
+
parse_compute!(raw)
|
|
23
24
|
@template = raw["template"]
|
|
24
25
|
@publish_to = Array(raw["publish_to"])
|
|
25
26
|
@publish_each = raw["publish_each"]
|
|
26
27
|
@events = raw["events"] || {}
|
|
27
28
|
@inject_intro = raw["inject_intro"] == true
|
|
29
|
+
@index_filename = raw["index_filename"]
|
|
28
30
|
@format = resolve_format!(raw["format"])
|
|
29
31
|
|
|
30
32
|
validate_events!
|
|
@@ -32,6 +34,7 @@ module Textus
|
|
|
32
34
|
validate_format_matrix!
|
|
33
35
|
validate_publish_each!
|
|
34
36
|
validate_inject_intro!
|
|
37
|
+
validate_index_filename!
|
|
35
38
|
end
|
|
36
39
|
|
|
37
40
|
# Resolves the per-leaf target path (relative to repo root) for a full
|
|
@@ -54,18 +57,49 @@ module Textus
|
|
|
54
57
|
end
|
|
55
58
|
|
|
56
59
|
# Signal-based zone-kind predicates: derive the "kind" of a zone from its
|
|
57
|
-
#
|
|
60
|
+
# write_policy signals rather than its literal name, so detection keeps
|
|
58
61
|
# working when users rename the default zones.
|
|
59
62
|
def in_generator_zone?
|
|
60
|
-
zone_writers.include?("
|
|
63
|
+
zone_writers.include?("builder")
|
|
61
64
|
end
|
|
62
65
|
|
|
63
66
|
def in_proposal_zone?
|
|
64
|
-
zone_writers.include?("
|
|
67
|
+
zone_writers.include?("agent")
|
|
65
68
|
end
|
|
66
69
|
|
|
67
70
|
private
|
|
68
71
|
|
|
72
|
+
# `index_filename:` makes a nested entry treat a fixed basename (e.g.
|
|
73
|
+
# `SKILL.md`) as the per-directory row. The directory path becomes the
|
|
74
|
+
# key suffix; sibling files are not enumerated. Allows projecting
|
|
75
|
+
# spec-mandated filenames that would otherwise be rejected by the
|
|
76
|
+
# lowercase-only key segment grammar.
|
|
77
|
+
def validate_index_filename!
|
|
78
|
+
return if @index_filename.nil?
|
|
79
|
+
|
|
80
|
+
raise UsageError.new("entry '#{@key}': index_filename requires nested: true") unless @nested
|
|
81
|
+
unless @index_filename.is_a?(String) && !@index_filename.empty?
|
|
82
|
+
raise UsageError.new("entry '#{@key}': index_filename must be a non-empty string")
|
|
83
|
+
end
|
|
84
|
+
if @index_filename.include?("/") || File.basename(@index_filename) != @index_filename
|
|
85
|
+
raise UsageError.new("entry '#{@key}': index_filename must be a bare basename (no slashes)")
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
ext = File.extname(@index_filename)
|
|
89
|
+
inferred = Manifest::EXT_TO_FORMAT[ext]
|
|
90
|
+
if inferred.nil?
|
|
91
|
+
raise UsageError.new(
|
|
92
|
+
"entry '#{@key}': index_filename #{@index_filename.inspect} has unknown extension #{ext.inspect}",
|
|
93
|
+
)
|
|
94
|
+
end
|
|
95
|
+
return if inferred == @format
|
|
96
|
+
|
|
97
|
+
raise UsageError.new(
|
|
98
|
+
"entry '#{@key}': index_filename extension #{ext.inspect} implies format #{inferred.inspect}, " \
|
|
99
|
+
"but entry format is #{@format.inspect}",
|
|
100
|
+
)
|
|
101
|
+
end
|
|
102
|
+
|
|
69
103
|
def zone_writers
|
|
70
104
|
@manifest.zone_writers(@zone)
|
|
71
105
|
rescue UsageError => e
|
|
@@ -178,6 +212,32 @@ module Textus
|
|
|
178
212
|
end
|
|
179
213
|
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
180
214
|
|
|
215
|
+
def parse_compute!(raw)
|
|
216
|
+
src = raw["compute"]
|
|
217
|
+
unless src
|
|
218
|
+
@compute = nil
|
|
219
|
+
@projection = nil
|
|
220
|
+
@generator = nil
|
|
221
|
+
return
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
kind = src["kind"]
|
|
225
|
+
unless COMPUTE_KINDS.include?(kind)
|
|
226
|
+
raise BadManifest.new(
|
|
227
|
+
"entry '#{@key}': compute.kind must be one of #{COMPUTE_KINDS.join(", ")} (got #{kind.inspect})",
|
|
228
|
+
)
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
@compute = src.freeze
|
|
232
|
+
if kind == "projection"
|
|
233
|
+
@projection = @compute
|
|
234
|
+
@generator = nil
|
|
235
|
+
else
|
|
236
|
+
@generator = @compute
|
|
237
|
+
@projection = nil
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
181
241
|
def parse_intake!(src)
|
|
182
242
|
src ||= {}
|
|
183
243
|
@intake_handler = src["handler"]
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
module Textus
|
|
2
2
|
class Manifest
|
|
3
|
-
class
|
|
4
|
-
|
|
5
|
-
EMPTY_SET =
|
|
3
|
+
class Rules
|
|
4
|
+
RuleSet = Data.define(:refresh, :handler_allowlist, :promote, :retention)
|
|
5
|
+
EMPTY_SET = RuleSet.new(refresh: nil, handler_allowlist: nil, promote: nil, retention: nil)
|
|
6
6
|
|
|
7
7
|
def self.parse(raw)
|
|
8
8
|
new(Array(raw).map { |b| Block.new(b) })
|
|
@@ -21,7 +21,7 @@ module Textus
|
|
|
21
21
|
|
|
22
22
|
slots.each_key { |slot| slots[slot] << b if b.public_send(slot) }
|
|
23
23
|
end
|
|
24
|
-
|
|
24
|
+
RuleSet.new(
|
|
25
25
|
refresh: pick(slots[:refresh], :refresh, key),
|
|
26
26
|
handler_allowlist: pick(slots[:handler_allowlist], :handler_allowlist, key),
|
|
27
27
|
promote: pick(slots[:promote], :promote, key),
|
|
@@ -47,10 +47,10 @@ module Textus
|
|
|
47
47
|
attr_reader :match, :refresh, :handler_allowlist, :promote, :retention
|
|
48
48
|
|
|
49
49
|
def initialize(raw)
|
|
50
|
-
@match = raw["match"] or raise Textus::UsageError.new("
|
|
50
|
+
@match = raw["match"] or raise Textus::UsageError.new("rule block missing match:")
|
|
51
51
|
@refresh = parse_refresh(raw["refresh"])
|
|
52
|
-
@handler_allowlist = parse_handler_allowlist(raw["
|
|
53
|
-
@promote =
|
|
52
|
+
@handler_allowlist = parse_handler_allowlist(raw["intake_handler_allowlist"])
|
|
53
|
+
@promote = parse_promotion(raw["promotion"])
|
|
54
54
|
@retention = raw["retention"] # reserved — passthrough only
|
|
55
55
|
end
|
|
56
56
|
|
|
@@ -72,10 +72,12 @@ module Textus
|
|
|
72
72
|
Textus::Domain::Policy::HandlerAllowlist.new(handlers: arr)
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
def
|
|
76
|
-
return nil if
|
|
75
|
+
def parse_promotion(h)
|
|
76
|
+
return nil if h.nil?
|
|
77
|
+
|
|
78
|
+
raise Textus::BadManifest.new("promotion: must be a hash with a 'requires:' array") unless h.is_a?(Hash) && h.key?("requires")
|
|
77
79
|
|
|
78
|
-
Textus::Domain::Policy::Promote.new(requires:
|
|
80
|
+
Textus::Domain::Policy::Promote.new(requires: Array(h["requires"]))
|
|
79
81
|
end
|
|
80
82
|
end
|
|
81
83
|
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
module Textus
|
|
2
|
+
class Manifest
|
|
3
|
+
module Schema
|
|
4
|
+
ROOT_KEYS = %w[version zones entries rules].freeze
|
|
5
|
+
ZONE_KEYS = %w[name write_policy read_policy].freeze
|
|
6
|
+
ENTRY_KEYS = %w[
|
|
7
|
+
key path zone schema owner nested format
|
|
8
|
+
compute template publish_to publish_each
|
|
9
|
+
intake events inject_intro index_filename
|
|
10
|
+
].freeze
|
|
11
|
+
COMPUTE_KEYS = %w[kind select pluck sort_by limit transform command sources].freeze
|
|
12
|
+
INTAKE_KEYS = %w[handler config].freeze
|
|
13
|
+
RULE_KEYS = %w[match refresh intake_handler_allowlist promotion retention].freeze
|
|
14
|
+
REFRESH_KEYS = %w[ttl on_stale sync_budget_ms].freeze
|
|
15
|
+
PROMOTION_KEYS = %w[requires].freeze
|
|
16
|
+
|
|
17
|
+
def self.validate!(raw)
|
|
18
|
+
raise BadManifest.new("manifest must be a hash") unless raw.is_a?(Hash)
|
|
19
|
+
|
|
20
|
+
walk(raw, ROOT_KEYS, "$")
|
|
21
|
+
Array(raw["zones"]).each_with_index do |z, i|
|
|
22
|
+
walk(z, ZONE_KEYS, "$.zones[#{i}]")
|
|
23
|
+
end
|
|
24
|
+
Array(raw["entries"]).each_with_index do |e, i|
|
|
25
|
+
path = "$.entries[#{i}]"
|
|
26
|
+
walk(e, ENTRY_KEYS, path)
|
|
27
|
+
walk(e["compute"], COMPUTE_KEYS, "#{path}.compute") if e["compute"].is_a?(Hash)
|
|
28
|
+
walk(e["intake"], INTAKE_KEYS, "#{path}.intake") if e["intake"].is_a?(Hash)
|
|
29
|
+
end
|
|
30
|
+
Array(raw["rules"]).each_with_index do |r, i|
|
|
31
|
+
path = "$.rules[#{i}]"
|
|
32
|
+
walk(r, RULE_KEYS, path)
|
|
33
|
+
walk(r["refresh"], REFRESH_KEYS, "#{path}.refresh") if r["refresh"].is_a?(Hash)
|
|
34
|
+
walk(r["promotion"], PROMOTION_KEYS, "#{path}.promotion") if r["promotion"].is_a?(Hash)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def self.walk(hash, allowed, path)
|
|
39
|
+
return unless hash.is_a?(Hash)
|
|
40
|
+
|
|
41
|
+
hash.each_key do |k|
|
|
42
|
+
next if allowed.include?(k)
|
|
43
|
+
|
|
44
|
+
raise BadManifest.new("unknown key '#{k}' at '#{path}'")
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
data/lib/textus/manifest.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "yaml"
|
|
2
|
+
require_relative "manifest/schema"
|
|
2
3
|
|
|
3
4
|
module Textus
|
|
4
5
|
class Manifest
|
|
@@ -10,10 +11,26 @@ module Textus
|
|
|
10
11
|
".txt" => "text",
|
|
11
12
|
}.freeze
|
|
12
13
|
|
|
14
|
+
TEXTUS_2_HINT = "Install textus 0.11.x to run the migrator, then upgrade to this version. " \
|
|
15
|
+
"See https://github.com/patrick204nqh/textus/blob/main/CHANGELOG.md#0110".freeze
|
|
16
|
+
|
|
17
|
+
def self.version_hint_for(version)
|
|
18
|
+
version == "textus/2" ? TEXTUS_2_HINT : nil
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private_class_method :version_hint_for
|
|
22
|
+
|
|
13
23
|
attr_reader :root, :entries, :raw
|
|
14
24
|
|
|
15
25
|
def zones
|
|
16
|
-
@zones ||= Array(@raw["zones"]).to_h { |z| [z["name"], Array(z["
|
|
26
|
+
@zones ||= Array(@raw["zones"]).to_h { |z| [z["name"], Array(z["write_policy"])] }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def zone_readers
|
|
30
|
+
@zone_readers ||= Array(@raw["zones"]).to_h do |z|
|
|
31
|
+
rp = z["read_policy"]
|
|
32
|
+
[z["name"], rp.nil? ? :all : Array(rp)]
|
|
33
|
+
end
|
|
17
34
|
end
|
|
18
35
|
|
|
19
36
|
def zone_writers(zone_name)
|
|
@@ -23,18 +40,35 @@ module Textus
|
|
|
23
40
|
def permission_for(zone_name)
|
|
24
41
|
Textus::Domain::Permission.new(
|
|
25
42
|
zone: zone_name,
|
|
26
|
-
|
|
27
|
-
|
|
43
|
+
write_policy: zone_writers(zone_name),
|
|
44
|
+
read_policy: zone_readers[zone_name] || :all,
|
|
28
45
|
)
|
|
29
46
|
end
|
|
30
47
|
|
|
48
|
+
def self.parse(yaml_text, root: ".")
|
|
49
|
+
raw = YAML.safe_load(yaml_text, aliases: false)
|
|
50
|
+
unless raw["version"] == PROTOCOL
|
|
51
|
+
raise BadFrontmatter.new(
|
|
52
|
+
"<string>",
|
|
53
|
+
"unsupported manifest version #{raw["version"].inspect}; expected #{PROTOCOL.inspect}",
|
|
54
|
+
hint: version_hint_for(raw["version"]),
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
new(root, raw)
|
|
59
|
+
end
|
|
60
|
+
|
|
31
61
|
def self.load(root)
|
|
32
62
|
manifest_path = File.join(root, "manifest.yaml")
|
|
33
63
|
raise IoError.new("manifest not found: #{manifest_path}") unless File.exist?(manifest_path)
|
|
34
64
|
|
|
35
65
|
raw = YAML.safe_load_file(manifest_path, aliases: false)
|
|
36
66
|
unless raw["version"] == PROTOCOL
|
|
37
|
-
raise BadFrontmatter.new(
|
|
67
|
+
raise BadFrontmatter.new(
|
|
68
|
+
manifest_path,
|
|
69
|
+
"unsupported manifest version #{raw["version"].inspect}; expected #{PROTOCOL.inspect}",
|
|
70
|
+
hint: version_hint_for(raw["version"]),
|
|
71
|
+
)
|
|
38
72
|
end
|
|
39
73
|
|
|
40
74
|
new(root, raw)
|
|
@@ -45,16 +79,18 @@ module Textus
|
|
|
45
79
|
@raw = raw
|
|
46
80
|
raise BadFrontmatter.new(File.join(root, "manifest.yaml"), "manifest must declare zones:") if Array(raw["zones"]).empty?
|
|
47
81
|
|
|
82
|
+
Schema.validate!(raw)
|
|
83
|
+
|
|
48
84
|
@entries = Array(raw["entries"]).map { |e| Manifest::Entry.new(self, e) }
|
|
49
85
|
validate_declared_keys!
|
|
50
86
|
end
|
|
51
87
|
|
|
52
|
-
def
|
|
53
|
-
@
|
|
88
|
+
def rules
|
|
89
|
+
@rules ||= Textus::Manifest::Rules.parse(@raw["rules"] || [])
|
|
54
90
|
end
|
|
55
91
|
|
|
56
|
-
def
|
|
57
|
-
|
|
92
|
+
def rules_for(key)
|
|
93
|
+
rules.for(key)
|
|
58
94
|
end
|
|
59
95
|
|
|
60
96
|
# Returns [Manifest::Entry, resolved_path, remaining_segments]
|
|
@@ -76,8 +112,12 @@ module Textus
|
|
|
76
112
|
else
|
|
77
113
|
raise UnknownKey.new(key, suggestions: suggestions_for(key)) unless entry.nested
|
|
78
114
|
|
|
79
|
-
|
|
80
|
-
|
|
115
|
+
path = if entry.index_filename
|
|
116
|
+
File.join(@root, "zones", entry.path, *remaining, entry.index_filename)
|
|
117
|
+
else
|
|
118
|
+
primary_ext = Textus::Entry.for_format(entry.format).extensions.first
|
|
119
|
+
File.join(@root, "zones", entry.path, *remaining) + primary_ext
|
|
120
|
+
end
|
|
81
121
|
[entry, path, remaining]
|
|
82
122
|
end
|
|
83
123
|
end
|
|
@@ -96,39 +136,11 @@ module Textus
|
|
|
96
136
|
|
|
97
137
|
# Enumerate all entry files reachable through the manifest. Returns
|
|
98
138
|
# [{ key:, path:, manifest_entry: }, ...]
|
|
99
|
-
# rubocop:disable Metrics/AbcSize
|
|
100
139
|
def enumerate(prefix: nil)
|
|
101
|
-
out =
|
|
102
|
-
@entries.each do |entry|
|
|
103
|
-
if entry.nested
|
|
104
|
-
base = File.join(@root, "zones", entry.path)
|
|
105
|
-
next unless File.directory?(base)
|
|
106
|
-
|
|
107
|
-
glob_pattern = nested_glob(entry.format)
|
|
108
|
-
Dir.glob(File.join(base, glob_pattern)).each do |fp|
|
|
109
|
-
rel = fp.sub(%r{\A#{Regexp.escape(base)}/?}, "")
|
|
110
|
-
stripped = rel.sub(/#{Regexp.escape(File.extname(rel))}\z/, "")
|
|
111
|
-
segs = stripped.split("/").reject(&:empty?)
|
|
112
|
-
next if segs.empty?
|
|
113
|
-
|
|
114
|
-
illegal = segs.find { |s| !valid_segment?(s) }
|
|
115
|
-
if illegal
|
|
116
|
-
warn("textus: skipping illegal key segment '#{illegal}' at #{fp} — run 'textus key migrate --dry-run'")
|
|
117
|
-
next
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
full_key = (entry.key.split(".") + segs).join(".")
|
|
121
|
-
out << { key: full_key, path: fp, manifest_entry: entry }
|
|
122
|
-
end
|
|
123
|
-
else
|
|
124
|
-
fp = resolve_leaf_path(entry)
|
|
125
|
-
out << { key: entry.key, path: fp, manifest_entry: entry } if File.exist?(fp)
|
|
126
|
-
end
|
|
127
|
-
end
|
|
140
|
+
out = @entries.flat_map { |entry| entry.nested ? enumerate_nested(entry) : enumerate_leaf(entry) }
|
|
128
141
|
out.select! { |row| row[:key] == prefix || row[:key].start_with?("#{prefix}.") } if prefix
|
|
129
142
|
out.sort_by { |row| row[:key] }
|
|
130
143
|
end
|
|
131
|
-
# rubocop:enable Metrics/AbcSize
|
|
132
144
|
|
|
133
145
|
def validate_key!(key)
|
|
134
146
|
raise UsageError.new("empty key") if key.nil? || key.empty?
|
|
@@ -138,6 +150,34 @@ module Textus
|
|
|
138
150
|
|
|
139
151
|
private
|
|
140
152
|
|
|
153
|
+
def enumerate_leaf(entry)
|
|
154
|
+
fp = resolve_leaf_path(entry)
|
|
155
|
+
File.exist?(fp) ? [{ key: entry.key, path: fp, manifest_entry: entry }] : []
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def enumerate_nested(entry)
|
|
159
|
+
base = File.join(@root, "zones", entry.path)
|
|
160
|
+
return [] unless File.directory?(base)
|
|
161
|
+
|
|
162
|
+
glob_pattern = entry.index_filename ? "**/#{entry.index_filename}" : nested_glob(entry.format)
|
|
163
|
+
Dir.glob(File.join(base, glob_pattern)).filter_map { |path| nested_row_for(entry, base, path) }
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def nested_row_for(entry, base, path)
|
|
167
|
+
rel = path.sub(%r{\A#{Regexp.escape(base)}/?}, "")
|
|
168
|
+
stripped = entry.index_filename ? File.dirname(rel) : rel.sub(/#{Regexp.escape(File.extname(rel))}\z/, "")
|
|
169
|
+
segs = stripped.split("/").reject { |s| s.empty? || s == "." }
|
|
170
|
+
return nil if segs.empty?
|
|
171
|
+
|
|
172
|
+
illegal = segs.find { |s| !valid_segment?(s) }
|
|
173
|
+
if illegal
|
|
174
|
+
warn("textus: skipping illegal key segment '#{illegal}' at #{path} — run 'textus key normalize --dry-run'")
|
|
175
|
+
return nil
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
{ key: (entry.key.split(".") + segs).join("."), path: path, manifest_entry: entry }
|
|
179
|
+
end
|
|
180
|
+
|
|
141
181
|
def valid_segment?(seg)
|
|
142
182
|
return false if seg.nil? || seg.empty?
|
|
143
183
|
return false if seg.length > Key::Grammar::MAX_SEGMENT_LEN
|
data/lib/textus/migrate_keys.rb
CHANGED
data/lib/textus/projection.rb
CHANGED
|
@@ -38,14 +38,14 @@ module Textus
|
|
|
38
38
|
private
|
|
39
39
|
|
|
40
40
|
def apply_reducer(rows)
|
|
41
|
-
name = @spec["
|
|
42
|
-
callable = @store.registry.rpc_callable(:
|
|
41
|
+
name = @spec["transform"] or return rows
|
|
42
|
+
callable = @store.registry.rpc_callable(:transform_rows, name)
|
|
43
43
|
view = Application::Context.new(store: @store, role: "human")
|
|
44
44
|
Timeout.timeout(REDUCER_TIMEOUT_SECONDS) do
|
|
45
|
-
callable.call(store: view, rows: rows, config: @spec["
|
|
45
|
+
callable.call(store: view, rows: rows, config: @spec["transform_config"] || {})
|
|
46
46
|
end
|
|
47
47
|
rescue Timeout::Error
|
|
48
|
-
raise UsageError.new("
|
|
48
|
+
raise UsageError.new("transform_rows '#{name}' exceeded #{REDUCER_TIMEOUT_SECONDS}s timeout")
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
def collect_keys
|
data/lib/textus/refresh.rb
CHANGED
|
@@ -5,7 +5,7 @@ module Textus
|
|
|
5
5
|
Textus::Composition.refresh_worker(ctx).run(key)
|
|
6
6
|
end
|
|
7
7
|
|
|
8
|
-
def self.refresh_stale(store, prefix: nil, zone: nil, as: "
|
|
8
|
+
def self.refresh_stale(store, prefix: nil, zone: nil, as: "runner")
|
|
9
9
|
ctx = Textus::Composition.context(store, role: as)
|
|
10
10
|
Textus::Application::Refresh::All.call(ctx, prefix: prefix, zone: zone)
|
|
11
11
|
end
|
data/lib/textus/store/mover.rb
CHANGED
|
@@ -2,8 +2,12 @@ require "fileutils"
|
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
4
|
class Store
|
|
5
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
6
5
|
class Mover
|
|
6
|
+
MovePlan = Data.define(
|
|
7
|
+
:old_key, :new_key, :old_path, :new_path,
|
|
8
|
+
:new_mentry, :uid, :etag_before, :as
|
|
9
|
+
)
|
|
10
|
+
|
|
7
11
|
def initialize(store:, reader:, writer:, manifest:, audit_log:)
|
|
8
12
|
@store = store
|
|
9
13
|
@reader = reader
|
|
@@ -13,6 +17,22 @@ module Textus
|
|
|
13
17
|
end
|
|
14
18
|
|
|
15
19
|
def call(old_key, new_key, as: Role::DEFAULT, dry_run: false, correlation_id: nil)
|
|
20
|
+
plan, pre_env = prepare_plan(old_key, new_key, as: as)
|
|
21
|
+
return dry_run_result(plan) if dry_run
|
|
22
|
+
|
|
23
|
+
plan = ensure_uid!(plan, pre_env: pre_env)
|
|
24
|
+
etag_after = perform_move!(plan)
|
|
25
|
+
new_envelope = record_move(plan, etag_after: etag_after, correlation_id: correlation_id)
|
|
26
|
+
success_result(plan, new_envelope: new_envelope)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# Validates inputs, resolves manifest entries, and reads the source
|
|
32
|
+
# envelope. Returns [MovePlan, pre_envelope]; the pre_envelope is only
|
|
33
|
+
# needed by ensure_uid! and is threaded separately to keep MovePlan
|
|
34
|
+
# focused on the planned operation.
|
|
35
|
+
def prepare_plan(old_key, new_key, as:)
|
|
16
36
|
@manifest.validate_key!(old_key)
|
|
17
37
|
@manifest.validate_key!(new_key)
|
|
18
38
|
raise UsageError.new("mv: old and new keys are identical") if old_key == new_key
|
|
@@ -21,81 +41,103 @@ module Textus
|
|
|
21
41
|
raise UnknownKey.new(old_key) unless File.exist?(old_path)
|
|
22
42
|
|
|
23
43
|
new_mentry, new_path, = @manifest.resolve(new_key)
|
|
44
|
+
validate_zone_and_format!(old_mentry, new_mentry)
|
|
45
|
+
validate_writer!(old_mentry, old_key, as)
|
|
46
|
+
raise UsageError.new("mv: target '#{new_key}' already exists at #{new_path}") if File.exist?(new_path)
|
|
24
47
|
|
|
48
|
+
pre_env = @reader.get(old_key)
|
|
49
|
+
plan = MovePlan.new(
|
|
50
|
+
old_key: old_key, new_key: new_key,
|
|
51
|
+
old_path: old_path, new_path: new_path,
|
|
52
|
+
new_mentry: new_mentry,
|
|
53
|
+
uid: pre_env["uid"], etag_before: pre_env["etag"], as: as
|
|
54
|
+
)
|
|
55
|
+
[plan, pre_env]
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def validate_zone_and_format!(old_mentry, new_mentry)
|
|
25
59
|
if old_mentry.zone != new_mentry.zone
|
|
26
60
|
raise UsageError.new(
|
|
27
61
|
"mv: cross-zone move refused (#{old_mentry.zone} → #{new_mentry.zone}). " \
|
|
28
62
|
"Use put+delete for cross-zone moves.",
|
|
29
63
|
)
|
|
30
64
|
end
|
|
31
|
-
if old_mentry.format
|
|
32
|
-
raise UsageError.new(
|
|
33
|
-
"mv: format mismatch (#{old_mentry.format} → #{new_mentry.format}); refusing.",
|
|
34
|
-
)
|
|
35
|
-
end
|
|
65
|
+
return if old_mentry.format == new_mentry.format
|
|
36
66
|
|
|
37
|
-
|
|
38
|
-
|
|
67
|
+
raise UsageError.new(
|
|
68
|
+
"mv: format mismatch (#{old_mentry.format} → #{new_mentry.format}); refusing.",
|
|
69
|
+
)
|
|
70
|
+
end
|
|
39
71
|
|
|
40
|
-
|
|
72
|
+
def validate_writer!(mentry, key, as)
|
|
73
|
+
writers = @manifest.zone_writers(mentry.zone)
|
|
74
|
+
return if writers.include?(as)
|
|
41
75
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
current_uid = pre_env["uid"]
|
|
45
|
-
etag_before = pre_env["etag"]
|
|
46
|
-
|
|
47
|
-
if dry_run
|
|
48
|
-
return {
|
|
49
|
-
"protocol" => PROTOCOL, "ok" => true, "dry_run" => true,
|
|
50
|
-
"from_key" => old_key, "to_key" => new_key,
|
|
51
|
-
"from_path" => old_path, "to_path" => new_path,
|
|
52
|
-
"uid" => current_uid
|
|
53
|
-
}
|
|
54
|
-
end
|
|
76
|
+
raise WriteForbidden.new(key, mentry.zone, writers: writers)
|
|
77
|
+
end
|
|
55
78
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
pre_env = @writer.put(old_key,
|
|
59
|
-
meta: pre_env["_meta"],
|
|
60
|
-
body: pre_env["body"],
|
|
61
|
-
content: pre_env["content"],
|
|
62
|
-
as: as,
|
|
63
|
-
suppress_events: true)
|
|
64
|
-
current_uid = pre_env["uid"]
|
|
65
|
-
etag_before = pre_env["etag"]
|
|
66
|
-
end
|
|
79
|
+
def ensure_uid!(plan, pre_env:)
|
|
80
|
+
return plan if plan.uid
|
|
67
81
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
82
|
+
env = @writer.put(
|
|
83
|
+
plan.old_key,
|
|
84
|
+
meta: pre_env["_meta"],
|
|
85
|
+
body: pre_env["body"],
|
|
86
|
+
content: pre_env["content"],
|
|
87
|
+
as: plan.as,
|
|
88
|
+
suppress_events: true,
|
|
89
|
+
)
|
|
90
|
+
plan.with(uid: env["uid"], etag_before: env["etag"])
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def perform_move!(plan)
|
|
94
|
+
FileUtils.mkdir_p(File.dirname(plan.new_path))
|
|
95
|
+
FileUtils.mv(plan.old_path, plan.new_path)
|
|
96
|
+
rewrite_name_for_mv!(plan.new_mentry, plan.new_path, plan.new_key)
|
|
97
|
+
Etag.for_file(plan.new_path)
|
|
98
|
+
end
|
|
72
99
|
|
|
100
|
+
def record_move(plan, etag_after:, correlation_id:)
|
|
73
101
|
extras = {
|
|
74
|
-
"from_key" => old_key, "to_key" => new_key,
|
|
75
|
-
"from_path" => old_path, "to_path" => new_path,
|
|
76
|
-
"uid" =>
|
|
102
|
+
"from_key" => plan.old_key, "to_key" => plan.new_key,
|
|
103
|
+
"from_path" => plan.old_path, "to_path" => plan.new_path,
|
|
104
|
+
"uid" => plan.uid
|
|
77
105
|
}
|
|
78
106
|
extras["correlation_id"] = correlation_id if correlation_id
|
|
79
107
|
|
|
80
108
|
@audit_log.append(
|
|
81
|
-
role: as, verb: "mv", key: new_key,
|
|
82
|
-
etag_before: etag_before, etag_after: etag_after,
|
|
109
|
+
role: plan.as, verb: "mv", key: plan.new_key,
|
|
110
|
+
etag_before: plan.etag_before, etag_after: etag_after,
|
|
83
111
|
extras: extras
|
|
84
112
|
)
|
|
113
|
+
new_envelope = @reader.get(plan.new_key)
|
|
114
|
+
@store.fire_event(
|
|
115
|
+
:entry_renamed,
|
|
116
|
+
key: plan.new_key, from_key: plan.old_key, to_key: plan.new_key,
|
|
117
|
+
envelope: new_envelope
|
|
118
|
+
)
|
|
119
|
+
new_envelope
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def dry_run_result(plan)
|
|
123
|
+
{
|
|
124
|
+
"protocol" => PROTOCOL, "ok" => true, "dry_run" => true,
|
|
125
|
+
"from_key" => plan.old_key, "to_key" => plan.new_key,
|
|
126
|
+
"from_path" => plan.old_path, "to_path" => plan.new_path,
|
|
127
|
+
"uid" => plan.uid
|
|
128
|
+
}
|
|
129
|
+
end
|
|
85
130
|
|
|
86
|
-
|
|
87
|
-
@store.fire_event(:mv, key: new_key, from_key: old_key, to_key: new_key, envelope: new_envelope)
|
|
131
|
+
def success_result(plan, new_envelope:)
|
|
88
132
|
{
|
|
89
133
|
"protocol" => PROTOCOL, "ok" => true,
|
|
90
|
-
"from_key" => old_key, "to_key" => new_key,
|
|
91
|
-
"from_path" => old_path, "to_path" => new_path,
|
|
92
|
-
"uid" =>
|
|
134
|
+
"from_key" => plan.old_key, "to_key" => plan.new_key,
|
|
135
|
+
"from_path" => plan.old_path, "to_path" => plan.new_path,
|
|
136
|
+
"uid" => plan.uid,
|
|
93
137
|
"envelope" => new_envelope
|
|
94
138
|
}
|
|
95
139
|
end
|
|
96
140
|
|
|
97
|
-
private
|
|
98
|
-
|
|
99
141
|
# If the moved file carries a `name:` field (markdown) or `_meta.name`
|
|
100
142
|
# (json/yaml), rewrite it to the new basename so enforce_name_match! stays
|
|
101
143
|
# happy on the next read. Only touches the bytes when name actually changes.
|
|
@@ -121,6 +163,5 @@ module Textus
|
|
|
121
163
|
end
|
|
122
164
|
end
|
|
123
165
|
end
|
|
124
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
125
166
|
end
|
|
126
167
|
end
|