textus 0.52.0 → 0.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/README.md +62 -54
- data/SPEC.md +62 -187
- data/docs/architecture/README.md +88 -77
- data/exe/textus +1 -1
- data/lib/textus/action/accept.rb +53 -0
- data/lib/textus/action/audit.rb +133 -0
- data/lib/textus/action/base.rb +42 -0
- data/lib/textus/{read → action}/blame.rb +30 -22
- data/lib/textus/action/boot.rb +26 -0
- data/lib/textus/action/data_mv.rb +71 -0
- data/lib/textus/action/deps.rb +48 -0
- data/lib/textus/action/doctor.rb +26 -0
- data/lib/textus/action/drain.rb +41 -0
- data/lib/textus/action/enqueue.rb +55 -0
- data/lib/textus/action/get.rb +80 -0
- data/lib/textus/action/jobs.rb +38 -0
- data/lib/textus/action/key_delete.rb +46 -0
- data/lib/textus/action/key_delete_prefix.rb +46 -0
- data/lib/textus/action/key_mv.rb +143 -0
- data/lib/textus/action/key_mv_prefix.rb +59 -0
- data/lib/textus/action/list.rb +44 -0
- data/lib/textus/action/propose.rb +54 -0
- data/lib/textus/action/published.rb +26 -0
- data/lib/textus/action/pulse/scanner.rb +118 -0
- data/lib/textus/action/pulse.rb +87 -0
- data/lib/textus/action/put.rb +63 -0
- data/lib/textus/action/rdeps.rb +49 -0
- data/lib/textus/action/reject.rb +49 -0
- data/lib/textus/action/rule_explain.rb +95 -0
- data/lib/textus/action/rule_lint.rb +70 -0
- data/lib/textus/action/rule_list.rb +46 -0
- data/lib/textus/action/schema_envelope.rb +31 -0
- data/lib/textus/action/uid.rb +35 -0
- data/lib/textus/action/where.rb +38 -0
- data/lib/textus/action/write_verb.rb +58 -0
- data/lib/textus/background/job/base.rb +27 -0
- data/lib/textus/background/job/materialize.rb +31 -0
- data/lib/textus/background/job/refresh.rb +22 -0
- data/lib/textus/background/job/sweep.rb +31 -0
- data/lib/textus/background/job.rb +19 -0
- data/lib/textus/background/plan.rb +9 -0
- data/lib/textus/background/planner/plan.rb +113 -0
- data/lib/textus/{maintenance → background}/retention/apply.rb +7 -9
- data/lib/textus/background/worker.rb +67 -0
- data/lib/textus/boot.rb +53 -45
- data/lib/textus/command.rb +36 -0
- data/lib/textus/container.rb +1 -1
- data/lib/textus/{domain → core}/duration.rb +1 -1
- data/lib/textus/{domain → core}/freshness/evaluator.rb +11 -11
- data/lib/textus/{domain → core}/freshness/verdict.rb +2 -2
- data/lib/textus/{domain → core}/freshness.rb +2 -2
- data/lib/textus/{domain → core}/retention/sweep.rb +7 -7
- data/lib/textus/{domain → core}/retention.rb +2 -2
- data/lib/textus/{domain → core}/sentinel.rb +1 -1
- data/lib/textus/doctor/check/generator_drift.rb +1 -1
- data/lib/textus/doctor/check/handler_permit.rb +34 -0
- data/lib/textus/doctor/check/hooks.rb +11 -18
- data/lib/textus/doctor/check/illegal_keys.rb +1 -1
- data/lib/textus/doctor/check/intake_registration.rb +5 -5
- data/lib/textus/doctor/check/proposal_targets.rb +3 -3
- data/lib/textus/doctor/check/rule_ambiguity.rb +2 -2
- data/lib/textus/doctor/check/schema_violations.rb +8 -2
- data/lib/textus/doctor/check.rb +12 -9
- data/lib/textus/{read → doctor}/validator.rb +22 -13
- data/lib/textus/doctor.rb +6 -6
- data/lib/textus/envelope/io/writer.rb +65 -36
- data/lib/textus/envelope.rb +5 -3
- data/lib/textus/errors.rb +17 -9
- data/lib/textus/events.rb +21 -0
- data/lib/textus/gate/auth.rb +181 -0
- data/lib/textus/gate.rb +114 -0
- data/lib/textus/init/templates/machine_intake.rb +39 -35
- data/lib/textus/init/templates/orientation_reducer.rb +15 -11
- data/lib/textus/init.rb +90 -73
- data/lib/textus/key/path.rb +9 -2
- data/lib/textus/layout.rb +13 -0
- data/lib/textus/manifest/data.rb +14 -14
- data/lib/textus/manifest/entry/base.rb +15 -11
- data/lib/textus/manifest/entry/parser.rb +6 -6
- data/lib/textus/manifest/entry/produced.rb +3 -2
- data/lib/textus/manifest/entry/publish/mode.rb +1 -1
- data/lib/textus/manifest/entry/publish/to_paths.rb +2 -1
- data/lib/textus/manifest/entry/validators/events.rb +1 -1
- data/lib/textus/{domain/policy/handler_allowlist.rb → manifest/policy/handler_permit.rb} +4 -4
- data/lib/textus/{domain → manifest}/policy/matcher.rb +2 -2
- data/lib/textus/{domain → manifest}/policy/publish_target.rb +2 -2
- data/lib/textus/manifest/policy/react.rb +30 -0
- data/lib/textus/{domain → manifest}/policy/retention.rb +3 -3
- data/lib/textus/{domain → manifest}/policy/source.rb +24 -19
- data/lib/textus/manifest/policy.rb +36 -48
- data/lib/textus/manifest/resolver.rb +3 -2
- data/lib/textus/manifest/rules.rb +4 -4
- data/lib/textus/manifest/schema/keys.rb +17 -11
- data/lib/textus/manifest/schema/validator.rb +24 -22
- data/lib/textus/manifest/schema/vocabulary.rb +1 -1
- data/lib/textus/manifest/schema.rb +2 -2
- data/lib/textus/manifest.rb +2 -2
- data/lib/textus/{produce → pipeline}/acquire/handler.rb +2 -2
- data/lib/textus/{produce → pipeline}/acquire/intake.rb +22 -20
- data/lib/textus/{produce → pipeline}/acquire/projection.rb +13 -11
- data/lib/textus/{produce → pipeline}/acquire/serializer/json.rb +2 -2
- data/lib/textus/{produce → pipeline}/acquire/serializer/text.rb +1 -1
- data/lib/textus/{produce → pipeline}/acquire/serializer/yaml.rb +2 -2
- data/lib/textus/{produce → pipeline}/acquire/serializer.rb +1 -1
- data/lib/textus/{produce → pipeline}/engine.rb +7 -5
- data/lib/textus/{produce → pipeline}/render.rb +3 -1
- data/lib/textus/ports/audit_log.rb +31 -5
- data/lib/textus/ports/audit_subscriber.rb +4 -4
- data/lib/textus/{domain/jobs → ports/queue}/job.rb +19 -12
- data/lib/textus/ports/queue.rb +1 -1
- data/lib/textus/ports/sentinel_store.rb +2 -2
- data/lib/textus/ports/watcher_lock.rb +48 -0
- data/lib/textus/projection.rb +8 -8
- data/lib/textus/schema/tools.rb +4 -3
- data/lib/textus/session.rb +6 -3
- data/lib/textus/step/base.rb +35 -0
- data/lib/textus/step/builtin/csv_fetch.rb +19 -0
- data/lib/textus/step/builtin/ical_events_fetch.rb +30 -0
- data/lib/textus/step/builtin/json_fetch.rb +18 -0
- data/lib/textus/step/builtin/markdown_links_fetch.rb +20 -0
- data/lib/textus/step/builtin/rss_fetch.rb +26 -0
- data/lib/textus/step/builtin.rb +22 -0
- data/lib/textus/{hooks → step}/catalog.rb +3 -3
- data/lib/textus/{hooks → step}/context.rb +15 -13
- data/lib/textus/step/discovery.rb +24 -0
- data/lib/textus/{hooks → step}/error_log.rb +1 -1
- data/lib/textus/{hooks → step}/event_bus.rb +15 -16
- data/lib/textus/step/fetch.rb +13 -0
- data/lib/textus/{hooks → step}/fire_report.rb +1 -1
- data/lib/textus/step/loader.rb +108 -0
- data/lib/textus/step/observe.rb +31 -0
- data/lib/textus/step/registry_store.rb +66 -0
- data/lib/textus/{hooks → step}/signature.rb +1 -1
- data/lib/textus/step/transform.rb +12 -0
- data/lib/textus/step/validate.rb +11 -0
- data/lib/textus/step.rb +10 -0
- data/lib/textus/store.rb +17 -15
- data/lib/textus/surfaces/cli/group/data.rb +11 -0
- data/lib/textus/surfaces/cli/group/key.rb +11 -0
- data/lib/textus/surfaces/cli/group/mcp.rb +11 -0
- data/lib/textus/surfaces/cli/group/rule.rb +11 -0
- data/lib/textus/surfaces/cli/group/schema.rb +11 -0
- data/lib/textus/surfaces/cli/group.rb +50 -0
- data/lib/textus/surfaces/cli/runner.rb +236 -0
- data/lib/textus/surfaces/cli/verb/doctor.rb +21 -0
- data/lib/textus/surfaces/cli/verb/get.rb +21 -0
- data/lib/textus/surfaces/cli/verb/init.rb +20 -0
- data/lib/textus/surfaces/cli/verb/mcp_serve.rb +24 -0
- data/lib/textus/surfaces/cli/verb/put.rb +30 -0
- data/lib/textus/surfaces/cli/verb/schema_diff.rb +17 -0
- data/lib/textus/surfaces/cli/verb/schema_init.rb +21 -0
- data/lib/textus/surfaces/cli/verb/schema_migrate.rb +21 -0
- data/lib/textus/surfaces/cli/verb/watch.rb +19 -0
- data/lib/textus/surfaces/cli/verb.rb +111 -0
- data/lib/textus/surfaces/cli.rb +148 -0
- data/lib/textus/surfaces/mcp/catalog.rb +99 -0
- data/lib/textus/surfaces/mcp/errors.rb +34 -0
- data/lib/textus/surfaces/mcp/server.rb +145 -0
- data/lib/textus/surfaces/mcp/session.rb +9 -0
- data/lib/textus/surfaces/mcp/tool_schemas.rb +17 -0
- data/lib/textus/surfaces/mcp.rb +8 -0
- data/lib/textus/surfaces/role_scope.rb +38 -0
- data/lib/textus/surfaces/watcher.rb +38 -0
- data/lib/textus/version.rb +1 -1
- data/lib/textus.rb +64 -22
- metadata +132 -118
- data/lib/textus/cli/group/hook.rb +0 -9
- data/lib/textus/cli/group/key.rb +0 -9
- data/lib/textus/cli/group/mcp.rb +0 -9
- data/lib/textus/cli/group/rule.rb +0 -9
- data/lib/textus/cli/group/schema.rb +0 -9
- data/lib/textus/cli/group/zone.rb +0 -9
- data/lib/textus/cli/group.rb +0 -48
- data/lib/textus/cli/runner.rb +0 -193
- data/lib/textus/cli/verb/doctor.rb +0 -17
- data/lib/textus/cli/verb/get.rb +0 -18
- data/lib/textus/cli/verb/hook_run.rb +0 -48
- data/lib/textus/cli/verb/hooks.rb +0 -50
- data/lib/textus/cli/verb/init.rb +0 -18
- data/lib/textus/cli/verb/mcp_serve.rb +0 -22
- data/lib/textus/cli/verb/put.rb +0 -30
- data/lib/textus/cli/verb/schema_diff.rb +0 -15
- data/lib/textus/cli/verb/schema_init.rb +0 -19
- data/lib/textus/cli/verb/schema_migrate.rb +0 -19
- data/lib/textus/cli/verb/serve.rb +0 -19
- data/lib/textus/cli/verb.rb +0 -116
- data/lib/textus/cli.rb +0 -138
- data/lib/textus/dispatcher.rb +0 -54
- data/lib/textus/doctor/check/handler_allowlist.rb +0 -34
- data/lib/textus/domain/action.rb +0 -9
- data/lib/textus/domain/jobs/registry.rb +0 -37
- data/lib/textus/domain/permission.rb +0 -7
- data/lib/textus/domain/policy/base_guards.rb +0 -25
- data/lib/textus/domain/policy/evaluation.rb +0 -15
- data/lib/textus/domain/policy/guard.rb +0 -35
- data/lib/textus/domain/policy/guard_factory.rb +0 -40
- data/lib/textus/domain/policy/predicates/author_held.rb +0 -33
- data/lib/textus/domain/policy/predicates/etag_match.rb +0 -32
- data/lib/textus/domain/policy/predicates/fresh_within.rb +0 -59
- data/lib/textus/domain/policy/predicates/registry.rb +0 -39
- data/lib/textus/domain/policy/predicates/schema_valid.rb +0 -61
- data/lib/textus/domain/policy/predicates/target_is_canon.rb +0 -33
- data/lib/textus/domain/policy/predicates/zone_writable_by.rb +0 -39
- data/lib/textus/hooks/builtin.rb +0 -70
- data/lib/textus/hooks/loader.rb +0 -54
- data/lib/textus/hooks/rpc_registry.rb +0 -43
- data/lib/textus/jobs/handlers.rb +0 -62
- data/lib/textus/jobs/scheduler.rb +0 -36
- data/lib/textus/jobs/seeder.rb +0 -57
- data/lib/textus/maintenance/drain.rb +0 -42
- data/lib/textus/maintenance/key_delete_prefix.rb +0 -48
- data/lib/textus/maintenance/key_mv_prefix.rb +0 -68
- data/lib/textus/maintenance/rule_lint.rb +0 -66
- data/lib/textus/maintenance/serve.rb +0 -30
- data/lib/textus/maintenance/worker.rb +0 -74
- data/lib/textus/maintenance/zone_mv.rb +0 -64
- data/lib/textus/maintenance.rb +0 -15
- data/lib/textus/mcp/catalog.rb +0 -70
- data/lib/textus/mcp/errors.rb +0 -32
- data/lib/textus/mcp/server.rb +0 -138
- data/lib/textus/mcp/session.rb +0 -7
- data/lib/textus/mcp/tool_schemas.rb +0 -15
- data/lib/textus/mcp.rb +0 -6
- data/lib/textus/mustache.rb +0 -117
- data/lib/textus/ports/produce_on_write_subscriber.rb +0 -73
- data/lib/textus/produce/events.rb +0 -36
- data/lib/textus/read/audit.rb +0 -130
- data/lib/textus/read/boot.rb +0 -26
- data/lib/textus/read/capabilities.rb +0 -70
- data/lib/textus/read/deps.rb +0 -38
- data/lib/textus/read/doctor.rb +0 -27
- data/lib/textus/read/freshness.rb +0 -152
- data/lib/textus/read/get.rb +0 -73
- data/lib/textus/read/jobs.rb +0 -31
- data/lib/textus/read/list.rb +0 -24
- data/lib/textus/read/published.rb +0 -22
- data/lib/textus/read/pulse.rb +0 -98
- data/lib/textus/read/rdeps.rb +0 -39
- data/lib/textus/read/rule_explain.rb +0 -96
- data/lib/textus/read/rule_list.rb +0 -54
- data/lib/textus/read/schema_envelope.rb +0 -25
- data/lib/textus/read/uid.rb +0 -29
- data/lib/textus/read/validate_all.rb +0 -36
- data/lib/textus/read/where.rb +0 -24
- data/lib/textus/role_scope.rb +0 -78
- data/lib/textus/write/accept.rb +0 -58
- data/lib/textus/write/enqueue.rb +0 -50
- data/lib/textus/write/key_delete.rb +0 -65
- data/lib/textus/write/key_mv.rb +0 -141
- data/lib/textus/write/propose.rb +0 -54
- data/lib/textus/write/put.rb +0 -74
- data/lib/textus/write/reject.rb +0 -68
|
@@ -14,34 +14,36 @@ module Textus
|
|
|
14
14
|
raise BadManifest.new("manifest must be a hash") unless raw.is_a?(Hash)
|
|
15
15
|
|
|
16
16
|
walk(raw, ROOT_KEYS, "$")
|
|
17
|
+
raise BadManifest.new("manifest must declare lanes:") if Array(raw["lanes"]).empty?
|
|
18
|
+
|
|
17
19
|
validate_roles!(raw["roles"])
|
|
18
|
-
|
|
20
|
+
validate_lanes!(raw["lanes"])
|
|
19
21
|
validate_entries!(raw["entries"])
|
|
20
|
-
validate_owners!(raw["
|
|
22
|
+
validate_owners!(raw["lanes"], raw["entries"])
|
|
21
23
|
validate_rules!(raw["rules"])
|
|
22
24
|
walk(raw["audit"], AUDIT_KEYS, "$.audit") if raw["audit"].is_a?(Hash)
|
|
23
25
|
validate_single_queue!(raw)
|
|
24
26
|
validate_single_machine!(raw)
|
|
25
|
-
|
|
27
|
+
validate_lane_kind_consistency!(raw)
|
|
26
28
|
end
|
|
27
29
|
|
|
28
|
-
def
|
|
29
|
-
Array(
|
|
30
|
-
walk(z,
|
|
30
|
+
def validate_lanes!(lanes)
|
|
31
|
+
Array(lanes).each_with_index do |z, i|
|
|
32
|
+
walk(z, LANE_KEYS, "$.lanes[#{i}]")
|
|
31
33
|
if z["kind"].nil?
|
|
32
|
-
raise BadManifest.new("
|
|
34
|
+
raise BadManifest.new("lane '#{z["name"]}' at '$.lanes[#{i}]' must declare a kind (one of: #{LANE_KINDS.join(", ")})")
|
|
33
35
|
end
|
|
34
|
-
next if
|
|
36
|
+
next if LANE_KINDS.include?(z["kind"])
|
|
35
37
|
|
|
36
38
|
if %w[quarantine derived].include?(z["kind"])
|
|
37
39
|
raise BadManifest.new(
|
|
38
|
-
"
|
|
40
|
+
"lane kind '#{z["kind"]}' at '$.lanes[#{i}]' was folded into 'machine' (ADR 0091) — " \
|
|
39
41
|
"use `kind: machine`",
|
|
40
42
|
)
|
|
41
43
|
end
|
|
42
44
|
|
|
43
45
|
raise BadManifest.new(
|
|
44
|
-
"unknown
|
|
46
|
+
"unknown lane kind '#{z["kind"]}' at '$.lanes[#{i}]' (known: #{LANE_KINDS.join(", ")})",
|
|
45
47
|
)
|
|
46
48
|
end
|
|
47
49
|
end
|
|
@@ -217,9 +219,9 @@ module Textus
|
|
|
217
219
|
# (ADR 0045 D1) so attribution can't bypass the closed-name guarantee.
|
|
218
220
|
# Applies to both zone owners and entry owners; owner is optional, so a
|
|
219
221
|
# nil owner is not an error.
|
|
220
|
-
def validate_owners!(
|
|
221
|
-
Array(
|
|
222
|
-
check_owner!(z["owner"], "$.
|
|
222
|
+
def validate_owners!(lanes, entries)
|
|
223
|
+
Array(lanes).each_with_index do |z, i|
|
|
224
|
+
check_owner!(z["owner"], "$.lanes[#{i}]")
|
|
223
225
|
end
|
|
224
226
|
Array(entries).each_with_index do |e, i|
|
|
225
227
|
check_owner!(e["owner"], "$.entries[#{i}]")
|
|
@@ -265,20 +267,20 @@ module Textus
|
|
|
265
267
|
end
|
|
266
268
|
|
|
267
269
|
def validate_single_queue!(raw)
|
|
268
|
-
queues = Array(raw["
|
|
270
|
+
queues = Array(raw["lanes"]).select { |z| z["kind"] == "queue" }.map { |z| z["name"] }
|
|
269
271
|
return if queues.size <= 1
|
|
270
272
|
|
|
271
273
|
raise BadManifest.new(
|
|
272
|
-
"at most one
|
|
274
|
+
"at most one lane may declare kind: queue (found: #{queues.join(", ")})",
|
|
273
275
|
)
|
|
274
276
|
end
|
|
275
277
|
|
|
276
278
|
def validate_single_machine!(raw)
|
|
277
|
-
machines = Array(raw["
|
|
279
|
+
machines = Array(raw["lanes"]).select { |z| z["kind"] == "machine" }.map { |z| z["name"] }
|
|
278
280
|
return if machines.size <= 1
|
|
279
281
|
|
|
280
282
|
raise BadManifest.new(
|
|
281
|
-
"at most one
|
|
283
|
+
"at most one lane may declare kind: machine (found: #{machines.join(", ")})",
|
|
282
284
|
)
|
|
283
285
|
end
|
|
284
286
|
|
|
@@ -300,20 +302,20 @@ module Textus
|
|
|
300
302
|
end
|
|
301
303
|
end
|
|
302
304
|
|
|
303
|
-
# Write authority is derived from capabilities (ADR 0030): a
|
|
305
|
+
# Write authority is derived from capabilities (ADR 0030): a lane of a
|
|
304
306
|
# given kind can only be written by a role that holds the kind's required
|
|
305
|
-
# verb. Reject a manifest declaring a
|
|
307
|
+
# verb. Reject a manifest declaring a lane whose required verb is held by
|
|
306
308
|
# no role. Capabilities.resolve returns the defaults when `roles:` is nil,
|
|
307
309
|
# so the capability union is all four verbs and every kind is satisfied.
|
|
308
|
-
def
|
|
310
|
+
def validate_lane_kind_consistency!(raw)
|
|
309
311
|
held = Capabilities.resolve(raw["roles"]).values.flatten.uniq
|
|
310
312
|
|
|
311
|
-
Array(raw["
|
|
313
|
+
Array(raw["lanes"]).each_with_index do |z, i|
|
|
312
314
|
verb = KIND_REQUIRES_VERB[z["kind"]]
|
|
313
315
|
next if verb.nil? || held.include?(verb)
|
|
314
316
|
|
|
315
317
|
raise BadManifest.new(
|
|
316
|
-
"
|
|
318
|
+
"lane '#{z["name"]}' (#{z["kind"]}) at '$.lanes[#{i}]' " \
|
|
317
319
|
"needs a role with capability '#{verb}'; none declared",
|
|
318
320
|
)
|
|
319
321
|
end
|
|
@@ -7,13 +7,13 @@ module Textus
|
|
|
7
7
|
module Schema
|
|
8
8
|
# Re-export the vocabulary.
|
|
9
9
|
LANES = Vocabulary::LANES
|
|
10
|
-
|
|
10
|
+
LANE_KINDS = Vocabulary::LANE_KINDS
|
|
11
11
|
CAPABILITIES = Vocabulary::CAPABILITIES
|
|
12
12
|
KIND_REQUIRES_VERB = Vocabulary::KIND_REQUIRES_VERB
|
|
13
13
|
# Re-export the keys + registry.
|
|
14
14
|
ROOT_KEYS = Keys::ROOT_KEYS
|
|
15
15
|
ROLE_KEYS = Keys::ROLE_KEYS
|
|
16
|
-
|
|
16
|
+
LANE_KEYS = Keys::LANE_KEYS
|
|
17
17
|
ENTRY_KEYS = Keys::ENTRY_KEYS
|
|
18
18
|
PUBLISH_KEYS = Keys::PUBLISH_KEYS
|
|
19
19
|
SOURCE_KEYS = Keys::SOURCE_KEYS
|
data/lib/textus/manifest.rb
CHANGED
|
@@ -6,8 +6,8 @@ module Textus
|
|
|
6
6
|
#
|
|
7
7
|
# * data — frozen value: raw, root, zones, entries, audit_config, role_caps
|
|
8
8
|
# * resolver — resolves keys → entry + path
|
|
9
|
-
# * policy — zone/role authority (
|
|
10
|
-
#
|
|
9
|
+
# * policy — zone/role authority (lane_writers, declared_kind, derived_entry?,
|
|
10
|
+
# queue_lane?, permission_for, …)
|
|
11
11
|
# * rules — match-block rule engine (lifecycle, handler allowlist, materialize, …)
|
|
12
12
|
#
|
|
13
13
|
# Use `manifest.data.entries`, `manifest.policy.declared_kind(z)`, etc.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
require "timeout"
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
|
-
module
|
|
4
|
+
module Pipeline
|
|
5
5
|
module Acquire
|
|
6
6
|
# Invokes a :resolve_handler hook handler by name under a timeout — the single
|
|
7
7
|
# home for "call the intake handler under a deadline" (ADR 0048 D1). Shared by
|
|
@@ -18,7 +18,7 @@ module Textus
|
|
|
18
18
|
|
|
19
19
|
def invoke(caps:, handler:, config:, args:, label:, timeout: FETCH_TIMEOUT_SECONDS)
|
|
20
20
|
Timeout.timeout(timeout) do
|
|
21
|
-
caps.
|
|
21
|
+
caps.steps.invoke(:fetch, handler, caps: caps, config: config, args: args)
|
|
22
22
|
end
|
|
23
23
|
rescue Timeout::Error
|
|
24
24
|
raise Textus::UsageError.new("#{label} '#{handler}' exceeded #{timeout}s timeout")
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
require "timeout"
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
|
-
module
|
|
4
|
+
module Pipeline
|
|
5
5
|
module Acquire
|
|
6
6
|
# Internal ingest executor for one machine-zone intake entry. No longer a
|
|
7
7
|
# public verb (ADR 0079 collapsed the `fetch` surface): used by the
|
|
8
8
|
# converge sweep (drain/serve) and `textus hook run` only — ingest is system-pushed
|
|
9
9
|
# (ADR 0089 removed the read-through that once also drove it).
|
|
10
10
|
class Intake
|
|
11
|
-
FETCH_TIMEOUT_SECONDS = Textus::
|
|
11
|
+
FETCH_TIMEOUT_SECONDS = Textus::Pipeline::Acquire::Handler::FETCH_TIMEOUT_SECONDS
|
|
12
12
|
|
|
13
13
|
def initialize(container:, call:)
|
|
14
14
|
@container = container
|
|
15
15
|
@call = call
|
|
16
16
|
@manifest = container.manifest
|
|
17
17
|
@schemas = container.schemas
|
|
18
|
-
@
|
|
18
|
+
@steps = container.steps
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
# call(key) is the primary entry; run is kept as an alias for
|
|
@@ -61,10 +61,6 @@ module Textus
|
|
|
61
61
|
|
|
62
62
|
private
|
|
63
63
|
|
|
64
|
-
def fetch_events
|
|
65
|
-
@fetch_events ||= Textus::Produce::Events.from(container: @container, call: @call)
|
|
66
|
-
end
|
|
67
|
-
|
|
68
64
|
# ADR 0079: a per-rule fetch_timeout_seconds override was an accepted loss
|
|
69
65
|
# in the fetch:/retention: → lifecycle: collapse; the constant ceiling
|
|
70
66
|
# applies to every intake.
|
|
@@ -72,36 +68,35 @@ module Textus
|
|
|
72
68
|
FETCH_TIMEOUT_SECONDS
|
|
73
69
|
end
|
|
74
70
|
|
|
71
|
+
def steps_ctx
|
|
72
|
+
@steps_ctx ||= Textus::Step::Context.for(container: @container, call: @call)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
75
|
def fetch_with_events(key, mentry, remaining)
|
|
76
|
-
|
|
76
|
+
@steps.publish(:entry_fetch_started, ctx: steps_ctx, key: key, mode: :sync)
|
|
77
77
|
call_intake(key, mentry, remaining)
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
def call_intake(key, mentry, remaining)
|
|
81
|
-
Textus::
|
|
81
|
+
Textus::Pipeline::Acquire::Handler.invoke(
|
|
82
82
|
caps: @container, handler: mentry.handler,
|
|
83
83
|
config: mentry.config,
|
|
84
84
|
args: { trigger_key: key, leaf_segments: remaining || [] },
|
|
85
85
|
label: "intake", timeout: fetch_timeout_for(key)
|
|
86
86
|
)
|
|
87
87
|
rescue Textus::Error => e
|
|
88
|
-
|
|
88
|
+
@steps.publish(:entry_fetch_failed, ctx: steps_ctx, key: key,
|
|
89
|
+
error_class: e.class.name, error_message: e.message)
|
|
89
90
|
raise
|
|
90
91
|
rescue StandardError => e
|
|
91
|
-
|
|
92
|
+
@steps.publish(:entry_fetch_failed, ctx: steps_ctx, key: key,
|
|
93
|
+
error_class: e.class.name, error_message: e.message)
|
|
92
94
|
raise UsageError.new("intake '#{mentry.handler}' raised: #{e.class}: #{e.message}")
|
|
93
95
|
end
|
|
94
96
|
|
|
95
97
|
def persist_and_notify(key, mentry, result, before_etag)
|
|
96
98
|
normalized = self.class.normalize_action_result(result, format: mentry.format)
|
|
97
|
-
|
|
98
|
-
manifest: @manifest, schemas: @schemas,
|
|
99
|
-
).for(:converge, key).check!(
|
|
100
|
-
Textus::Domain::Policy::Evaluation.new(
|
|
101
|
-
actor: @call.role, transition: :converge, origin: nil,
|
|
102
|
-
target: key, envelope: nil, manifest: @manifest
|
|
103
|
-
),
|
|
104
|
-
)
|
|
99
|
+
auth.check_action!(action: :converge, actor: @call.role, key: key)
|
|
105
100
|
envelope = writer.put(
|
|
106
101
|
key,
|
|
107
102
|
mentry: mentry,
|
|
@@ -110,7 +105,10 @@ module Textus
|
|
|
110
105
|
),
|
|
111
106
|
)
|
|
112
107
|
change = detect_change(before_etag, envelope)
|
|
113
|
-
|
|
108
|
+
unless change == :unchanged
|
|
109
|
+
@steps.publish(:entry_fetched, ctx: steps_ctx, key: key,
|
|
110
|
+
envelope: envelope, change: change)
|
|
111
|
+
end
|
|
114
112
|
envelope
|
|
115
113
|
end
|
|
116
114
|
|
|
@@ -124,6 +122,10 @@ module Textus
|
|
|
124
122
|
def writer
|
|
125
123
|
@writer ||= Textus::Envelope::IO::Writer.from(container: @container, call: @call)
|
|
126
124
|
end
|
|
125
|
+
|
|
126
|
+
def auth
|
|
127
|
+
@auth ||= Textus::Gate::Auth.new(@container)
|
|
128
|
+
end
|
|
127
129
|
end
|
|
128
130
|
end
|
|
129
131
|
end
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
require "fileutils"
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
|
-
module
|
|
4
|
+
module Pipeline
|
|
5
5
|
module Acquire
|
|
6
6
|
# Builds an entry's DATA artifact (ADR 0094) by running the projection
|
|
7
7
|
# pipeline; rendering is a publish concern. External entries are NOT built
|
|
@@ -34,13 +34,13 @@ module Textus
|
|
|
34
34
|
end
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
Deps = Data.define(:manifest, :reader, :lister, :
|
|
37
|
+
Deps = Data.define(:manifest, :reader, :lister, :steps, :transform_context)
|
|
38
38
|
|
|
39
39
|
def self.renderers
|
|
40
40
|
@renderers ||= {
|
|
41
|
-
"text" =>
|
|
42
|
-
"json" =>
|
|
43
|
-
"yaml" =>
|
|
41
|
+
"text" => Textus::Pipeline::Acquire::Serializer::Text,
|
|
42
|
+
"json" => Textus::Pipeline::Acquire::Serializer::Json,
|
|
43
|
+
"yaml" => Textus::Pipeline::Acquire::Serializer::Yaml,
|
|
44
44
|
}
|
|
45
45
|
end
|
|
46
46
|
|
|
@@ -49,14 +49,16 @@ module Textus
|
|
|
49
49
|
@call = call
|
|
50
50
|
@manifest = container.manifest
|
|
51
51
|
@file_store = container.file_store
|
|
52
|
-
@
|
|
52
|
+
@steps = container.steps
|
|
53
53
|
@root = container.root
|
|
54
54
|
end
|
|
55
55
|
|
|
56
56
|
# Runs the projection pipeline for `mentry` and returns the on-disk
|
|
57
57
|
# target_path string.
|
|
58
58
|
def run(mentry)
|
|
59
|
-
reader
|
|
59
|
+
reader = lambda do |key|
|
|
60
|
+
Textus::Action::Get.new(key: key).call(container: @container, call: @call)
|
|
61
|
+
end
|
|
60
62
|
# Projections must be able to read source data from any nested entry,
|
|
61
63
|
# including keyless (publish_tree) ones like knowledge.decisions.
|
|
62
64
|
# The `include_keyless: true` option makes the resolver walk those dirs
|
|
@@ -64,15 +66,15 @@ module Textus
|
|
|
64
66
|
resolver = @manifest.resolver
|
|
65
67
|
lister = lambda do |prefix:|
|
|
66
68
|
resolver.enumerate(prefix: prefix, include_keyless: true)
|
|
67
|
-
.map { |row| { "key" => row[:key], "
|
|
69
|
+
.map { |row| { "key" => row[:key], "lane" => row[:manifest_entry].lane, "path" => row[:path] } }
|
|
68
70
|
end
|
|
69
71
|
self.class.pipeline_run(
|
|
70
72
|
mentry: mentry,
|
|
71
73
|
deps: Deps.new(
|
|
72
74
|
manifest: @manifest,
|
|
73
|
-
reader: reader
|
|
75
|
+
reader: reader,
|
|
74
76
|
lister: lister,
|
|
75
|
-
|
|
77
|
+
steps: @steps,
|
|
76
78
|
transform_context: @container,
|
|
77
79
|
),
|
|
78
80
|
)
|
|
@@ -95,7 +97,7 @@ module Textus
|
|
|
95
97
|
reader: deps.reader,
|
|
96
98
|
spec: mentry.source.projection_spec,
|
|
97
99
|
lister: deps.lister,
|
|
98
|
-
|
|
100
|
+
steps: deps.steps,
|
|
99
101
|
transform_context: deps.transform_context,
|
|
100
102
|
).run
|
|
101
103
|
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
require "json"
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
|
-
module
|
|
4
|
+
module Pipeline
|
|
5
5
|
module Acquire
|
|
6
6
|
class Serializer
|
|
7
7
|
class Json < Serializer
|
|
8
8
|
def call(mentry:, data:)
|
|
9
9
|
content = default_shape(mentry, data)
|
|
10
|
-
final =
|
|
10
|
+
final = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
|
|
11
11
|
Entry.for_format("json").serialize(meta: {}, body: "", content: final)
|
|
12
12
|
end
|
|
13
13
|
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
require "yaml"
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
|
-
module
|
|
4
|
+
module Pipeline
|
|
5
5
|
module Acquire
|
|
6
6
|
class Serializer
|
|
7
7
|
class Yaml < Serializer
|
|
8
8
|
def call(mentry:, data:)
|
|
9
9
|
content = default_shape(mentry, data)
|
|
10
|
-
final =
|
|
10
|
+
final = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
|
|
11
11
|
Entry.for_format("yaml").serialize(meta: {}, body: "", content: final)
|
|
12
12
|
end
|
|
13
13
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
module Textus
|
|
2
|
-
module
|
|
2
|
+
module Pipeline
|
|
3
3
|
# The single convergence engine (ADR 0093/0094). "Make these machine entries
|
|
4
4
|
# current from upstream." Acquire is per-`from`; publish is one uniform
|
|
5
5
|
# `publish_via` entry point for all kinds (ADR 0094):
|
|
@@ -25,9 +25,9 @@ module Textus
|
|
|
25
25
|
rescue Textus::BuildInProgress
|
|
26
26
|
nil
|
|
27
27
|
rescue Textus::Error => e
|
|
28
|
-
container.
|
|
28
|
+
container.steps.publish(
|
|
29
29
|
:produce_failed,
|
|
30
|
-
ctx: Textus::
|
|
30
|
+
ctx: Textus::Step::Context.for(container: container, call: call),
|
|
31
31
|
keys: keys, error: e.message
|
|
32
32
|
)
|
|
33
33
|
end
|
|
@@ -62,7 +62,7 @@ module Textus
|
|
|
62
62
|
entry = @manifest.resolver.resolve(key).entry
|
|
63
63
|
|
|
64
64
|
if entry.intake?
|
|
65
|
-
Textus::
|
|
65
|
+
Textus::Pipeline::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
|
|
66
66
|
entry.publish_via(context) # emit any targets
|
|
67
67
|
out[:produced] << key # a fetch is production
|
|
68
68
|
else
|
|
@@ -87,7 +87,9 @@ module Textus
|
|
|
87
87
|
def build_context(call)
|
|
88
88
|
Textus::Manifest::Entry::Base::PublishContext.new(
|
|
89
89
|
container: @container, call: call,
|
|
90
|
-
reader:
|
|
90
|
+
reader: lambda { |key|
|
|
91
|
+
Textus::Action::Get.new(key: key).call(container: @container, call: call)
|
|
92
|
+
}
|
|
91
93
|
)
|
|
92
94
|
end
|
|
93
95
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
require "mustache"
|
|
2
|
+
|
|
1
3
|
module Textus
|
|
2
|
-
module
|
|
4
|
+
module Pipeline
|
|
3
5
|
# Renders an entry's stored DATA into the bytes for one publish target
|
|
4
6
|
# (ADR 0094). Relocates the Mustache logic that used to live in the
|
|
5
7
|
# build-time Markdown renderer. Provenance is NOT added here — it lives in
|
|
@@ -78,16 +78,42 @@ module Textus
|
|
|
78
78
|
def verify_integrity
|
|
79
79
|
return [] unless File.exist?(@path)
|
|
80
80
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
[].tap do |out|
|
|
82
|
+
iterate_with_prev_seq do |line, lineno, prev_seq|
|
|
83
|
+
check_line_integrity(line, lineno, prev_seq, out)
|
|
84
|
+
end
|
|
85
85
|
end
|
|
86
|
-
out
|
|
87
86
|
end
|
|
88
87
|
|
|
89
88
|
private
|
|
90
89
|
|
|
90
|
+
def iterate_with_prev_seq
|
|
91
|
+
prev_seq = nil
|
|
92
|
+
File.foreach(@path).with_index(1) do |line, lineno|
|
|
93
|
+
yield line.chomp, lineno, prev_seq
|
|
94
|
+
parsed = parse_row(line.chomp)
|
|
95
|
+
prev_seq = parsed["seq"] if parsed&.dig("seq").is_a?(Integer)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def check_line_integrity(line, lineno, prev_seq, out)
|
|
100
|
+
violation = check_line(line, lineno)
|
|
101
|
+
if violation
|
|
102
|
+
out << violation
|
|
103
|
+
return
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
parsed = parse_row(line)
|
|
107
|
+
return unless parsed && (seq = parsed["seq"]).is_a?(Integer)
|
|
108
|
+
return unless prev_seq && seq != prev_seq + 1
|
|
109
|
+
|
|
110
|
+
out << {
|
|
111
|
+
"lineno" => lineno,
|
|
112
|
+
"reason" => "seq_gap",
|
|
113
|
+
"detail" => "expected #{prev_seq + 1}, got #{seq}",
|
|
114
|
+
}
|
|
115
|
+
end
|
|
116
|
+
|
|
91
117
|
def rotated(n)
|
|
92
118
|
File.join(Textus::Layout.audit_dir(@root), "audit.log.#{n}")
|
|
93
119
|
end
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Ports
|
|
5
5
|
# Writes an "event_error" audit row when a user hook raises during
|
|
6
|
-
#
|
|
6
|
+
# Step::EventBus publish. Attached at Store boot.
|
|
7
7
|
#
|
|
8
|
-
# Integration: uses
|
|
8
|
+
# Integration: uses Step::EventBus#on_error callback (chosen over a
|
|
9
9
|
# synthetic :hook_error event because the bus already owns the
|
|
10
10
|
# rescue and the failure is a bus-internal concern, not a domain
|
|
11
11
|
# event subscribers should be able to filter by key glob).
|
|
@@ -19,8 +19,8 @@ module Textus
|
|
|
19
19
|
@audit_log = audit_log
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
def attach(
|
|
23
|
-
|
|
22
|
+
def attach(registry)
|
|
23
|
+
registry.on_error do |event:, hook:, key:, kwargs:, error:|
|
|
24
24
|
record_error(event: event, hook: hook, key: key, kwargs: kwargs, error: error)
|
|
25
25
|
end
|
|
26
26
|
self
|
|
@@ -2,12 +2,12 @@ require "digest"
|
|
|
2
2
|
require "json"
|
|
3
3
|
|
|
4
4
|
module Textus
|
|
5
|
-
module
|
|
6
|
-
|
|
5
|
+
module Ports
|
|
6
|
+
class Queue
|
|
7
7
|
# A unit of deferred work. Pure data. The id is `<type>:<digest>` where the
|
|
8
8
|
# digest is over the args with sorted keys, so logically-identical enqueues
|
|
9
|
-
# collide on the same id — which is how
|
|
10
|
-
#
|
|
9
|
+
# collide on the same id — which is how Queue dedups (file already exists).
|
|
10
|
+
# At-least-once delivery means handlers must be idempotent.
|
|
11
11
|
class Job
|
|
12
12
|
DIGEST_BYTES = 16
|
|
13
13
|
|
|
@@ -15,10 +15,10 @@ module Textus
|
|
|
15
15
|
attr_accessor :attempts, :last_error
|
|
16
16
|
|
|
17
17
|
def initialize(type:, args:, enqueued_by: nil, attempts: 0, max_attempts: 3, last_error: nil)
|
|
18
|
-
@type
|
|
19
|
-
@args
|
|
18
|
+
@type = type.to_s
|
|
19
|
+
@args = stringify(args)
|
|
20
20
|
@enqueued_by = enqueued_by
|
|
21
|
-
@attempts
|
|
21
|
+
@attempts = attempts
|
|
22
22
|
@max_attempts = max_attempts
|
|
23
23
|
@last_error = last_error
|
|
24
24
|
end
|
|
@@ -29,16 +29,23 @@ module Textus
|
|
|
29
29
|
|
|
30
30
|
def to_h
|
|
31
31
|
{
|
|
32
|
-
"type" => @type,
|
|
33
|
-
"
|
|
32
|
+
"type" => @type,
|
|
33
|
+
"args" => @args,
|
|
34
|
+
"enqueued_by" => @enqueued_by,
|
|
35
|
+
"attempts" => @attempts,
|
|
36
|
+
"max_attempts" => @max_attempts,
|
|
37
|
+
"last_error" => @last_error,
|
|
34
38
|
}
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
def self.from_h(hash)
|
|
38
42
|
new(
|
|
39
|
-
type: hash["type"],
|
|
40
|
-
|
|
41
|
-
|
|
43
|
+
type: hash["type"],
|
|
44
|
+
args: hash["args"] || {},
|
|
45
|
+
enqueued_by: hash["enqueued_by"],
|
|
46
|
+
attempts: hash["attempts"] || 0,
|
|
47
|
+
max_attempts: hash["max_attempts"] || 3,
|
|
48
|
+
last_error: hash["last_error"],
|
|
42
49
|
)
|
|
43
50
|
end
|
|
44
51
|
|
data/lib/textus/ports/queue.rb
CHANGED
|
@@ -42,7 +42,7 @@ module Textus
|
|
|
42
42
|
rescue Errno::ENOENT
|
|
43
43
|
next # another worker won this one
|
|
44
44
|
end
|
|
45
|
-
job =
|
|
45
|
+
job = Job.from_h(JSON.parse(File.read(dst)))
|
|
46
46
|
stamp_lease(dst, worker_id: worker_id, expires_at: Time.now.utc + lease_ttl)
|
|
47
47
|
return Leased.new(job: job, leased_path: dst)
|
|
48
48
|
end
|
|
@@ -7,7 +7,7 @@ module Textus
|
|
|
7
7
|
# Persistence adapter for sentinel files. Owns the on-disk JSON shape, the
|
|
8
8
|
# path layout (<store_root>/.run/sentinels/<target-rel-to-repo>.textus-managed.json
|
|
9
9
|
# — runtime, git-ignored, ADR 0070), and all File/FileUtils I/O.
|
|
10
|
-
#
|
|
10
|
+
# Core::Sentinel is a pure value object that depends on this port for
|
|
11
11
|
# reads and writes.
|
|
12
12
|
class SentinelStore
|
|
13
13
|
SUFFIX = ".textus-managed.json".freeze
|
|
@@ -26,7 +26,7 @@ module Textus
|
|
|
26
26
|
|
|
27
27
|
def load(path, repo_root)
|
|
28
28
|
raw = JSON.parse(File.read(path))
|
|
29
|
-
Textus::
|
|
29
|
+
Textus::Core::Sentinel.new(
|
|
30
30
|
target: absolutize(raw["target"], repo_root),
|
|
31
31
|
source: absolutize(raw["source"], repo_root),
|
|
32
32
|
sha256: raw["sha256"],
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Textus
|
|
6
|
+
module Ports
|
|
7
|
+
# Flock-based watcher presence lock. Held for the watcher's lifetime.
|
|
8
|
+
# Process death releases the flock automatically.
|
|
9
|
+
class WatcherLock
|
|
10
|
+
def initialize(root)
|
|
11
|
+
@path = Textus::Layout.watcher_lock(root)
|
|
12
|
+
@file = nil
|
|
13
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def self.running?(root)
|
|
17
|
+
path = Textus::Layout.watcher_lock(root)
|
|
18
|
+
return false unless File.exist?(path)
|
|
19
|
+
|
|
20
|
+
File.open(path, "r+") do |file|
|
|
21
|
+
got = file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
22
|
+
file.flock(File::LOCK_UN) if got
|
|
23
|
+
return !got
|
|
24
|
+
end
|
|
25
|
+
rescue Errno::ENOENT
|
|
26
|
+
false
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def acquire
|
|
30
|
+
@file = File.open(@path, File::RDWR | File::CREAT, 0o644)
|
|
31
|
+
raise "watcher already running" unless @file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
32
|
+
|
|
33
|
+
@file.write("pid=#{Process.pid} host=#{Socket.gethostname}\n")
|
|
34
|
+
@file.flush
|
|
35
|
+
self
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def release
|
|
39
|
+
return unless @file
|
|
40
|
+
|
|
41
|
+
@file.flock(File::LOCK_UN)
|
|
42
|
+
@file.close
|
|
43
|
+
FileUtils.rm_f(@path)
|
|
44
|
+
@file = nil
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|