textus 0.50.0 → 0.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +41 -43
- data/SPEC.md +176 -176
- data/docs/architecture/README.md +46 -42
- data/docs/reference/conventions.md +31 -26
- data/lib/textus/boot.rb +15 -17
- data/lib/textus/call.rb +1 -1
- data/lib/textus/cli/runner.rb +15 -10
- data/lib/textus/cli/verb/get.rb +1 -3
- data/lib/textus/cli/verb/hook_run.rb +1 -1
- data/lib/textus/cli/verb/put.rb +4 -20
- data/lib/textus/cli/verb/serve.rb +19 -0
- data/lib/textus/cli.rb +1 -3
- data/lib/textus/dispatcher.rb +3 -3
- data/lib/textus/doctor/check/generator_drift.rb +4 -3
- data/lib/textus/doctor/check/handler_allowlist.rb +1 -1
- data/lib/textus/doctor/check/intake_registration.rb +5 -5
- data/lib/textus/doctor/check/rule_ambiguity.rb +3 -3
- data/lib/textus/doctor/check/sentinels.rb +2 -2
- data/lib/textus/doctor/check/templates.rb +13 -11
- data/lib/textus/doctor.rb +0 -2
- data/lib/textus/domain/freshness/evaluator.rb +150 -14
- data/lib/textus/domain/freshness/verdict.rb +28 -6
- data/lib/textus/domain/freshness.rb +4 -33
- data/lib/textus/domain/jobs/job.rb +58 -0
- data/lib/textus/domain/jobs/registry.rb +37 -0
- data/lib/textus/domain/policy/base_guards.rb +1 -1
- data/lib/textus/domain/policy/predicates/fresh_within.rb +1 -1
- data/lib/textus/domain/policy/publish_target.rb +34 -0
- data/lib/textus/domain/policy/retention.rb +29 -0
- data/lib/textus/domain/policy/source.rb +73 -0
- data/lib/textus/domain/retention/sweep.rb +57 -0
- data/lib/textus/domain/retention.rb +11 -0
- data/lib/textus/errors.rb +4 -4
- data/lib/textus/hooks/builtin.rb +5 -5
- data/lib/textus/hooks/catalog.rb +7 -7
- data/lib/textus/hooks/context.rb +5 -10
- data/lib/textus/init/templates/machine_intake.rb +4 -4
- data/lib/textus/init.rb +47 -47
- data/lib/textus/jobs/handlers.rb +62 -0
- data/lib/textus/jobs/scheduler.rb +36 -0
- data/lib/textus/jobs/seeder.rb +57 -0
- data/lib/textus/key/matching.rb +24 -0
- data/lib/textus/layout.rb +8 -0
- data/lib/textus/maintenance/drain.rb +42 -0
- data/lib/textus/maintenance/retention/apply.rb +52 -0
- data/lib/textus/maintenance/serve.rb +30 -0
- data/lib/textus/maintenance/worker.rb +74 -0
- data/lib/textus/manifest/capabilities.rb +1 -1
- data/lib/textus/manifest/data.rb +18 -3
- data/lib/textus/manifest/entry/base.rb +28 -9
- data/lib/textus/manifest/entry/nested.rb +3 -4
- data/lib/textus/manifest/entry/parser.rb +25 -21
- data/lib/textus/manifest/entry/produced.rb +56 -0
- data/lib/textus/manifest/entry/publish/subtree_mirror.rb +7 -6
- data/lib/textus/manifest/entry/publish/to_paths.rb +62 -11
- data/lib/textus/manifest/entry/validators/format_matrix.rb +3 -11
- data/lib/textus/manifest/entry/validators/publish.rb +3 -1
- data/lib/textus/manifest/entry/validators.rb +0 -1
- data/lib/textus/manifest/policy.rb +16 -4
- data/lib/textus/manifest/resolver.rb +10 -4
- data/lib/textus/manifest/rules.rb +37 -36
- data/lib/textus/manifest/schema/keys.rb +98 -0
- data/lib/textus/manifest/schema/validator.rb +324 -0
- data/lib/textus/manifest/schema/vocabulary.rb +24 -0
- data/lib/textus/manifest/schema.rb +27 -247
- data/lib/textus/manifest.rb +5 -3
- data/lib/textus/mcp/server.rb +1 -1
- data/lib/textus/ports/audit_log.rb +6 -0
- data/lib/textus/ports/build_lock.rb +6 -0
- data/lib/textus/ports/clock.rb +4 -3
- data/lib/textus/ports/produce_on_write_subscriber.rb +73 -0
- data/lib/textus/ports/publisher.rb +11 -7
- data/lib/textus/ports/queue.rb +130 -0
- data/lib/textus/produce/acquire/handler.rb +29 -0
- data/lib/textus/produce/acquire/intake.rb +130 -0
- data/lib/textus/produce/acquire/projection.rb +127 -0
- data/lib/textus/produce/acquire/serializer/json.rb +31 -0
- data/lib/textus/produce/acquire/serializer/text.rb +16 -0
- data/lib/textus/produce/acquire/serializer/yaml.rb +31 -0
- data/lib/textus/produce/acquire/serializer.rb +17 -0
- data/lib/textus/produce/engine.rb +95 -0
- data/lib/textus/produce/events.rb +36 -0
- data/lib/textus/produce/render.rb +23 -0
- data/lib/textus/projection.rb +17 -6
- data/lib/textus/read/deps.rb +3 -3
- data/lib/textus/read/freshness.rb +61 -31
- data/lib/textus/read/get.rb +20 -102
- data/lib/textus/read/jobs.rb +31 -0
- data/lib/textus/read/rdeps.rb +3 -3
- data/lib/textus/read/rule_explain.rb +41 -23
- data/lib/textus/read/rule_list.rb +25 -8
- data/lib/textus/read/validate_all.rb +14 -0
- data/lib/textus/role.rb +2 -1
- data/lib/textus/schemas.rb +8 -0
- data/lib/textus/store.rb +1 -0
- data/lib/textus/version.rb +1 -1
- data/lib/textus/write/enqueue.rb +50 -0
- data/lib/textus/write/put.rb +1 -1
- metadata +35 -30
- data/lib/textus/builder/pipeline.rb +0 -88
- data/lib/textus/builder/renderer/json.rb +0 -45
- data/lib/textus/builder/renderer/markdown.rb +0 -24
- data/lib/textus/builder/renderer/text.rb +0 -14
- data/lib/textus/builder/renderer/yaml.rb +0 -45
- data/lib/textus/builder/renderer.rb +0 -17
- data/lib/textus/cli/verb/boot.rb +0 -14
- data/lib/textus/cli/verb/build.rb +0 -15
- data/lib/textus/doctor/check/fetch_locks.rb +0 -49
- data/lib/textus/doctor/check/lifecycle_action_invalid.rb +0 -39
- data/lib/textus/domain/freshness/policy.rb +0 -18
- data/lib/textus/domain/lifecycle.rb +0 -83
- data/lib/textus/domain/outcome.rb +0 -10
- data/lib/textus/domain/policy/lifecycle.rb +0 -35
- data/lib/textus/domain/staleness/generator_check.rb +0 -109
- data/lib/textus/domain/staleness.rb +0 -29
- data/lib/textus/maintenance/tend.rb +0 -110
- data/lib/textus/manifest/entry/derived.rb +0 -67
- data/lib/textus/manifest/entry/intake.rb +0 -31
- data/lib/textus/manifest/entry/validators/inject_boot.rb +0 -21
- data/lib/textus/mcp/tools.rb +0 -14
- data/lib/textus/ports/fetch/detached.rb +0 -52
- data/lib/textus/ports/fetch/lock.rb +0 -44
- data/lib/textus/write/build.rb +0 -90
- data/lib/textus/write/fetch_events.rb +0 -42
- data/lib/textus/write/fetch_orchestrator.rb +0 -101
- data/lib/textus/write/fetch_worker.rb +0 -127
- data/lib/textus/write/intake_fetch.rb +0 -25
- data/lib/textus/write/materializer.rb +0 -51
|
@@ -2,11 +2,11 @@ module Textus
|
|
|
2
2
|
module Doctor
|
|
3
3
|
class Check
|
|
4
4
|
# Flags entries whose key is matched by two or more rule blocks of the
|
|
5
|
-
# SAME specificity in the same slot (
|
|
6
|
-
# guard). Ties are non-deterministic in the parser's pick step, so
|
|
5
|
+
# SAME specificity in the same slot (lifecycle / handler_allowlist /
|
|
6
|
+
# guard / materialize). Ties are non-deterministic in the parser's pick step, so
|
|
7
7
|
# they're a configuration smell — surface them.
|
|
8
8
|
class RuleAmbiguity < Check
|
|
9
|
-
SLOTS =
|
|
9
|
+
SLOTS = Textus::Manifest::Schema::FIELD_REGISTRY.select { |_, m| m[:in_ambiguity] }.keys.freeze
|
|
10
10
|
|
|
11
11
|
def call
|
|
12
12
|
out = []
|
|
@@ -31,7 +31,7 @@ module Textus
|
|
|
31
31
|
"level" => "warning",
|
|
32
32
|
"subject" => sentinel_path,
|
|
33
33
|
"message" => "sentinel is not valid JSON",
|
|
34
|
-
"fix" => "delete #{sentinel_path} and re-run 'textus
|
|
34
|
+
"fix" => "delete #{sentinel_path} and re-run 'textus drain' to regenerate",
|
|
35
35
|
}
|
|
36
36
|
end
|
|
37
37
|
|
|
@@ -51,7 +51,7 @@ module Textus
|
|
|
51
51
|
"level" => "warning",
|
|
52
52
|
"subject" => sentinel.target,
|
|
53
53
|
"message" => "published file at #{sentinel.target} was modified out-of-band",
|
|
54
|
-
"fix" => "re-run 'textus
|
|
54
|
+
"fix" => "re-run 'textus drain' to overwrite, or copy the manual edit back into the store source",
|
|
55
55
|
}
|
|
56
56
|
end
|
|
57
57
|
end
|
|
@@ -5,19 +5,21 @@ module Textus
|
|
|
5
5
|
def call
|
|
6
6
|
out = []
|
|
7
7
|
manifest.data.entries.each do |entry|
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
entry.publish_targets.each do |target|
|
|
9
|
+
template = target.template
|
|
10
|
+
next if template.nil?
|
|
10
11
|
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
tp = File.join(root, "templates", template)
|
|
13
|
+
next if File.exist?(tp)
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
out << {
|
|
16
|
+
"code" => "template.missing",
|
|
17
|
+
"level" => "error",
|
|
18
|
+
"subject" => entry.key,
|
|
19
|
+
"message" => "template '#{template}' not found at #{tp}",
|
|
20
|
+
"fix" => "create the file at #{tp} or update the publish target's template: field",
|
|
21
|
+
}
|
|
22
|
+
end
|
|
21
23
|
end
|
|
22
24
|
out
|
|
23
25
|
end
|
data/lib/textus/doctor.rb
CHANGED
|
@@ -23,11 +23,9 @@ module Textus
|
|
|
23
23
|
Check::SchemaViolations,
|
|
24
24
|
Check::RuleAmbiguity,
|
|
25
25
|
Check::HandlerAllowlist,
|
|
26
|
-
Check::FetchLocks,
|
|
27
26
|
Check::OrphanedPublishTargets,
|
|
28
27
|
Check::PublishTreeIndexOverlap,
|
|
29
28
|
Check::ProposalTargets,
|
|
30
|
-
Check::LifecycleActionInvalid,
|
|
31
29
|
Check::GeneratorDrift,
|
|
32
30
|
].freeze
|
|
33
31
|
|
|
@@ -2,27 +2,163 @@ require "time"
|
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Domain
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
module Freshness
|
|
6
|
+
# The single currency evaluator (ADR 0099). Answers "is the stored data
|
|
7
|
+
# stale relative to its source?" for every produce-method:
|
|
8
|
+
# - intake (source.from: handler) -> AGE signal: now - basis > source.ttl,
|
|
9
|
+
# basis = _meta.last_fetched_at (else file mtime). No ttl -> :no_policy
|
|
10
|
+
# (skipped — a cadence-less handler is not auto-re-pulled).
|
|
11
|
+
# - derived/external -> DRIFT signal: a source changed since generated.at
|
|
12
|
+
# (surfaced by the doctor generator_drift check; derived entries annotate
|
|
13
|
+
# fresh at read time because converge runs them reactively).
|
|
14
|
+
# Replaces Domain::IntakeStaleness and Domain::Staleness::GeneratorCheck and
|
|
15
|
+
# the inline copies in Read::Get / Read::Freshness.
|
|
16
|
+
class Evaluator
|
|
17
|
+
def initialize(manifest:, file_stat:, clock:)
|
|
18
|
+
@manifest = manifest
|
|
19
|
+
@file_stat = file_stat
|
|
20
|
+
@clock = clock
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Per-entry currency Verdict (drives Read::Get's annotation). Non-intake
|
|
24
|
+
# entries are always fresh (retention is GC, not content currency).
|
|
25
|
+
def verdict(mentry)
|
|
26
|
+
return fresh unless mentry.intake?
|
|
27
|
+
|
|
28
|
+
ttl = mentry.source.ttl_seconds
|
|
29
|
+
return fresh if ttl.nil?
|
|
30
|
+
|
|
31
|
+
stale = age_stale?(intake_basis(mentry), ttl)
|
|
32
|
+
Verdict.build(stale: stale, reason: stale ? "ttl exceeded" : nil, fetching: false)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Keys of intake entries past their source.ttl — the converge produce
|
|
36
|
+
# scope (replaces Domain::IntakeStaleness#call). A ttl-less intake entry
|
|
37
|
+
# is :no_policy and skipped; a never-recorded one (with a ttl) is stale.
|
|
38
|
+
def stale_intake_keys(prefix: nil, zone: nil)
|
|
39
|
+
@manifest.data.entries.select { |m| due?(m, prefix: prefix, zone: zone) }.map(&:key)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Age basis as a Time (or nil): _meta.last_fetched_at when present, else
|
|
43
|
+
# file mtime. The single definition the three call sites used to repeat.
|
|
44
|
+
def intake_basis(mentry)
|
|
45
|
+
path = @manifest.resolver.resolve(mentry.key).path
|
|
46
|
+
return nil unless @file_stat.exists?(path)
|
|
47
|
+
|
|
48
|
+
last_fetched_at(mentry, path) || @file_stat.mtime(path)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Generator-drift rows for one entry (replaces Staleness::GeneratorCheck#
|
|
52
|
+
# rows_for) — consumed by the doctor generator_drift check.
|
|
53
|
+
def drift_rows(mentry)
|
|
54
|
+
return [] unless drift_applicable?(mentry)
|
|
55
|
+
|
|
56
|
+
path = Textus::Key::Path.resolve(@manifest.data, mentry)
|
|
57
|
+
reason = drift_reason(mentry, path)
|
|
58
|
+
reason ? [drift_row(mentry, path, reason)] : []
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def fresh = Verdict.build(stale: false, reason: nil, fetching: false)
|
|
64
|
+
|
|
65
|
+
def due?(mentry, prefix:, zone:)
|
|
66
|
+
return false unless mentry.intake?
|
|
67
|
+
return false if zone && mentry.zone != zone
|
|
68
|
+
return false if prefix && !mentry.key.start_with?(prefix)
|
|
69
|
+
|
|
70
|
+
ttl = mentry.source.ttl_seconds
|
|
71
|
+
return false if ttl.nil? # no declared cadence -> :no_policy, skip (ADR 0099)
|
|
72
|
+
|
|
73
|
+
path = @manifest.resolver.resolve(mentry.key).path
|
|
74
|
+
return true unless @file_stat.exists?(path)
|
|
75
|
+
|
|
76
|
+
age_stale?(intake_basis(mentry), ttl)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# The one age comparison. A never-recorded entry (nil basis) is stale.
|
|
80
|
+
def age_stale?(basis, ttl)
|
|
81
|
+
return true if basis.nil?
|
|
82
|
+
|
|
83
|
+
(@clock.now - basis).to_i > ttl
|
|
84
|
+
end
|
|
8
85
|
|
|
9
|
-
def
|
|
10
|
-
|
|
86
|
+
def last_fetched_at(mentry, path)
|
|
87
|
+
meta = Entry.for_format(mentry.format).parse(@file_stat.read(path), path: path)["_meta"]
|
|
88
|
+
Time.parse(meta["last_fetched_at"].to_s) if meta && meta["last_fetched_at"]
|
|
89
|
+
rescue StandardError
|
|
90
|
+
nil
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# --- generator drift (lifted from Staleness::GeneratorCheck) ---
|
|
94
|
+
|
|
95
|
+
def drift_applicable?(mentry) = mentry.derived? && mentry.external?
|
|
96
|
+
|
|
97
|
+
def drift_reason(mentry, path)
|
|
98
|
+
return "derived entry has never been generated" unless @file_stat.exists?(path)
|
|
99
|
+
|
|
100
|
+
generated_at = generated_at_of(mentry, path)
|
|
101
|
+
return "missing generated.at frontmatter" unless generated_at
|
|
102
|
+
|
|
103
|
+
gen_time = parse_time(generated_at)
|
|
104
|
+
return "unparseable generated.at: #{generated_at.inspect}" unless gen_time
|
|
11
105
|
|
|
12
|
-
|
|
13
|
-
|
|
106
|
+
offender = newest_source_after(mentry.source, gen_time)
|
|
107
|
+
"source '#{offender}' modified after generated.at" if offender
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def generated_at_of(mentry, path)
|
|
111
|
+
Entry.for_format(mentry.format).parse(@file_stat.read(path), path: path)["_meta"].dig("generated", "at")
|
|
112
|
+
end
|
|
14
113
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
114
|
+
def parse_time(str)
|
|
115
|
+
Time.parse(str.to_s)
|
|
116
|
+
rescue StandardError
|
|
117
|
+
nil
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def newest_source_after(external_src, gen_time)
|
|
121
|
+
Array(external_src.sources).each do |src|
|
|
122
|
+
offender = check_source(src, gen_time)
|
|
123
|
+
return offender if offender
|
|
124
|
+
end
|
|
125
|
+
nil
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def check_source(src, gen_time)
|
|
129
|
+
if src.match?(/\A[a-z0-9.][a-z0-9._-]*\z/) && !src.include?("/")
|
|
130
|
+
@manifest.resolver.enumerate(prefix: src).each do |row|
|
|
131
|
+
return src if @file_stat.mtime(row[:path]) > gen_time
|
|
132
|
+
end
|
|
18
133
|
nil
|
|
134
|
+
else
|
|
135
|
+
check_filesystem_source(src, gen_time)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def check_filesystem_source(src, gen_time)
|
|
140
|
+
abs = absolutize_source(src)
|
|
141
|
+
if @file_stat.directory?(abs)
|
|
142
|
+
dir_has_newer_file?(abs, gen_time) ? src : nil
|
|
143
|
+
elsif @file_stat.exists?(abs) && @file_stat.mtime(abs) > gen_time
|
|
144
|
+
src
|
|
19
145
|
end
|
|
20
|
-
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def absolutize_source(src)
|
|
149
|
+
File.absolute_path?(src) ? src : File.join(File.dirname(@manifest.data.root), src)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def dir_has_newer_file?(abs, gen_time)
|
|
153
|
+
@file_stat.glob(File.join(abs, "**", "*")).any? do |fpath|
|
|
154
|
+
file?(fpath) && @file_stat.mtime(fpath) > gen_time
|
|
155
|
+
end
|
|
156
|
+
end
|
|
21
157
|
|
|
22
|
-
|
|
23
|
-
return Verdict.fresh if age <= policy.ttl_seconds
|
|
158
|
+
def file?(fpath) = !@file_stat.directory?(fpath) && @file_stat.exists?(fpath)
|
|
24
159
|
|
|
25
|
-
|
|
160
|
+
def drift_row(mentry, path, reason)
|
|
161
|
+
{ "key" => mentry.key, "path" => path, "generator" => mentry.raw["compute"], "reason" => reason }
|
|
26
162
|
end
|
|
27
163
|
end
|
|
28
164
|
end
|
|
@@ -1,11 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Textus
|
|
2
4
|
module Domain
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
module Freshness
|
|
6
|
+
# Value object describing the freshness annotation attached to an Envelope
|
|
7
|
+
# after a currency evaluation (ADR 0099 — was Domain::Freshness).
|
|
8
|
+
#
|
|
9
|
+
# Note on wire format: `#to_h_for_wire` is intentionally narrower than the
|
|
10
|
+
# full field set. It emits the legacy keys ("stale", "stale_reason",
|
|
11
|
+
# "fetching", and "fetch_error" when present) so the CLI JSON wire stays
|
|
12
|
+
# byte-identical with textus/3. The gem-side fields `checked_at` and
|
|
13
|
+
# `ttl_remaining_ms` are NOT emitted on the wire.
|
|
14
|
+
Verdict = Data.define(
|
|
15
|
+
:stale, :fetching, :reason, :fetch_error, :checked_at, :ttl_remaining_ms
|
|
16
|
+
) do
|
|
17
|
+
def self.build(stale:, fetching: false, reason: nil, fetch_error: nil,
|
|
18
|
+
checked_at: nil, ttl_remaining_ms: nil)
|
|
19
|
+
new(
|
|
20
|
+
stale: stale, fetching: fetching, reason: reason,
|
|
21
|
+
fetch_error: fetch_error, checked_at: checked_at,
|
|
22
|
+
ttl_remaining_ms: ttl_remaining_ms
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def to_h_for_wire
|
|
27
|
+
h = { "stale" => stale, "stale_reason" => reason, "fetching" => fetching }
|
|
28
|
+
h["fetch_error"] = fetch_error unless fetch_error.nil?
|
|
29
|
+
h
|
|
30
|
+
end
|
|
9
31
|
end
|
|
10
32
|
end
|
|
11
33
|
end
|
|
@@ -2,39 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Domain
|
|
5
|
-
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
|
|
9
|
-
# Note on wire format: `#to_h_for_wire` is intentionally narrower than the
|
|
10
|
-
# full field set. It emits the legacy keys ("stale", "stale_reason",
|
|
11
|
-
# "fetching", and "fetch_error" when present) so the CLI JSON wire
|
|
12
|
-
# stays byte-identical with textus/3. The gem-side fields `checked_at`
|
|
13
|
-
# and `ttl_remaining_ms` are NOT emitted on the wire in this phase.
|
|
14
|
-
Freshness = Data.define(
|
|
15
|
-
:stale, :fetching, :reason, :fetch_error, :checked_at, :ttl_remaining_ms
|
|
16
|
-
) do
|
|
17
|
-
def self.build(stale:, fetching: false, reason: nil, fetch_error: nil,
|
|
18
|
-
checked_at: nil, ttl_remaining_ms: nil)
|
|
19
|
-
new(
|
|
20
|
-
stale: stale,
|
|
21
|
-
fetching: fetching,
|
|
22
|
-
reason: reason,
|
|
23
|
-
fetch_error: fetch_error,
|
|
24
|
-
checked_at: checked_at,
|
|
25
|
-
ttl_remaining_ms: ttl_remaining_ms,
|
|
26
|
-
)
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def to_h_for_wire
|
|
30
|
-
h = {
|
|
31
|
-
"stale" => stale,
|
|
32
|
-
"stale_reason" => reason,
|
|
33
|
-
"fetching" => fetching,
|
|
34
|
-
}
|
|
35
|
-
h["fetch_error"] = fetch_error unless fetch_error.nil?
|
|
36
|
-
h
|
|
37
|
-
end
|
|
5
|
+
# Currency — "is the stored data stale relative to its source?" (ADR 0099).
|
|
6
|
+
# The home of the single Freshness evaluator and its Verdict value object.
|
|
7
|
+
# Distinct from Domain::Retention (GC dueness, Q2).
|
|
8
|
+
module Freshness
|
|
38
9
|
end
|
|
39
10
|
end
|
|
40
11
|
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
require "digest"
|
|
2
|
+
require "json"
|
|
3
|
+
|
|
4
|
+
module Textus
|
|
5
|
+
module Domain
|
|
6
|
+
module Jobs
|
|
7
|
+
# A unit of deferred work. Pure data. The id is `<type>:<digest>` where the
|
|
8
|
+
# digest is over the args with sorted keys, so logically-identical enqueues
|
|
9
|
+
# collide on the same id — which is how the Queue dedups (the file already
|
|
10
|
+
# exists). At-least-once delivery means handlers must be idempotent.
|
|
11
|
+
class Job
|
|
12
|
+
DIGEST_BYTES = 16
|
|
13
|
+
|
|
14
|
+
attr_reader :type, :args, :enqueued_by, :max_attempts
|
|
15
|
+
attr_accessor :attempts, :last_error
|
|
16
|
+
|
|
17
|
+
def initialize(type:, args:, enqueued_by: nil, attempts: 0, max_attempts: 3, last_error: nil)
|
|
18
|
+
@type = type.to_s
|
|
19
|
+
@args = stringify(args)
|
|
20
|
+
@enqueued_by = enqueued_by
|
|
21
|
+
@attempts = attempts
|
|
22
|
+
@max_attempts = max_attempts
|
|
23
|
+
@last_error = last_error
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def id
|
|
27
|
+
"#{@type}:#{digest}"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def to_h
|
|
31
|
+
{
|
|
32
|
+
"type" => @type, "args" => @args, "enqueued_by" => @enqueued_by,
|
|
33
|
+
"attempts" => @attempts, "max_attempts" => @max_attempts, "last_error" => @last_error
|
|
34
|
+
}
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def self.from_h(hash)
|
|
38
|
+
new(
|
|
39
|
+
type: hash["type"], args: hash["args"] || {}, enqueued_by: hash["enqueued_by"],
|
|
40
|
+
attempts: hash["attempts"] || 0, max_attempts: hash["max_attempts"] || 3,
|
|
41
|
+
last_error: hash["last_error"]
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def digest
|
|
48
|
+
canonical = JSON.dump(@args.sort.to_h)
|
|
49
|
+
Digest::SHA256.hexdigest(canonical)[0, DIGEST_BYTES]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def stringify(hash)
|
|
53
|
+
hash.transform_keys(&:to_s)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module Textus
|
|
2
|
+
module Domain
|
|
3
|
+
module Jobs
|
|
4
|
+
# Closed allow-list of runnable job types. The general `enqueue` surface
|
|
5
|
+
# (a later phase) can only push types registered here — that is the safety
|
|
6
|
+
# boundary that stops the "general runner" from running arbitrary code.
|
|
7
|
+
class Registry
|
|
8
|
+
Entry = Struct.new(:handler, :max_attempts, :required_role, keyword_init: true)
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
@entries = {}
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# required_role: a role the caller must hold to enqueue this type via the
|
|
15
|
+
# general `enqueue` surface (nil = any caller). The closed allow-list is
|
|
16
|
+
# the primary safety boundary; this is defence-in-depth for destructive
|
|
17
|
+
# types.
|
|
18
|
+
def register(type, handler:, max_attempts: 3, required_role: nil)
|
|
19
|
+
@entries[type.to_s] = Entry.new(handler: handler, max_attempts: max_attempts, required_role: required_role)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def registered?(type)
|
|
23
|
+
@entries.key?(type.to_s)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def lookup(type)
|
|
27
|
+
@entries.fetch(type.to_s) do
|
|
28
|
+
raise Textus::UsageError.new(
|
|
29
|
+
"unregistered job type '#{type}'",
|
|
30
|
+
hint: "register the type in Domain::Jobs::Registry before enqueueing it",
|
|
31
|
+
)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -25,7 +25,7 @@ module Textus
|
|
|
25
25
|
written = written_at(eval.envelope)
|
|
26
26
|
return true if written.nil?
|
|
27
27
|
|
|
28
|
-
now = @now || Textus::Ports::Clock.now
|
|
28
|
+
now = @now || Textus::Ports::Clock.new.now
|
|
29
29
|
return true if now - written <= @seconds
|
|
30
30
|
|
|
31
31
|
@reason = "entry older than #{@seconds}s (written #{written.iso8601})"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
module Textus
|
|
2
|
+
module Domain
|
|
3
|
+
module Policy
|
|
4
|
+
# One publish destination (ADR 0094). Exactly one of:
|
|
5
|
+
# to-target { to:, template:?, inject_boot:? } — render data through a
|
|
6
|
+
# template, or copy verbatim when no template
|
|
7
|
+
# tree-target { tree: } — ADR 0052 subtree mirror
|
|
8
|
+
# Provenance is NOT a publish flag — it lives in the data's `_meta`.
|
|
9
|
+
class PublishTarget
|
|
10
|
+
attr_reader :to, :tree, :template, :inject_boot
|
|
11
|
+
|
|
12
|
+
def initialize(raw)
|
|
13
|
+
if raw.key?("provenance")
|
|
14
|
+
raise Textus::BadManifest.new("publish `provenance:` was removed (ADR 0094): provenance lives in the data's `_meta`")
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
@to = raw["to"]
|
|
18
|
+
@tree = raw["tree"]
|
|
19
|
+
raise Textus::BadManifest.new("a publish target needs exactly one of `to:` or `tree:`") unless @to.nil? ^ @tree.nil?
|
|
20
|
+
|
|
21
|
+
@template = raw["template"]
|
|
22
|
+
@inject_boot = raw["inject_boot"] == true
|
|
23
|
+
return unless tree_target? && (@template || @inject_boot)
|
|
24
|
+
|
|
25
|
+
raise Textus::BadManifest.new("a tree target takes no template/inject_boot (ADR 0094)")
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def to_target? = !@to.nil?
|
|
29
|
+
def tree_target? = !@tree.nil?
|
|
30
|
+
def renders? = to_target? && !@template.nil?
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
module Textus
|
|
2
|
+
module Domain
|
|
3
|
+
module Policy
|
|
4
|
+
# Garbage collection (ADR 0093). A glob-matched rule slot: when an entry
|
|
5
|
+
# ages past `ttl`, retire it. Destructive only — runs on the full
|
|
6
|
+
# `converge` pass, never on a write (ADR 0079's invariant). Orthogonal to
|
|
7
|
+
# production (`source:`): an intake entry can re-pull hourly AND archive
|
|
8
|
+
# after 90 days. `warn`/`refresh` are gone (refresh is implied by an
|
|
9
|
+
# intake source; warn never fired after ADR 0089's pure-read get).
|
|
10
|
+
class Retention
|
|
11
|
+
ACTIONS = %i[drop archive].freeze
|
|
12
|
+
|
|
13
|
+
attr_reader :action
|
|
14
|
+
|
|
15
|
+
def initialize(raw)
|
|
16
|
+
@ttl = raw["ttl"] or
|
|
17
|
+
raise Textus::BadManifest.new("retention requires a 'ttl:'")
|
|
18
|
+
@action = (raw["action"] || "").to_s.to_sym
|
|
19
|
+
return if ACTIONS.include?(@action)
|
|
20
|
+
|
|
21
|
+
raise Textus::BadManifest.new("retention action must be one of #{ACTIONS.join("|")}, got #{raw["action"].inspect}")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def ttl_seconds = Textus::Domain::Duration.seconds(@ttl)
|
|
25
|
+
def destructive? = true
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
module Textus
|
|
2
|
+
module Domain
|
|
3
|
+
module Policy
|
|
4
|
+
# An entry's data-acquisition declaration (ADR 0094). `source:` says HOW the
|
|
5
|
+
# entry's data is acquired; rendering is a publish concern, so there are no
|
|
6
|
+
# template/render fields here. `from` is the acquire + staleness axis:
|
|
7
|
+
# from: project -> derived (internal projection; observable -> rdeps staleness)
|
|
8
|
+
# from: handler -> intake (external fetch; unobservable -> ttl staleness)
|
|
9
|
+
# from: command -> external (out-of-band runner; staleness only, textus never runs it)
|
|
10
|
+
# Materialization is async-only (job-queue model): a write enqueues a
|
|
11
|
+
# `materialize` job, converged by a worker. There is no per-entry write
|
|
12
|
+
# trigger knob.
|
|
13
|
+
class Source
|
|
14
|
+
FROMS = %w[project handler command].freeze
|
|
15
|
+
|
|
16
|
+
attr_reader :from, :handler, :config, :command, :sources
|
|
17
|
+
|
|
18
|
+
def initialize(raw)
|
|
19
|
+
@from = raw["from"].to_s
|
|
20
|
+
unless FROMS.include?(@from)
|
|
21
|
+
raise Textus::BadManifest.new("source.from must be one of #{FROMS.join("|")}, got #{raw["from"].inspect}")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
@ttl = raw["ttl"]
|
|
25
|
+
@projection = {}
|
|
26
|
+
|
|
27
|
+
case @from
|
|
28
|
+
when "project" then init_project(raw)
|
|
29
|
+
when "handler" then init_handler(raw)
|
|
30
|
+
when "command" then init_command(raw)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def kind = @from == "handler" ? :intake : :derived
|
|
35
|
+
def external? = @from == "command"
|
|
36
|
+
def projection? = @from == "project"
|
|
37
|
+
def ttl_seconds = @ttl.nil? ? nil : Textus::Domain::Duration.seconds(@ttl)
|
|
38
|
+
|
|
39
|
+
# Flattened projection accessors (ADR 0094) — read directly off the source
|
|
40
|
+
# block; nil when absent or not a projection source.
|
|
41
|
+
def select = @projection["select"]
|
|
42
|
+
def pluck = @projection["pluck"]
|
|
43
|
+
def sort_by = @projection["sort_by"]
|
|
44
|
+
def transform = @projection["transform"]
|
|
45
|
+
|
|
46
|
+
# The projection spec hash fed to Textus::Projection (string keys, only the
|
|
47
|
+
# present fields). {} when not a projection.
|
|
48
|
+
def projection_spec = @projection.dup
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def init_project(raw)
|
|
53
|
+
%w[select pluck sort_by transform].each { |f| @projection[f] = raw[f] if raw.key?(f) }
|
|
54
|
+
return unless @projection["select"].nil? && @projection["transform"].nil?
|
|
55
|
+
|
|
56
|
+
raise Textus::BadManifest.new("source (from: project) requires `select:` and/or `transform:`")
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def init_handler(raw)
|
|
60
|
+
@handler = raw["handler"] or
|
|
61
|
+
raise Textus::BadManifest.new("source (from: handler) requires a `handler:` field")
|
|
62
|
+
@config = raw["config"] || {}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def init_command(raw)
|
|
66
|
+
@command = raw["command"] or
|
|
67
|
+
raise Textus::BadManifest.new("source (from: command) requires a `command:` field")
|
|
68
|
+
@sources = raw["sources"] || []
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
require "time"
|
|
2
|
+
|
|
3
|
+
module Textus
|
|
4
|
+
module Domain
|
|
5
|
+
module Retention
|
|
6
|
+
# Retention sweep reporter (ADR 0093/0099). Which entries are past their
|
|
7
|
+
# `retention:` ttl and the destructive action that applies. Age basis: file
|
|
8
|
+
# mtime. Only drop/archive. Renamed off the Domain::Retention vs
|
|
9
|
+
# Domain::Policy::Retention collision (ADR 0099).
|
|
10
|
+
class Sweep
|
|
11
|
+
def self.expired?(ttl_seconds:, mtime:, now:)
|
|
12
|
+
return false if ttl_seconds.nil? || mtime.nil?
|
|
13
|
+
|
|
14
|
+
(now - mtime).to_i > ttl_seconds
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def initialize(manifest:, file_stat:, clock:)
|
|
18
|
+
@manifest = manifest
|
|
19
|
+
@file_stat = file_stat
|
|
20
|
+
@clock = clock
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def call(prefix: nil, zone: nil)
|
|
24
|
+
@manifest.data.entries
|
|
25
|
+
.select { |m| matches?(m, prefix: prefix, zone: zone) }
|
|
26
|
+
.flat_map { |m| rows_for(m) }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def matches?(mentry, prefix:, zone:)
|
|
32
|
+
return false if zone && mentry.zone != zone
|
|
33
|
+
return false if prefix && !Textus::Key::Matching.matches_prefix?(
|
|
34
|
+
mentry.key, prefix, nested: mentry.is_a?(Textus::Manifest::Entry::Nested)
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
true
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def rows_for(mentry)
|
|
41
|
+
policy = @manifest.rules.for(mentry.key).retention
|
|
42
|
+
return [] if policy.nil?
|
|
43
|
+
|
|
44
|
+
@manifest.resolver.enumerate(prefix: mentry.key).filter_map do |row|
|
|
45
|
+
path = row[:path]
|
|
46
|
+
next unless @file_stat.exists?(path)
|
|
47
|
+
next unless self.class.expired?(
|
|
48
|
+
ttl_seconds: policy.ttl_seconds, mtime: @file_stat.mtime(path), now: @clock.now,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
{ "key" => row[:key], "path" => path, "action" => policy.action.to_s }
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Textus
|
|
4
|
+
module Domain
|
|
5
|
+
# Retention — "is the entry old enough to retire?" (Q2, ADR 0093/0099).
|
|
6
|
+
# GC dueness, orthogonal to Freshness (content currency). The reporter is
|
|
7
|
+
# Domain::Retention::Sweep; the manifest rule policy is Domain::Policy::Retention.
|
|
8
|
+
module Retention
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|