textus 0.54.2 → 0.55.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +37 -0
- data/README.md +8 -1
- data/SPEC.md +27 -0
- data/docs/architecture/README.md +20 -8
- data/docs/reference/conventions.md +1 -1
- data/exe/textus +1 -1
- data/lib/textus/action/accept.rb +23 -21
- data/lib/textus/action/audit.rb +24 -61
- data/lib/textus/action/base.rb +9 -9
- data/lib/textus/action/blame.rb +18 -36
- data/lib/textus/action/boot.rb +2 -4
- data/lib/textus/action/data_mv.rb +20 -31
- data/lib/textus/action/deps.rb +3 -18
- data/lib/textus/action/doctor.rb +2 -9
- data/lib/textus/action/drain.rb +11 -19
- data/lib/textus/action/enqueue.rb +14 -30
- data/lib/textus/action/get.rb +12 -56
- data/lib/textus/action/ingest.rb +74 -78
- data/lib/textus/action/jobs.rb +6 -15
- data/lib/textus/action/key_delete.rb +6 -16
- data/lib/textus/action/key_delete_prefix.rb +8 -17
- data/lib/textus/action/key_mv.rb +54 -61
- data/lib/textus/action/key_mv_prefix.rb +13 -22
- data/lib/textus/action/list.rb +7 -21
- data/lib/textus/action/propose.rb +16 -26
- data/lib/textus/action/published.rb +3 -5
- data/lib/textus/action/pulse.rb +19 -26
- data/lib/textus/action/put.rb +15 -29
- data/lib/textus/action/rdeps.rb +3 -18
- data/lib/textus/action/reject.rb +12 -21
- data/lib/textus/action/rule_explain.rb +12 -22
- data/lib/textus/action/rule_lint.rb +10 -16
- data/lib/textus/action/rule_list.rb +5 -9
- data/lib/textus/action/schema_envelope.rb +3 -10
- data/lib/textus/action/uid.rb +3 -17
- data/lib/textus/action/where.rb +3 -18
- data/lib/textus/boot.rb +7 -15
- data/lib/textus/contract/arg.rb +10 -0
- data/lib/textus/contract/dsl.rb +88 -0
- data/lib/textus/contract/spec.rb +25 -0
- data/lib/textus/contract.rb +0 -162
- data/lib/textus/doctor/check/audit_log.rb +2 -2
- data/lib/textus/doctor/check/generator_drift.rb +2 -2
- data/lib/textus/doctor/check/orphaned_publish_targets.rb +3 -3
- data/lib/textus/doctor/check/protocol_version.rb +2 -2
- data/lib/textus/doctor/check/raw_asset_paths.rb +1 -1
- data/lib/textus/doctor/check/schema_parse_error.rb +1 -1
- data/lib/textus/doctor/check/schema_violations.rb +2 -2
- data/lib/textus/doctor/check/schemas.rb +1 -1
- data/lib/textus/doctor/check/sentinels.rb +4 -4
- data/lib/textus/doctor/check/templates.rb +1 -1
- data/lib/textus/doctor/check/unowned_schema_fields.rb +1 -1
- data/lib/textus/doctor/check.rb +4 -7
- data/lib/textus/doctor.rb +1 -1
- data/lib/textus/errors.rb +6 -0
- data/lib/textus/format/base.rb +0 -4
- data/lib/textus/format/json.rb +5 -6
- data/lib/textus/format/markdown.rb +5 -6
- data/lib/textus/format/shared.rb +17 -0
- data/lib/textus/format/text.rb +5 -4
- data/lib/textus/format/yaml.rb +30 -6
- data/lib/textus/format.rb +6 -0
- data/lib/textus/gate/auth.rb +2 -17
- data/lib/textus/gate/binder.rb +50 -0
- data/lib/textus/gate.rb +64 -88
- data/lib/textus/init.rb +2 -4
- data/lib/textus/jobs.rb +3 -9
- data/lib/textus/manifest/capabilities.rb +3 -3
- data/lib/textus/manifest/entry/base.rb +1 -1
- data/lib/textus/manifest/entry/publish/subtree_mirror.rb +2 -2
- data/lib/textus/manifest/entry/publish/to_paths.rb +1 -1
- data/lib/textus/manifest/schema/semantics/cross_field.rb +53 -0
- data/lib/textus/manifest/schema/semantics/invariants.rb +125 -0
- data/lib/textus/manifest/schema/semantics/migration.rb +83 -0
- data/lib/textus/manifest/schema/semantics.rb +11 -216
- data/lib/textus/meta.rb +54 -0
- data/lib/textus/{ports → port}/audit_log.rb +44 -4
- data/lib/textus/{ports → port}/build_lock.rb +2 -2
- data/lib/textus/{ports → port}/clock.rb +1 -1
- data/lib/textus/{ports → port}/publisher.rb +5 -5
- data/lib/textus/{ports → port}/sentinel_store.rb +3 -3
- data/lib/textus/{ports → port}/storage/file_stat.rb +1 -1
- data/lib/textus/{ports → port}/storage/file_store.rb +2 -2
- data/lib/textus/port/store.rb +93 -0
- data/lib/textus/{ports → port}/watcher_lock.rb +3 -3
- data/lib/textus/produce/engine.rb +1 -1
- data/lib/textus/schema/tools.rb +11 -7
- data/lib/textus/store/compositor.rb +34 -0
- data/lib/textus/store/container.rb +43 -0
- data/lib/textus/store/cursor.rb +26 -0
- data/lib/textus/store/envelope/reader.rb +43 -0
- data/lib/textus/store/envelope/writer.rb +195 -0
- data/lib/textus/store/geometry.rb +81 -0
- data/lib/textus/store/index/builder.rb +74 -0
- data/lib/textus/store/index/lookup.rb +60 -0
- data/lib/textus/store/jobs/base.rb +13 -0
- data/lib/textus/store/jobs/index.rb +15 -0
- data/lib/textus/store/jobs/materialize.rb +15 -0
- data/lib/textus/store/jobs/plan.rb +11 -0
- data/lib/textus/store/jobs/planner.rb +104 -0
- data/lib/textus/store/jobs/queue.rb +154 -0
- data/lib/textus/store/jobs/registry.rb +19 -0
- data/lib/textus/store/jobs/retention.rb +50 -0
- data/lib/textus/store/jobs/sweep.rb +21 -0
- data/lib/textus/store/jobs/worker.rb +64 -0
- data/lib/textus/store/session.rb +37 -0
- data/lib/textus/store.rb +21 -13
- data/lib/textus/{surfaces → surface}/cli/group/data.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/group/key.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/group/mcp.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/group/rule.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/group/schema.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/group.rb +2 -2
- data/lib/textus/{surfaces → surface}/cli/runner.rb +10 -63
- data/lib/textus/surface/cli/sources.rb +41 -0
- data/lib/textus/{surfaces → surface}/cli/verb/doctor.rb +4 -6
- data/lib/textus/{surfaces → surface}/cli/verb/get.rb +4 -4
- data/lib/textus/{surfaces → surface}/cli/verb/init.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/verb/mcp_serve.rb +3 -3
- data/lib/textus/{surfaces → surface}/cli/verb/put.rb +6 -11
- data/lib/textus/{surfaces → surface}/cli/verb/schema_diff.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/verb/schema_init.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/verb/schema_migrate.rb +1 -1
- data/lib/textus/{surfaces → surface}/cli/verb/watch.rb +2 -2
- data/lib/textus/{surfaces → surface}/cli/verb.rb +3 -8
- data/lib/textus/{surfaces → surface}/cli.rb +1 -1
- data/lib/textus/{surfaces → surface}/mcp/catalog.rb +9 -26
- data/lib/textus/{surfaces → surface}/mcp/errors.rb +1 -1
- data/lib/textus/{surfaces → surface}/mcp/server.rb +5 -5
- data/lib/textus/{surfaces → surface}/mcp.rb +2 -2
- data/lib/textus/surface/projector.rb +27 -0
- data/lib/textus/{surfaces → surface}/role_scope.rb +1 -1
- data/lib/textus/{surfaces → surface}/watcher.rb +8 -8
- data/lib/textus/value/call.rb +30 -0
- data/lib/textus/value/command.rb +16 -0
- data/lib/textus/value/envelope.rb +89 -0
- data/lib/textus/value/etag.rb +39 -0
- data/lib/textus/value/result.rb +26 -0
- data/lib/textus/value/role.rb +38 -0
- data/lib/textus/value/types.rb +13 -0
- data/lib/textus/{uid.rb → value/uid.rb} +9 -7
- data/lib/textus/version.rb +1 -1
- data/lib/textus/workflow/loader.rb +4 -4
- data/lib/textus/workflow/runner.rb +4 -18
- data/lib/textus.rb +9 -10
- metadata +100 -63
- data/lib/textus/action/write_verb.rb +0 -44
- data/lib/textus/call.rb +0 -28
- data/lib/textus/command.rb +0 -41
- data/lib/textus/container.rb +0 -26
- data/lib/textus/contract/around.rb +0 -29
- data/lib/textus/contract/binder.rb +0 -88
- data/lib/textus/contract/resources/build_lock.rb +0 -17
- data/lib/textus/contract/resources/cursor.rb +0 -26
- data/lib/textus/contract/sources.rb +0 -39
- data/lib/textus/contract/view.rb +0 -15
- data/lib/textus/cursor_store.rb +0 -24
- data/lib/textus/envelope/reader.rb +0 -46
- data/lib/textus/envelope/writer.rb +0 -209
- data/lib/textus/envelope.rb +0 -79
- data/lib/textus/etag.rb +0 -36
- data/lib/textus/jobs/base.rb +0 -23
- data/lib/textus/jobs/materialize.rb +0 -20
- data/lib/textus/jobs/plan.rb +0 -9
- data/lib/textus/jobs/planner.rb +0 -101
- data/lib/textus/jobs/retention.rb +0 -48
- data/lib/textus/jobs/sweep.rb +0 -27
- data/lib/textus/jobs/worker.rb +0 -67
- data/lib/textus/layout.rb +0 -91
- data/lib/textus/ports/job_store/job.rb +0 -65
- data/lib/textus/ports/job_store.rb +0 -123
- data/lib/textus/ports/raw_index.rb +0 -61
- data/lib/textus/role.rb +0 -36
- data/lib/textus/session.rb +0 -35
- data/lib/textus/types.rb +0 -15
data/lib/textus/action/deps.rb
CHANGED
|
@@ -3,31 +3,16 @@
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
5
|
class Deps < Base
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
8
6
|
verb :deps
|
|
9
7
|
summary "List the keys a derived entry depends on (its projection/external sources)."
|
|
10
8
|
surfaces :cli, :mcp
|
|
11
9
|
arg :key, String, required: true, positional: true,
|
|
12
10
|
description: "dotted key of the derived entry whose source keys you want"
|
|
13
11
|
|
|
14
|
-
def
|
|
15
|
-
|
|
16
|
-
@key = key
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def call(container:, **)
|
|
20
|
-
entry = container.manifest.data.entries.find { |e| e.key == @key }
|
|
12
|
+
def self.call(container:, key:, **)
|
|
13
|
+
entry = container.manifest.data.entries.find { |e| e.key == key }
|
|
21
14
|
deps = entry&.external? ? Array(entry.source&.sources).compact : []
|
|
22
|
-
{ "key" =>
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def self.new(*args, **kwargs)
|
|
26
|
-
return super(**kwargs) unless args.any?
|
|
27
|
-
|
|
28
|
-
positional = instance_method(:initialize).parameters.slice(:keyreq, :key).map(&:last)
|
|
29
|
-
mapped = positional.zip(args).to_h
|
|
30
|
-
super(**mapped.merge(kwargs))
|
|
15
|
+
Success({ "key" => key, "deps" => deps.uniq })
|
|
31
16
|
end
|
|
32
17
|
end
|
|
33
18
|
end
|
data/lib/textus/action/doctor.rb
CHANGED
|
@@ -3,21 +3,14 @@
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
5
|
class Doctor < Base
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
8
6
|
verb :doctor
|
|
9
7
|
summary "Run health checks on the textus store and report any issues."
|
|
10
8
|
surfaces :cli
|
|
11
9
|
cli "doctor"
|
|
12
10
|
arg :checks, Array, required: false, description: "subset of check names to run (default: all)"
|
|
13
11
|
|
|
14
|
-
def
|
|
15
|
-
|
|
16
|
-
@checks = checks
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def call(container:, call:, **)
|
|
20
|
-
Textus::Doctor.build(container: container, checks: @checks, role: call.role)
|
|
12
|
+
def self.call(container:, call:, checks: nil, **)
|
|
13
|
+
Success(Textus::Doctor.build(container: container, checks: checks, role: call.role))
|
|
21
14
|
end
|
|
22
15
|
end
|
|
23
16
|
end
|
data/lib/textus/action/drain.rb
CHANGED
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
5
|
class Drain < Base
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
8
6
|
verb :drain
|
|
9
7
|
summary "Seed materialize + sweep jobs then drain the queue to empty. " \
|
|
10
8
|
"Identical to one Watcher tick. Use when no watcher is running."
|
|
@@ -12,27 +10,21 @@ module Textus
|
|
|
12
10
|
arg :prefix, String, description: "restrict to keys under this dotted prefix"
|
|
13
11
|
arg :lane, String, description: "restrict to entries in this lane"
|
|
14
12
|
|
|
15
|
-
def
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@lane = lane
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def call(container:, call:)
|
|
22
|
-
queue = Textus::Ports::JobStore.new(root: container.root)
|
|
23
|
-
Textus::Jobs::Planner.seed(
|
|
13
|
+
def self.call(container:, call:, prefix: nil, lane: nil) # rubocop:disable Lint/UnusedMethodArgument
|
|
14
|
+
queue = Textus::Store::Jobs::Queue.new(store: container.job_store)
|
|
15
|
+
Textus::Store::Jobs::Planner.seed(
|
|
24
16
|
container: container,
|
|
25
17
|
queue: queue,
|
|
26
18
|
role: call.role,
|
|
27
19
|
)
|
|
28
|
-
queue.reclaim(now: Textus::
|
|
29
|
-
summary = Textus::Jobs::Worker.for(container:, queue:).drain
|
|
30
|
-
{
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
20
|
+
queue.reclaim(now: Textus::Port::Clock.new.now)
|
|
21
|
+
summary = Textus::Store::Jobs::Worker.for(container:, queue:).drain
|
|
22
|
+
Success({
|
|
23
|
+
"protocol" => Textus::PROTOCOL,
|
|
24
|
+
"ok" => summary.failed.zero?,
|
|
25
|
+
"completed" => summary.completed,
|
|
26
|
+
"failed" => summary.failed,
|
|
27
|
+
})
|
|
36
28
|
end
|
|
37
29
|
end
|
|
38
30
|
end
|
|
@@ -2,9 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
|
-
class Enqueue <
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
5
|
+
class Enqueue < Base
|
|
8
6
|
verb :enqueue
|
|
9
7
|
summary "Push a registered job type onto the convergence queue, to be run by drain/serve."
|
|
10
8
|
surfaces :cli, :mcp
|
|
@@ -14,39 +12,25 @@ module Textus
|
|
|
14
12
|
arg :args, Hash, default: {},
|
|
15
13
|
description: "type-specific arguments (e.g. { key: ... } or { scope: ... })"
|
|
16
14
|
|
|
17
|
-
def
|
|
18
|
-
|
|
19
|
-
@type = type
|
|
20
|
-
@job_args = args
|
|
21
|
-
end
|
|
15
|
+
def self.call(container:, call:, type:, args: {})
|
|
16
|
+
action_class = Textus::Jobs.fetch(type.to_s)
|
|
22
17
|
|
|
23
|
-
def args
|
|
24
|
-
{ type: @type, args: @job_args }
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def call(container:, call:)
|
|
28
|
-
action_class = begin
|
|
29
|
-
Textus::Jobs.fetch(@type.to_s)
|
|
30
|
-
rescue Textus::UsageError
|
|
31
|
-
raise Textus::UsageError.new("unregistered job type '#{@type}'")
|
|
32
|
-
end
|
|
33
18
|
if action_class.const_defined?(:REQUIRED_ROLE) && call.role != action_class::REQUIRED_ROLE
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
details: { "role" => call.role, "required_role" => action_class::REQUIRED_ROLE },
|
|
38
|
-
exit_code: 77,
|
|
39
|
-
)
|
|
19
|
+
return Failure(code: :forbidden,
|
|
20
|
+
message: "role '#{call.role}' is not authorized to enqueue this job type",
|
|
21
|
+
details: { "role" => call.role, "required_role" => action_class::REQUIRED_ROLE })
|
|
40
22
|
end
|
|
41
23
|
|
|
42
|
-
job = Textus::
|
|
43
|
-
type:
|
|
44
|
-
args:
|
|
45
|
-
|
|
24
|
+
job = Textus::Store::Jobs::Queue::Job.new(
|
|
25
|
+
type: type,
|
|
26
|
+
args: args,
|
|
27
|
+
role: call.role,
|
|
46
28
|
max_attempts: 3,
|
|
47
29
|
)
|
|
48
|
-
Textus::
|
|
49
|
-
{ "protocol" => Textus::PROTOCOL, "ok" => true, "id" => job.id }
|
|
30
|
+
Textus::Store::Jobs::Queue.new(store: container.job_store).enqueue(job)
|
|
31
|
+
Success({ "protocol" => Textus::PROTOCOL, "ok" => true, "id" => job.id })
|
|
32
|
+
rescue Textus::UsageError
|
|
33
|
+
Failure(code: :usage_error, message: "unregistered job type '#{type}'")
|
|
50
34
|
end
|
|
51
35
|
end
|
|
52
36
|
end
|
data/lib/textus/action/get.rb
CHANGED
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
5
|
class Get < Base
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
8
6
|
verb :get
|
|
9
7
|
summary "Read one entry - a pure on-disk read annotated with a freshness " \
|
|
10
8
|
"verdict; never ingests (quarantine freshness is drain + hook " \
|
|
@@ -13,64 +11,22 @@ module Textus
|
|
|
13
11
|
surfaces :cli, :mcp
|
|
14
12
|
arg :key, String, required: true, positional: true,
|
|
15
13
|
description: "dotted entry key to read, e.g. 'knowledge.project'"
|
|
16
|
-
view { |v, _i| v
|
|
14
|
+
view(:default) { |v, _i| v&.to_h_for_wire }
|
|
17
15
|
|
|
18
|
-
def
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
end
|
|
16
|
+
def self.call(container:, call:, key:)
|
|
17
|
+
envelope = container.compositor.read(key)
|
|
18
|
+
return Failure(code: :not_found, message: "no entry at #{key}") unless envelope
|
|
22
19
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
@manifest = container.manifest
|
|
27
|
-
@file_store = container.file_store
|
|
28
|
-
@file_stat = file_stat
|
|
29
|
-
annotated_envelope(@key)
|
|
20
|
+
entry = container.manifest.resolver.resolve(key).entry
|
|
21
|
+
file_stat = Textus::Port::Storage::FileStat.new
|
|
22
|
+
Success(envelope.with(freshness: freshness_evaluator(container, call, file_stat).verdict(entry)))
|
|
30
23
|
end
|
|
31
24
|
|
|
32
|
-
def self.
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
super(**mapped.merge(kwargs))
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
private
|
|
41
|
-
|
|
42
|
-
def annotated_envelope(key)
|
|
43
|
-
envelope = read_raw_envelope(key)
|
|
44
|
-
return nil if envelope.nil?
|
|
45
|
-
|
|
46
|
-
entry = @manifest.resolver.resolve(key).entry
|
|
47
|
-
envelope.with(freshness: evaluator.verdict(entry))
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def evaluator
|
|
51
|
-
@evaluator ||= Textus::Core::Freshness::Evaluator.new(
|
|
52
|
-
manifest: @manifest,
|
|
53
|
-
file_stat: @file_stat,
|
|
54
|
-
clock: @call,
|
|
55
|
-
)
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def read_raw_envelope(key)
|
|
59
|
-
res = @manifest.resolver.resolve(key)
|
|
60
|
-
mentry = res.entry
|
|
61
|
-
path = res.path
|
|
62
|
-
return nil unless @file_store.exists?(path)
|
|
63
|
-
|
|
64
|
-
raw = @file_store.read(path)
|
|
65
|
-
parsed = Textus::Format.for(mentry.format).parse(raw, path: path)
|
|
66
|
-
Textus::Envelope.build(
|
|
67
|
-
key: key,
|
|
68
|
-
mentry: mentry,
|
|
69
|
-
path: path,
|
|
70
|
-
meta: parsed["_meta"],
|
|
71
|
-
body: parsed["body"],
|
|
72
|
-
etag: Textus::Etag.for_bytes(raw),
|
|
73
|
-
content: parsed["content"],
|
|
25
|
+
def self.freshness_evaluator(container, call, file_stat)
|
|
26
|
+
Textus::Core::Freshness::Evaluator.new(
|
|
27
|
+
manifest: container.manifest,
|
|
28
|
+
file_stat: file_stat,
|
|
29
|
+
clock: call,
|
|
74
30
|
)
|
|
75
31
|
end
|
|
76
32
|
end
|
data/lib/textus/action/ingest.rb
CHANGED
|
@@ -7,8 +7,6 @@ require "digest"
|
|
|
7
7
|
module Textus
|
|
8
8
|
module Action
|
|
9
9
|
class Ingest < Base
|
|
10
|
-
extend Textus::Contract::DSL
|
|
11
|
-
|
|
12
10
|
verb :ingest
|
|
13
11
|
summary "Capture external source material into the raw lane. Write-once, agent-owned."
|
|
14
12
|
surfaces :cli, :mcp
|
|
@@ -26,120 +24,118 @@ module Textus
|
|
|
26
24
|
CONTENT_HASH_ALGO = "sha256"
|
|
27
25
|
TOMBSTONE_RETAIN = %w[ingested_at].freeze
|
|
28
26
|
|
|
29
|
-
def
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
@slug = slug
|
|
33
|
-
@url = url
|
|
34
|
-
@path = path
|
|
35
|
-
@zone = zone
|
|
36
|
-
@label = label
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
def call(container:, call:)
|
|
40
|
-
validate_inputs!
|
|
27
|
+
def self.call(container:, call:, kind:, slug:, url: nil, path: nil, zone: nil, label: nil, **) # rubocop:disable Metrics/ParameterLists
|
|
28
|
+
validation = validate_inputs(kind:, url:, path:, zone:)
|
|
29
|
+
return validation if validation.is_a?(Dry::Monads::Result::Failure)
|
|
41
30
|
|
|
42
31
|
now = Time.now.utc
|
|
43
|
-
key = derive_key(now)
|
|
32
|
+
key = derive_key(now, kind:, slug:)
|
|
44
33
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
content_hash = compute_content_hash
|
|
50
|
-
writer = Textus::Envelope::Writer.from(container: container, call: call)
|
|
34
|
+
content_hash = compute_content_hash(kind:, url:, path:)
|
|
35
|
+
writer = Textus::Store::Envelope::Writer.from(container: container, call: call)
|
|
51
36
|
mentry = container.manifest.resolver.resolve(key).entry
|
|
52
37
|
ts = now.iso8601
|
|
53
|
-
structured = build_structured(ts, container, now, content_hash)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
38
|
+
structured = build_structured(ts, container, now, content_hash, kind:, url:, path:, label:, zone:)
|
|
39
|
+
|
|
40
|
+
store = container.job_store
|
|
41
|
+
index = Textus::Store::Index::Lookup.new(store: store)
|
|
42
|
+
duplicate_key = find_duplicate(index, content_hash, kind:, url:)
|
|
43
|
+
|
|
44
|
+
result = if duplicate_key && duplicate_key != key
|
|
45
|
+
supersede_entry(duplicate_key, key, structured, container, call, store: store, kind:, zone:)
|
|
46
|
+
else
|
|
47
|
+
env = write_raw_entry(key, structured, mentry, writer)
|
|
48
|
+
rebuild_index(container, store)
|
|
49
|
+
env
|
|
50
|
+
end
|
|
51
|
+
Success(result)
|
|
65
52
|
end
|
|
66
53
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
raise Textus::UsageError.new(
|
|
72
|
-
"ingest kind must be one of #{SOURCE_KINDS.join("|")}, got #{@kind.inspect}",
|
|
73
|
-
)
|
|
54
|
+
def self.validate_inputs(kind:, url:, path:, zone:)
|
|
55
|
+
unless SOURCE_KINDS.include?(kind)
|
|
56
|
+
return Failure(code: :usage_error,
|
|
57
|
+
message: "ingest kind must be one of #{SOURCE_KINDS.join("|")}, got #{kind.inspect}")
|
|
74
58
|
end
|
|
75
|
-
case
|
|
59
|
+
case kind
|
|
76
60
|
when "url"
|
|
77
|
-
|
|
61
|
+
return Failure(code: :usage_error, message: "ingest url requires --url") unless url
|
|
78
62
|
when "file"
|
|
79
|
-
|
|
63
|
+
return Failure(code: :usage_error, message: "ingest file requires --path") unless path
|
|
80
64
|
when "asset"
|
|
81
|
-
|
|
82
|
-
|
|
65
|
+
return Failure(code: :usage_error, message: "ingest asset requires --path") unless path
|
|
66
|
+
return Failure(code: :usage_error, message: "ingest asset requires --zone") unless zone
|
|
83
67
|
end
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Key derivation for Gate pre-dispatch auth. Must match the runtime
|
|
72
|
+
# derivation in #call so the same key is checked by auth and used by
|
|
73
|
+
# the action body.
|
|
74
|
+
def self.dispatch_key(kind:, slug:, **)
|
|
75
|
+
derive_key(Time.now.utc, kind:, slug:)
|
|
84
76
|
end
|
|
85
77
|
|
|
86
|
-
def derive_key(now)
|
|
78
|
+
def self.derive_key(now, kind:, slug:)
|
|
87
79
|
date = now.strftime("%Y.%m.%d")
|
|
88
|
-
"raw.#{date}.#{
|
|
80
|
+
"raw.#{date}.#{kind}-#{slug}"
|
|
89
81
|
end
|
|
90
82
|
|
|
91
|
-
def compute_content_hash
|
|
83
|
+
def self.compute_content_hash(kind:, url:, path:)
|
|
92
84
|
digest = Digest::SHA256.new
|
|
93
|
-
case
|
|
85
|
+
case kind
|
|
94
86
|
when "url"
|
|
95
|
-
digest.update(
|
|
87
|
+
digest.update(url)
|
|
96
88
|
when "file", "asset"
|
|
97
|
-
digest.file(
|
|
89
|
+
digest.file(path)
|
|
98
90
|
end
|
|
99
91
|
"#{CONTENT_HASH_ALGO}:#{digest.hexdigest}"
|
|
100
92
|
end
|
|
101
93
|
|
|
102
|
-
def build_structured(timestamp, container, now, content_hash)
|
|
94
|
+
def self.build_structured(timestamp, container, now, content_hash, kind:, url:, path:, label:, zone:) # rubocop:disable Metrics/ParameterLists
|
|
103
95
|
base = { "ingested_at" => timestamp, "content_hash" => content_hash }
|
|
104
|
-
case
|
|
96
|
+
case kind
|
|
105
97
|
when "url"
|
|
106
|
-
base.merge("source" => { "kind" => "url", "url" =>
|
|
98
|
+
base.merge("source" => { "kind" => "url", "url" => url, "label" => label || url },
|
|
107
99
|
"body" => nil)
|
|
108
100
|
when "file"
|
|
109
|
-
body_content = File.read(
|
|
110
|
-
base.merge("source" => { "kind" => "file", "path" =>
|
|
111
|
-
"label" =>
|
|
101
|
+
body_content = File.read(path)
|
|
102
|
+
base.merge("source" => { "kind" => "file", "path" => path,
|
|
103
|
+
"label" => label || File.basename(path) },
|
|
112
104
|
"body" => body_content)
|
|
113
105
|
when "asset"
|
|
114
|
-
asset_rel = copy_asset_file(container, now)
|
|
106
|
+
asset_rel = copy_asset_file(container, now, path:, zone:)
|
|
115
107
|
base.merge("source" => { "kind" => "asset",
|
|
116
|
-
"label" =>
|
|
108
|
+
"label" => label || File.basename(path) },
|
|
117
109
|
"asset" => asset_rel,
|
|
118
110
|
"body" => nil)
|
|
119
111
|
end
|
|
120
112
|
end
|
|
121
113
|
|
|
122
|
-
def write_raw_entry(key, structured, mentry, writer)
|
|
114
|
+
def self.write_raw_entry(key, structured, mentry, writer)
|
|
123
115
|
writer.put(key, mentry: mentry,
|
|
124
|
-
payload: Textus::Envelope::Writer::Payload.new(
|
|
116
|
+
payload: Textus::Store::Envelope::Writer::Payload.new(
|
|
125
117
|
meta: nil, body: nil, content: structured,
|
|
126
118
|
))
|
|
127
119
|
end
|
|
128
120
|
|
|
129
|
-
def find_duplicate(index, content_hash)
|
|
121
|
+
def self.find_duplicate(index, content_hash, kind:, url:)
|
|
130
122
|
dup = index.find_by_hash(content_hash)
|
|
131
123
|
return dup if dup
|
|
132
124
|
|
|
133
|
-
return unless
|
|
125
|
+
return unless kind == "url"
|
|
126
|
+
|
|
127
|
+
index.find_by_url(url)
|
|
128
|
+
end
|
|
134
129
|
|
|
135
|
-
|
|
130
|
+
def self.rebuild_index(container, store)
|
|
131
|
+
Textus::Store::Index::Builder.new(store: store).rebuild!(resolver: container.manifest.resolver)
|
|
136
132
|
end
|
|
137
133
|
|
|
138
|
-
def supersede_entry(old_key, new_key, structured, container, call,
|
|
134
|
+
def self.supersede_entry(old_key, new_key, structured, container, call, store:, kind:, zone:) # rubocop:disable Metrics/ParameterLists
|
|
139
135
|
old_mentry = container.manifest.resolver.resolve(old_key).entry
|
|
140
|
-
writer = Textus::Envelope::Writer.from(container: container, call: call)
|
|
136
|
+
writer = Textus::Store::Envelope::Writer.from(container: container, call: call)
|
|
141
137
|
|
|
142
|
-
reader = Textus::Envelope::Reader.from(container: container)
|
|
138
|
+
reader = Textus::Store::Envelope::Reader.from(container: container)
|
|
143
139
|
old_env = reader.read(old_key)
|
|
144
140
|
old_content = old_env&.content || {}
|
|
145
141
|
tombstone = {}
|
|
@@ -151,27 +147,27 @@ module Textus
|
|
|
151
147
|
tombstone["superseded_by"] = new_key
|
|
152
148
|
|
|
153
149
|
writer.put(old_key, mentry: old_mentry,
|
|
154
|
-
payload: Textus::Envelope::Writer::Payload.new(
|
|
150
|
+
payload: Textus::Store::Envelope::Writer::Payload.new(
|
|
155
151
|
meta: nil, body: nil, content: tombstone,
|
|
156
152
|
))
|
|
157
153
|
|
|
158
154
|
structured["supersedes"] = old_key
|
|
159
155
|
env = write_raw_entry(new_key, structured, container.manifest.resolver.resolve(new_key).entry, writer)
|
|
160
156
|
|
|
161
|
-
move_asset_file(container, old_content["asset"]) if
|
|
157
|
+
move_asset_file(container, old_content["asset"], zone:) if kind == "asset" && old_content["asset"]
|
|
162
158
|
|
|
163
|
-
|
|
159
|
+
rebuild_index(container, store)
|
|
164
160
|
env
|
|
165
161
|
end
|
|
166
162
|
|
|
167
|
-
def move_asset_file(container, old_asset_rel)
|
|
163
|
+
def self.move_asset_file(container, old_asset_rel, zone:)
|
|
168
164
|
old_path = File.join(container.root, "assets", old_asset_rel)
|
|
169
165
|
return unless File.exist?(old_path)
|
|
170
166
|
|
|
171
167
|
now = Time.now.utc
|
|
172
168
|
date_path = now.strftime("%Y/%m/%d")
|
|
173
169
|
filename = File.basename(old_path)
|
|
174
|
-
new_dir = File.join(container.root, "assets", "raw", date_path,
|
|
170
|
+
new_dir = File.join(container.root, "assets", "raw", date_path, zone)
|
|
175
171
|
new_path = File.join(new_dir, filename)
|
|
176
172
|
|
|
177
173
|
return if old_path == new_path
|
|
@@ -182,17 +178,17 @@ module Textus
|
|
|
182
178
|
warn "[textus ingest] could not move asset #{old_asset_rel}: #{e.message}"
|
|
183
179
|
end
|
|
184
180
|
|
|
185
|
-
def copy_asset_file(container, now)
|
|
181
|
+
def self.copy_asset_file(container, now, path:, zone:)
|
|
186
182
|
date_path = now.strftime("%Y/%m/%d")
|
|
187
|
-
filename = File.basename(
|
|
188
|
-
assets_dir = File.join(container.root, "assets", "raw", date_path,
|
|
183
|
+
filename = File.basename(path)
|
|
184
|
+
assets_dir = File.join(container.root, "assets", "raw", date_path, zone)
|
|
189
185
|
FileUtils.mkdir_p(assets_dir)
|
|
190
|
-
FileUtils.cp(
|
|
186
|
+
FileUtils.cp(path, File.join(assets_dir, filename))
|
|
191
187
|
create_gitignore_sentinel(container)
|
|
192
|
-
"raw/#{date_path}/#{
|
|
188
|
+
"raw/#{date_path}/#{zone}/#{filename}"
|
|
193
189
|
end
|
|
194
190
|
|
|
195
|
-
def create_gitignore_sentinel(container)
|
|
191
|
+
def self.create_gitignore_sentinel(container)
|
|
196
192
|
assets_root = File.join(container.root, "assets")
|
|
197
193
|
FileUtils.mkdir_p(assets_root)
|
|
198
194
|
sentinel = File.join(assets_root, ".gitignore")
|
data/lib/textus/action/jobs.rb
CHANGED
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
5
|
class Jobs < Base
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
8
6
|
verb :jobs
|
|
9
7
|
summary "List queued jobs by state; retry a dead-lettered job or purge a state."
|
|
10
8
|
surfaces :cli, :mcp
|
|
@@ -13,23 +11,16 @@ module Textus
|
|
|
13
11
|
arg :action, String, default: nil, description: "retry|purge (optional)"
|
|
14
12
|
arg :job_id, String, default: nil, description: "job id (required for action=retry)"
|
|
15
13
|
|
|
16
|
-
def
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
@action = action
|
|
20
|
-
@job_id = job_id
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
def call(container:, **)
|
|
24
|
-
queue = Textus::Ports::JobStore.new(root: container.root)
|
|
25
|
-
case @action
|
|
14
|
+
def self.call(container:, call:, state: "ready", action: nil, job_id: nil) # rubocop:disable Lint/UnusedMethodArgument
|
|
15
|
+
queue = Textus::Store::Jobs::Queue.new(store: container.job_store)
|
|
16
|
+
case action
|
|
26
17
|
when "retry"
|
|
27
|
-
queue.retry_failed(
|
|
18
|
+
queue.retry_failed(job_id)
|
|
28
19
|
when "purge"
|
|
29
|
-
queue.purge(
|
|
20
|
+
queue.purge(state)
|
|
30
21
|
end
|
|
31
22
|
|
|
32
|
-
{ "protocol" => Textus::PROTOCOL, "ok" => true, "state" =>
|
|
23
|
+
Success({ "protocol" => Textus::PROTOCOL, "ok" => true, "state" => state, "jobs" => queue.list(state) })
|
|
33
24
|
end
|
|
34
25
|
end
|
|
35
26
|
end
|
|
@@ -2,9 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
|
-
class KeyDelete <
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
5
|
+
class KeyDelete < Base
|
|
8
6
|
verb :key_delete
|
|
9
7
|
summary "Delete one entry by key. Single-key, lower blast radius than key_delete_prefix; " \
|
|
10
8
|
"guarded by an optional optimistic-concurrency etag. Returns {ok, key, deleted}."
|
|
@@ -15,21 +13,13 @@ module Textus
|
|
|
15
13
|
arg :if_etag, String,
|
|
16
14
|
description: "optimistic-concurrency guard: the etag you last read; the delete is rejected if the entry changed since"
|
|
17
15
|
|
|
18
|
-
def
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
@if_etag = if_etag
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def call(container:, call:)
|
|
25
|
-
run_with_cascade(@key, container:, call:) do
|
|
26
|
-
Textus::Manifest::Data.validate_key!(@key)
|
|
27
|
-
mentry = container.manifest.resolver.resolve(@key).entry
|
|
16
|
+
def self.call(container:, call:, key:, if_etag: nil)
|
|
17
|
+
Textus::Manifest::Data.validate_key!(key)
|
|
18
|
+
mentry = container.manifest.resolver.resolve(key).entry
|
|
28
19
|
|
|
29
|
-
|
|
20
|
+
container.compositor.delete(key, mentry: mentry, if_etag: if_etag, call: call)
|
|
30
21
|
|
|
31
|
-
|
|
32
|
-
end
|
|
22
|
+
Success({ "protocol" => Textus::PROTOCOL, "ok" => true, "key" => key, "deleted" => true })
|
|
33
23
|
end
|
|
34
24
|
end
|
|
35
25
|
end
|
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module Textus
|
|
4
4
|
module Action
|
|
5
5
|
class KeyDeletePrefix < Base
|
|
6
|
-
extend Textus::Contract::DSL
|
|
7
|
-
|
|
8
6
|
verb :key_delete_prefix
|
|
9
7
|
summary "Bulk-delete every leaf key under prefix."
|
|
10
8
|
surfaces :cli, :mcp
|
|
@@ -16,28 +14,21 @@ module Textus
|
|
|
16
14
|
"defaults to false, so omitting it deletes immediately"
|
|
17
15
|
view { |v, _i| v.to_h }
|
|
18
16
|
|
|
19
|
-
def
|
|
20
|
-
|
|
21
|
-
@prefix = prefix
|
|
22
|
-
@dry_run = dry_run
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def call(container:, call:)
|
|
26
|
-
raise UsageError.new("prefix required") if @prefix.nil? || @prefix.empty?
|
|
17
|
+
def self.call(container:, call:, prefix:, dry_run: false)
|
|
18
|
+
return Failure(code: :usage_error, message: "prefix required") if prefix.nil? || prefix.empty?
|
|
27
19
|
|
|
28
|
-
leaves = Textus::Action::List.
|
|
29
|
-
.map { |row| row.is_a?(Hash) ? (row["key"] || row[:key]) : row }
|
|
20
|
+
leaves = Textus::Action::List.leaf_keys(container: container, prefix: prefix)
|
|
30
21
|
|
|
31
|
-
warnings = leaves.empty? ? ["no keys under #{
|
|
22
|
+
warnings = leaves.empty? ? ["no keys under #{prefix}"] : []
|
|
32
23
|
steps = leaves.map { |key| { "op" => "delete", "key" => key } }
|
|
33
24
|
|
|
34
|
-
plan = Textus::Jobs::Plan.new(steps: steps, warnings: warnings)
|
|
35
|
-
return plan if
|
|
25
|
+
plan = Textus::Store::Jobs::Plan.new(steps: steps, warnings: warnings)
|
|
26
|
+
return Success(plan) if dry_run
|
|
36
27
|
|
|
37
28
|
steps.each do |step|
|
|
38
|
-
Textus::Action::KeyDelete.
|
|
29
|
+
Value::Result.unwrap(Textus::Action::KeyDelete.call(container: container, call: call, key: step["key"]))
|
|
39
30
|
end
|
|
40
|
-
plan
|
|
31
|
+
Success(plan)
|
|
41
32
|
end
|
|
42
33
|
end
|
|
43
34
|
end
|