textus 0.51.0 → 0.53.0

data/lib/textus/{produce → pipeline}/acquire/intake.rb

@@ -1,21 +1,21 @@
 require "timeout"
 
 module Textus
-  module Produce
+  module Pipeline
     module Acquire
       # Internal ingest executor for one machine-zone intake entry. No longer a
       # public verb (ADR 0079 collapsed the `fetch` surface): used by the
-      # `reconcile` sweep and `textus hook run` only — ingest is system-pushed
+      # converge sweep (drain/serve) and `textus hook run` only — ingest is system-pushed
       # (ADR 0089 removed the read-through that once also drove it).
       class Intake
-        FETCH_TIMEOUT_SECONDS = Textus::Produce::Acquire::Handler::FETCH_TIMEOUT_SECONDS
+        FETCH_TIMEOUT_SECONDS = Textus::Pipeline::Acquire::Handler::FETCH_TIMEOUT_SECONDS
 
         def initialize(container:, call:)
           @container    = container
           @call         = call
           @manifest     = container.manifest
           @schemas      = container.schemas
-          @rpc          = container.rpc
+          @steps          = container.steps
         end
 
         # call(key) is the primary entry; run is kept as an alias for
@@ -61,10 +61,6 @@ module Textus
 
         private
 
-        def fetch_events
-          @fetch_events ||= Textus::Produce::Events.from(container: @container, call: @call)
-        end
-
         # ADR 0079: a per-rule fetch_timeout_seconds override was an accepted loss
         # in the fetch:/retention: → lifecycle: collapse; the constant ceiling
         # applies to every intake.
@@ -72,36 +68,35 @@ module Textus
           FETCH_TIMEOUT_SECONDS
         end
 
+        def steps_ctx
+          @steps_ctx ||= Textus::Step::Context.for(container: @container, call: @call)
+        end
+
         def fetch_with_events(key, mentry, remaining)
-          fetch_events.started(key)
+          @steps.publish(:entry_fetch_started, ctx: steps_ctx, key: key, mode: :sync)
           call_intake(key, mentry, remaining)
         end
 
         def call_intake(key, mentry, remaining)
-          Textus::Produce::Acquire::Handler.invoke(
+          Textus::Pipeline::Acquire::Handler.invoke(
             caps: @container, handler: mentry.handler,
             config: mentry.config,
             args: { trigger_key: key, leaf_segments: remaining || [] },
             label: "intake", timeout: fetch_timeout_for(key)
           )
         rescue Textus::Error => e
-          fetch_events.failed(key, e)
+          @steps.publish(:entry_fetch_failed, ctx: steps_ctx, key: key,
+                                              error_class: e.class.name, error_message: e.message)
           raise
         rescue StandardError => e
-          fetch_events.failed(key, e)
+          @steps.publish(:entry_fetch_failed, ctx: steps_ctx, key: key,
+                                              error_class: e.class.name, error_message: e.message)
           raise UsageError.new("intake '#{mentry.handler}' raised: #{e.class}: #{e.message}")
         end
 
         def persist_and_notify(key, mentry, result, before_etag)
           normalized = self.class.normalize_action_result(result, format: mentry.format)
-          Textus::Domain::Policy::GuardFactory.new(
-            manifest: @manifest, schemas: @schemas,
-          ).for(:reconcile, key).check!(
-            Textus::Domain::Policy::Evaluation.new(
-              actor: @call.role, transition: :reconcile, origin: nil,
-              target: key, envelope: nil, manifest: @manifest
-            ),
-          )
+          auth.check_action!(action: :converge, actor: @call.role, key: key)
           envelope = writer.put(
             key,
             mentry: mentry,
@@ -110,7 +105,10 @@ module Textus
             ),
           )
           change = detect_change(before_etag, envelope)
-          fetch_events.fetched(key, envelope, change)
+          unless change == :unchanged
+            @steps.publish(:entry_fetched, ctx: steps_ctx, key: key,
+                                           envelope: envelope, change: change)
+          end
           envelope
         end
 
@@ -124,6 +122,10 @@ module Textus
         def writer
           @writer ||= Textus::Envelope::IO::Writer.from(container: @container, call: @call)
         end
+
+        def auth
+          @auth ||= Textus::Gate::Auth.new(@container)
+        end
       end
     end
   end

data/lib/textus/{produce → pipeline}/acquire/projection.rb

@@ -1,7 +1,7 @@
 require "fileutils"
 
 module Textus
-  module Produce
+  module Pipeline
     module Acquire
       # Builds an entry's DATA artifact (ADR 0094) by running the projection
       # pipeline; rendering is a publish concern. External entries are NOT built
@@ -34,13 +34,13 @@ module Textus
           end
         end
 
-        Deps = Data.define(:manifest, :reader, :lister, :rpc, :transform_context)
+        Deps = Data.define(:manifest, :reader, :lister, :steps, :transform_context)
 
         def self.renderers
           @renderers ||= {
-            "text" => Produce::Acquire::Serializer::Text,
-            "json" => Produce::Acquire::Serializer::Json,
-            "yaml" => Produce::Acquire::Serializer::Yaml,
+            "text" => Textus::Pipeline::Acquire::Serializer::Text,
+            "json" => Textus::Pipeline::Acquire::Serializer::Json,
+            "yaml" => Textus::Pipeline::Acquire::Serializer::Yaml,
           }
         end
 
@@ -49,14 +49,16 @@ module Textus
           @call       = call
           @manifest   = container.manifest
           @file_store = container.file_store
-          @rpc        = container.rpc
+          @steps      = container.steps
           @root       = container.root
         end
 
         # Runs the projection pipeline for `mentry` and returns the on-disk
         # target_path string.
         def run(mentry)
-          reader   = Textus::Read::Get.new(container: @container, call: @call)
+          reader = lambda do |key|
+            Textus::Action::Get.new(key: key).call(container: @container, call: @call)
+          end
           # Projections must be able to read source data from any nested entry,
           # including keyless (publish_tree) ones like knowledge.decisions.
           # The `include_keyless: true` option makes the resolver walk those dirs
@@ -64,15 +66,15 @@ module Textus
           resolver = @manifest.resolver
           lister = lambda do |prefix:|
             resolver.enumerate(prefix: prefix, include_keyless: true)
-                    .map { |row| { "key" => row[:key], "zone" => row[:manifest_entry].zone, "path" => row[:path] } }
+                    .map { |row| { "key" => row[:key], "lane" => row[:manifest_entry].lane, "path" => row[:path] } }
           end
           self.class.pipeline_run(
             mentry: mentry,
             deps: Deps.new(
               manifest: @manifest,
-              reader: reader.method(:call),
+              reader: reader,
               lister: lister,
-              rpc: @rpc,
+              steps: @steps,
               transform_context: @container,
             ),
           )
@@ -95,7 +97,7 @@ module Textus
               reader: deps.reader,
               spec: mentry.source.projection_spec,
               lister: deps.lister,
-              rpc: deps.rpc,
+              steps: deps.steps,
               transform_context: deps.transform_context,
             ).run
 

data/lib/textus/{produce → pipeline}/acquire/serializer/json.rb

@@ -1,13 +1,13 @@
 require "json"
 
 module Textus
-  module Produce
+  module Pipeline
     module Acquire
       class Serializer
         class Json < Serializer
           def call(mentry:, data:)
             content = default_shape(mentry, data)
-            final   = Produce::Acquire::Projection::InjectMeta.call(content, mentry)
+            final   = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
             Entry.for_format("json").serialize(meta: {}, body: "", content: final)
           end
 

data/lib/textus/{produce → pipeline}/acquire/serializer/text.rb

@@ -1,5 +1,5 @@
 module Textus
-  module Produce
+  module Pipeline
     module Acquire
       class Serializer
         class Text < Serializer

data/lib/textus/{produce → pipeline}/acquire/serializer/yaml.rb

@@ -1,13 +1,13 @@
 require "yaml"
 
 module Textus
-  module Produce
+  module Pipeline
     module Acquire
       class Serializer
         class Yaml < Serializer
           def call(mentry:, data:)
             content = default_shape(mentry, data)
-            final   = Produce::Acquire::Projection::InjectMeta.call(content, mentry)
+            final   = Textus::Pipeline::Acquire::Projection::InjectMeta.call(content, mentry)
             Entry.for_format("yaml").serialize(meta: {}, body: "", content: final)
           end
 

data/lib/textus/{produce → pipeline}/acquire/serializer.rb

@@ -1,5 +1,5 @@
 module Textus
-  module Produce
+  module Pipeline
     module Acquire
       # Abstract base for output serializers. Each concrete serializer owns
       # producing the bytes for one manifest format (json/yaml/text).

data/lib/textus/{produce → pipeline}/engine.rb

@@ -1,5 +1,5 @@
 module Textus
-  module Produce
+  module Pipeline
     # The single convergence engine (ADR 0093/0094). "Make these machine entries
     # current from upstream." Acquire is per-`from`; publish is one uniform
     # `publish_via` entry point for all kinds (ADR 0094):
@@ -8,17 +8,16 @@ module Textus
     #   derived (from: command)  -> skip the build; publish_via publishes
     #                               existing store bytes via mode resolution
     #                               (None when no targets -> skipped)
-    # Runs as the reconcile build actor (self-elevating); the passed `call`
+    # Runs as the converge build actor (self-elevating); the passed `call`
     # supplies only correlation_id/dry_run. Callers choose the key set: the
-    # write subscriber passes rdeps ∩ derived; reconcile passes
+    # write subscriber passes rdeps ∩ derived; the converge pass passes
     # all-derived + stale-intake.
     class Engine
-      # Locked + failure-isolated convergence — the shared entry point for the
-      # write trigger (ADR 0093). Both the sync path (inline, in the subscriber)
-      # and the async path (AsyncRunner) call this. A held lock is a soft miss
-      # (an in-flight build/reconcile already produces fresh output); any other
-      # error is republished as :produce_failed and never raised at the
-      # writer (ADR 0087 §5 failure isolation, preserved).
+      # Locked + failure-isolated convergence — the entry point worker handlers
+      # call to materialize a key set (ADR 0093 / job-queue model). A held lock
+      # is a soft miss (an in-flight build/converge already produces fresh
+      # output); any other error is republished as :produce_failed and never
+      # raised at the caller (ADR 0087 §5 failure isolation, preserved).
       def self.converge(container:, call:, keys:)
         Textus::Ports::BuildLock.with(root: container.root) do
           new(container: container, call: call).call(keys: keys)
@@ -26,9 +25,9 @@ module Textus
       rescue Textus::BuildInProgress
         nil
       rescue Textus::Error => e
-        container.events.publish(
+        container.steps.publish(
           :produce_failed,
-          ctx: Textus::Hooks::Context.for(container: container, call: call),
+          ctx: Textus::Step::Context.for(container: container, call: call),
           keys: keys, error: e.message
         )
       end
@@ -63,7 +62,7 @@ module Textus
         entry = @manifest.resolver.resolve(key).entry
 
         if entry.intake?
-          Textus::Produce::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
+          Textus::Pipeline::Acquire::Intake.new(container: @container, call: build_call).run(key) # acquire: re-pull
           entry.publish_via(context)                                                # emit any targets
           out[:produced] << key                                                     # a fetch is production
         else
@@ -73,10 +72,10 @@ module Textus
       end
 
       def build_actor_call
-        build_role = @manifest.policy.actor_for("reconcile") or
+        build_role = @manifest.policy.actor_for("converge") or
           raise Textus::UsageError.new(
-            "no role holds the 'reconcile' capability",
-            hint: "declare a role with `can: [reconcile]` in .textus/manifest.yaml",
+            "no role holds the 'converge' capability",
+            hint: "declare a role with `can: [converge]` in .textus/manifest.yaml",
           )
         Textus::Call.build(
           role: build_role,
@@ -88,56 +87,11 @@ module Textus
       def build_context(call)
         Textus::Manifest::Entry::Base::PublishContext.new(
           container: @container, call: call,
-          reader: Textus::Read::Get.new(container: @container, call: call)
+          reader: lambda { |key|
+            Textus::Action::Get.new(key: key).call(container: @container, call: call)
+          }
         )
       end
-
-      # In-process deferral for the async write trigger (ADR 0087/0093).
-      # Spawns a tracked thread that runs Produce.converge after the write
-      # returns; a one-time at_exit joins
-      # all pending threads so a short-lived CLI process cannot exit before an
-      # async rebuild completes. The write itself never blocks.
-      module AsyncRunner
-        @mutex   = Mutex.new
-        @threads = []
-        @hooked  = false
-
-        class << self
-          def enqueue(container:, call:, keys:)
-            thread = Thread.new { Textus::Produce::Engine.converge(container: container, call: call, keys: keys) }
-            track(thread)
-            thread
-          end
-
-          # Block until every spawned async rebuild has finished. Idempotent;
-          # safe to call from at_exit and directly from tests.
-          def drain
-            pending = @mutex.synchronize { @threads.dup }
-            pending.each(&:join)
-            @mutex.synchronize { @threads.delete_if { |t| !t.alive? } }
-            nil
-          end
-
-          private
-
-          def track(thread)
-            @mutex.synchronize do
-              @threads.delete_if { |t| !t.alive? }
-              @threads << thread
-              install_drain_hook
-            end
-          end
-
-          # Register the join-before-exit hook exactly once. Guarded by the
-          # caller holding @mutex.
-          def install_drain_hook
-            return if @hooked
-
-            @hooked = true
-            at_exit { drain }
-          end
-        end
-      end
     end
   end
 end

data/lib/textus/{produce → pipeline}/render.rb

@@ -1,5 +1,7 @@
+require "mustache"
+
 module Textus
-  module Produce
+  module Pipeline
     # Renders an entry's stored DATA into the bytes for one publish target
     # (ADR 0094). Relocates the Mustache logic that used to live in the
     # build-time Markdown renderer. Provenance is NOT added here — it lives in

data/lib/textus/ports/audit_log.rb

@@ -78,16 +78,42 @@ module Textus
       def verify_integrity
         return [] unless File.exist?(@path)
 
-        out = []
-        File.foreach(@path).with_index(1) do |line, lineno|
-          violation = check_line(line.chomp, lineno)
-          out << violation if violation
+        [].tap do |out|
+          iterate_with_prev_seq do |line, lineno, prev_seq|
+            check_line_integrity(line, lineno, prev_seq, out)
+          end
         end
-        out
       end
 
       private
 
+      def iterate_with_prev_seq
+        prev_seq = nil
+        File.foreach(@path).with_index(1) do |line, lineno|
+          yield line.chomp, lineno, prev_seq
+          parsed = parse_row(line.chomp)
+          prev_seq = parsed["seq"] if parsed&.dig("seq").is_a?(Integer)
+        end
+      end
+
+      def check_line_integrity(line, lineno, prev_seq, out)
+        violation = check_line(line, lineno)
+        if violation
+          out << violation
+          return
+        end
+
+        parsed = parse_row(line)
+        return unless parsed && (seq = parsed["seq"]).is_a?(Integer)
+        return unless prev_seq && seq != prev_seq + 1
+
+        out << {
+          "lineno" => lineno,
+          "reason" => "seq_gap",
+          "detail" => "expected #{prev_seq + 1}, got #{seq}",
+        }
+      end
+
       def rotated(n)
         File.join(Textus::Layout.audit_dir(@root), "audit.log.#{n}")
       end

data/lib/textus/ports/audit_subscriber.rb

@@ -3,9 +3,9 @@
 module Textus
   module Ports
     # Writes an "event_error" audit row when a user hook raises during
-    # Hooks::EventBus publish. Attached at Store boot.
+    # Step::EventBus publish. Attached at Store boot.
     #
-    # Integration: uses Hooks::EventBus#on_error callback (chosen over a
+    # Integration: uses Step::EventBus#on_error callback (chosen over a
     # synthetic :hook_error event because the bus already owns the
     # rescue and the failure is a bus-internal concern, not a domain
     # event subscribers should be able to filter by key glob).
@@ -19,8 +19,8 @@ module Textus
         @audit_log = audit_log
       end
 
-      def attach(bus)
-        bus.on_error do |event:, hook:, key:, kwargs:, error:|
+      def attach(registry)
+        registry.on_error do |event:, hook:, key:, kwargs:, error:|
           record_error(event: event, hook: hook, key: key, kwargs: kwargs, error: error)
         end
         self

data/lib/textus/ports/build_lock.rb

@@ -5,7 +5,7 @@ require "time"
 module Textus
   module Ports
     # Cross-process build lock: a pid/host-stamped lockfile under the store root
-    # that serializes reconcile's produce/sweep. An instantiable class — it holds
+    # that serializes converge's produce/sweep. An instantiable class — it holds
     # the root and lock state; `self.with(root:)` is a convenience that constructs
     # one and runs the block under the held lock. It already satisfied ADR 0109's
     # single-shape rule (every port is an instantiable class) before that ADR's

data/lib/textus/ports/queue/job.rb

@@ -0,0 +1,65 @@
+require "digest"
+require "json"
+
+module Textus
+  module Ports
+    class Queue
+      # A unit of deferred work. Pure data. The id is `<type>:<digest>` where the
+      # digest is over the args with sorted keys, so logically-identical enqueues
+      # collide on the same id — which is how Queue dedups (file already exists).
+      # At-least-once delivery means handlers must be idempotent.
+      class Job
+        DIGEST_BYTES = 16
+
+        attr_reader :type, :args, :enqueued_by, :max_attempts
+        attr_accessor :attempts, :last_error
+
+        def initialize(type:, args:, enqueued_by: nil, attempts: 0, max_attempts: 3, last_error: nil)
+          @type        = type.to_s
+          @args        = stringify(args)
+          @enqueued_by = enqueued_by
+          @attempts    = attempts
+          @max_attempts = max_attempts
+          @last_error = last_error
+        end
+
+        def id
+          "#{@type}:#{digest}"
+        end
+
+        def to_h
+          {
+            "type" => @type,
+            "args" => @args,
+            "enqueued_by" => @enqueued_by,
+            "attempts" => @attempts,
+            "max_attempts" => @max_attempts,
+            "last_error" => @last_error,
+          }
+        end
+
+        def self.from_h(hash)
+          new(
+            type: hash["type"],
+            args: hash["args"] || {},
+            enqueued_by: hash["enqueued_by"],
+            attempts: hash["attempts"] || 0,
+            max_attempts: hash["max_attempts"] || 3,
+            last_error: hash["last_error"],
+          )
+        end
+
+        private
+
+        def digest
+          canonical = JSON.dump(@args.sort.to_h)
+          Digest::SHA256.hexdigest(canonical)[0, DIGEST_BYTES]
+        end
+
+        def stringify(hash)
+          hash.transform_keys(&:to_s)
+        end
+      end
+    end
+  end
+end

data/lib/textus/ports/queue.rb

@@ -0,0 +1,130 @@
+require "fileutils"
+require "json"
+require "time"
+
+module Textus
+  module Ports
+    # File-backed durable job queue under `<root>/.run/queue/`. Each job state
+    # is a directory; a job is one `<id>.json` file. Claiming is an atomic
+    # `rename(2)` from ready/ to leased/ — the rename winner owns the job, so a
+    # worker pool needs no central lock. Dedup falls out of the id-as-filename:
+    # enqueueing an id that already exists is a no-op. ADR 0038 (runtime subtree),
+    # ADR 0108 (instantiable port).
+    class Queue
+      STATES = %i[ready leased done failed].freeze
+
+      def initialize(root:)
+        @root = root
+        STATES.each { |s| FileUtils.mkdir_p(Textus::Layout.queue_state(root, s)) }
+      end
+
+      def enqueue(job)
+        dest = path(:ready, job.id)
+        return if File.exist?(dest) # dedup: identical work already queued
+
+        write_atomic(dest, job.to_h)
+      end
+
+      def ready_ids
+        Dir.children(Textus::Layout.queue_state(@root, :ready)).map { |f| File.basename(f, ".json") }
+      end
+
+      # A claimed job plus the path it lives at, so ack/fail act on this copy.
+      Leased = Struct.new(:job, :leased_path, keyword_init: true)
+
+      def lease(worker_id:, lease_ttl:)
+        ready_dir = Textus::Layout.queue_state(@root, :ready)
+        Dir.children(ready_dir).each do |name|
+          src = File.join(ready_dir, name)
+          dst = File.join(Textus::Layout.queue_state(@root, :leased), name)
+          begin
+            File.rename(src, dst) # atomic claim; loser's rename raises ENOENT
+          rescue Errno::ENOENT
+            next # another worker won this one
+          end
+          job = Job.from_h(JSON.parse(File.read(dst)))
+          stamp_lease(dst, worker_id: worker_id, expires_at: Time.now.utc + lease_ttl)
+          return Leased.new(job: job, leased_path: dst)
+        end
+        nil
+      end
+
+      def ack(leased)
+        dest = File.join(Textus::Layout.queue_state(@root, :done), File.basename(leased.leased_path))
+        File.rename(leased.leased_path, dest)
+      end
+
+      # Increment attempts and either requeue (transient) or dead-letter (attempts
+      # exhausted). Returns :requeued or :dead_lettered so the worker can count
+      # terminal failures distinctly from retries.
+      def fail(leased, error:)
+        job = leased.job
+        job.attempts += 1
+        job.last_error = error
+        dead = job.attempts >= job.max_attempts
+        write_atomic(path(dead ? :failed : :ready, job.id), job.to_h)
+        File.delete(leased.leased_path)
+        dead ? :dead_lettered : :requeued
+      end
+
+      # Return expired leases to ready/ (the holding worker crashed). Returns the
+      # count reclaimed. At-least-once delivery: a job whose handler actually
+      # finished but whose ack was lost will re-run — handlers must be idempotent.
+      def reclaim(now:)
+        leased_dir = Textus::Layout.queue_state(@root, :leased)
+        count = 0
+        Dir.children(leased_dir).each do |name|
+          src = File.join(leased_dir, name)
+          data = JSON.parse(File.read(src))
+          expires = data.dig("lease", "expires_at")
+          next if expires && Time.parse(expires) > now
+
+          dst = File.join(Textus::Layout.queue_state(@root, :ready), name)
+          data.delete("lease")
+          File.write(src, JSON.pretty_generate(data))
+          File.rename(src, dst)
+          count += 1
+        rescue Errno::ENOENT
+          next # raced with another reclaimer / the worker's ack
+        end
+        count
+      end
+
+      def list(state)
+        Dir.children(Textus::Layout.queue_state(@root, state.to_sym)).map { |f| File.basename(f, ".json") }
+      end
+
+      def retry_failed(job_id)
+        src = path(:failed, job_id)
+        data = JSON.parse(File.read(src))
+        data["attempts"] = 0
+        data["last_error"] = nil
+        write_atomic(path(:ready, job_id), data)
+        File.delete(src)
+      end
+
+      def purge(state)
+        dir = Textus::Layout.queue_state(@root, state.to_sym)
+        Dir.children(dir).each { |f| File.delete(File.join(dir, f)) }
+      end
+
+      private
+
+      def stamp_lease(leased_path, worker_id:, expires_at:)
+        data = JSON.parse(File.read(leased_path))
+        data["lease"] = { "worker_id" => worker_id, "expires_at" => expires_at.iso8601 }
+        File.write(leased_path, JSON.pretty_generate(data))
+      end
+
+      def path(state, job_id)
+        File.join(Textus::Layout.queue_state(@root, state), "#{job_id}.json")
+      end
+
+      def write_atomic(dest, hash)
+        tmp = "#{dest}.#{Process.pid}.tmp"
+        File.write(tmp, JSON.pretty_generate(hash))
+        File.rename(tmp, dest) # atomic on same filesystem
+      end
+    end
+  end
+end

data/lib/textus/ports/sentinel_store.rb

@@ -7,7 +7,7 @@ module Textus
     # Persistence adapter for sentinel files. Owns the on-disk JSON shape, the
     # path layout (<store_root>/.run/sentinels/<target-rel-to-repo>.textus-managed.json
     # — runtime, git-ignored, ADR 0070), and all File/FileUtils I/O.
-    # Domain::Sentinel is a pure value object that depends on this port for
+    # Core::Sentinel is a pure value object that depends on this port for
     # reads and writes.
     class SentinelStore
       SUFFIX = ".textus-managed.json".freeze
@@ -26,7 +26,7 @@ module Textus
 
       def load(path, repo_root)
         raw = JSON.parse(File.read(path))
-        Textus::Domain::Sentinel.new(
+        Textus::Core::Sentinel.new(
           target: absolutize(raw["target"], repo_root),
           source: absolutize(raw["source"], repo_root),
           sha256: raw["sha256"],