textus 0.52.0 → 0.54.0

data/lib/textus/jobs/planner.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Textus
+  module Jobs
+    class Planner
+      ACTIONS_BY_TRIGGER = {
+        "convergence" => %w[materialize sweep],
+        "entry.written" => %w[materialize],
+        "entry.deleted" => %w[materialize],
+        "entry.moved" => %w[materialize],
+        "proposal.accepted" => %w[materialize],
+        "proposal.rejected" => %w[materialize],
+      }.freeze
+
+      SCOPE_RESOLVERS = {
+        "materialize" => :producible_keys,
+        "sweep" => :lane_keys,
+      }.freeze
+
+      def self.seed(container:, queue:, role:)
+        jobs = new(container: container).plan(
+          trigger: { "type" => "convergence" },
+          role: role,
+        )
+        jobs.each { |j| queue.enqueue(j) }
+      end
+
+      def initialize(container:)
+        @container = container
+        @manifest  = container.manifest
+      end
+
+      def plan(trigger:, role:)
+        type = trigger["type"] || trigger[:type]
+        trigger["target"] || trigger[:target]
+        return [] if type.nil?
+
+        blocks_with_react = @manifest.rules.blocks.select(&:react)
+        if blocks_with_react.any?
+          plan_from_rules(blocks_with_react, type, role)
+        else
+          plan_from_defaults(type, role)
+        end
+      end
+
+      private
+
+      def plan_from_rules(blocks, type, role)
+        jobs = []
+        blocks
+          .select { |b| matches_trigger?(b.react, type) }
+          .each do |block|
+            do_action = block.react.raw["do"]
+            Array(do_action).each do |action|
+              if action == "sweep"
+                jobs << Textus::Ports::JobStore::Job.new(
+                  type: "sweep", args: { "scope" => {} }, enqueued_by: role,
+                )
+              else
+                resolver = SCOPE_RESOLVERS.fetch(action, :producible_keys)
+                keys = send(resolver, nil)
+                keys.each { |key| jobs << job(action, key, role) }
+              end
+            end
+          end
+        jobs
+      end
+
+      def plan_from_defaults(type, role)
+        actions = ACTIONS_BY_TRIGGER.fetch(type, [])
+        jobs = []
+        producible_keys(nil).each { |k| jobs << job("materialize", k, role) } if actions.include?("materialize")
+        if actions.include?("sweep")
+          jobs << Textus::Ports::JobStore::Job.new(
+            type: "sweep", args: { "scope" => {} }, enqueued_by: role,
+          )
+        end
+        jobs
+      end
+
+      def matches_trigger?(react, type)
+        on = react.raw["on"]
+        Array(on).include?(type)
+      end
+
+      def job(type, key, enqueued_by)
+        Textus::Ports::JobStore::Job.new(type: type, args: { "key" => key }, enqueued_by: enqueued_by)
+      end
+
+      def producible_keys(_target)
+        @manifest.data.entries
+                 .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
+                 .map(&:key)
+      end
+
+      def lane_keys(_target)
+        @manifest.data.entries.map(&:key)
+      end
+    end
+  end
+end

data/lib/textus/jobs/retention.rb

@@ -0,0 +1,48 @@
+require "fileutils"
+
+module Textus
+  module Jobs
+    class Retention
+      def initialize(container:, call:)
+        @container = container
+        @call = call
+      end
+
+      def call(rows)
+        out = { dropped: [], archived: [], failed: [] }
+        rows.each do |row|
+          key = row["key"]
+          begin
+            case row["action"]
+            when "drop"
+              delete(key)
+              out[:dropped] << key
+            when "archive"
+              archive_leaf(row)
+              delete(key)
+              out[:archived] << key
+            end
+          rescue Textus::Error => e
+            out[:failed] << { "key" => key, "error" => e.message }
+          end
+        end
+        out
+      end
+
+      private
+
+      def archive_leaf(row)
+        src  = row["path"]
+        root = @container.root.to_s
+        rel  = src.delete_prefix("#{root}/")
+        dest = File.join(root, "archive", rel)
+        FileUtils.mkdir_p(File.dirname(dest))
+        FileUtils.cp(src, dest)
+      end
+
+      def delete(key)
+        Textus::Action::KeyDelete.new(key: key).call(container: @container, call: @call)
+      end
+    end
+  end
+end

data/lib/textus/jobs/sweep.rb

@@ -0,0 +1,27 @@
+module Textus
+  module Jobs
+    class Sweep < Base
+      REQUIRED_ROLE = Textus::Role::AUTOMATION
+      TYPE = "sweep"
+
+      def initialize(scope: nil, key: nil)
+        super()
+        @scope = scope || {}
+        @key   = key
+      end
+
+      def args = { scope: @scope, key: @key }.compact
+
+      def call(container:, call:)
+        prefix = @key || (@scope.is_a?(Hash) ? @scope["prefix"] : nil)
+        lane   = @scope.is_a?(Hash) ? @scope["lane"] : nil
+        rows = Textus::Core::Retention::Sweep.new(
+          manifest: container.manifest,
+          file_stat: Textus::Ports::Storage::FileStat.new,
+          clock: Textus::Ports::Clock.new,
+        ).call(prefix: prefix, lane: lane)
+        Textus::Jobs::Retention.new(container: container, call: call).call(rows)
+      end
+    end
+  end
+end

data/lib/textus/jobs/worker.rb

@@ -0,0 +1,67 @@
+module Textus
+  module Jobs
+    class Worker
+      Summary = Struct.new(:completed, :failed, keyword_init: true)
+
+      def self.for(container:, queue:)
+        new(queue: queue, container: container,
+            lease_ttl: container.manifest.data.worker_config[:lease_ttl])
+      end
+
+      def initialize(queue:, container:, lease_ttl: 60)
+        @queue     = queue
+        @container = container
+        @lease_ttl = lease_ttl
+      end
+
+      def drain(worker_id: "drain-#{Process.pid}")
+        completed = 0
+        failed    = 0
+        loop do
+          leased = @queue.lease(worker_id: worker_id, lease_ttl: @lease_ttl)
+          break unless leased
+
+          case run_one(leased)
+          when :completed     then completed += 1
+          when :dead_lettered then failed += 1
+          end
+        end
+        Summary.new(completed: completed, failed: failed)
+      end
+
+      def drain_pool(pool: 4)
+        summaries = []
+        mutex     = Mutex.new
+        threads   = Array.new(pool) do |i|
+          Thread.new do
+            s = drain(worker_id: "pool-#{Process.pid}-#{i}")
+            mutex.synchronize { summaries << s }
+          end
+        end
+        threads.each(&:join)
+        Summary.new(completed: summaries.sum(&:completed), failed: summaries.sum(&:failed))
+      end
+
+      private
+
+      def run_one(leased)
+        job    = leased.job
+        klass  = Textus::Jobs.fetch(job.type)
+        action = if klass.instance_method(:initialize).parameters.any?
+                   klass.new(**job.args.transform_keys(&:to_sym))
+                 else
+                   klass.new
+                 end
+        call   = Textus::Call.build(
+          role: job.enqueued_by || Textus::Role::AUTOMATION,
+          correlation_id: SecureRandom.uuid,
+        )
+        action.call(container: @container, call: call)
+        @queue.ack(leased)
+        :completed
+      rescue StandardError => e
+        @queue.fail(leased, error: e.message)
+      end
+    end
+  end
+end

data/lib/textus/jobs.rb

@@ -0,0 +1,15 @@
+module Textus
+  module Jobs
+    @registry = {}
+
+    def self.registry = @registry
+
+    def self.register(klass)
+      @registry[klass::TYPE] = klass if klass.const_defined?(:TYPE, false)
+    end
+
+    def self.fetch(type)
+      @registry.fetch(type) { raise Textus::UsageError.new("unknown job type: #{type}") }
+    end
+  end
+end

data/lib/textus/key/path.rb

@@ -9,13 +9,20 @@ module Textus
       # callers historically passed the whole Manifest but should now pass
       # `manifest.data`.
       def self.resolve(data, mentry)
-        primary_ext = Entry.for_format(mentry.format).extensions.first
+        primary_ext = Format.for(mentry.format).extensions.first
+        rel_path = normalize_relative_path(mentry.path)
         if File.extname(mentry.path) == ""
-          File.join(data.root, "zones", mentry.path + primary_ext)
+          File.join(data.root, rel_path + primary_ext)
         else
-          File.join(data.root, "zones", mentry.path)
+          File.join(data.root, rel_path)
         end
       end
+
+      def self.normalize_relative_path(path)
+        return path if path.start_with?("data/")
+
+        File.join("data", path)
+      end
     end
   end
 end

data/lib/textus/layout.rb

@@ -4,6 +4,15 @@ module Textus
   # tracked/disposable boundary is a directory boundary. ADR 0038.
   module Layout
     RUN = ".run"
+    DATA = "data"
+
+    def self.data(root)
+      File.join(root, DATA)
+    end
+
+    def self.data_lane(root, lane_name)
+      File.join(data(root), lane_name)
+    end
 
     def self.run(root)
       File.join(root, RUN)
@@ -25,6 +34,10 @@ module Textus
       File.join(run(root), "build.lock")
     end
 
+    def self.watcher_lock(root)
+      File.join(run(root), "watcher.lock")
+    end
+
     def self.queue(root)
       File.join(run(root), "queue")
     end

data/lib/textus/manifest/data.rb

@@ -5,15 +5,15 @@ module Textus
   class Manifest
     # Immutable, parsed view of a manifest YAML document.
     #
-    # Holds raw structural data (zones, entries, audit_config, role_caps)
-    # but no behaviour beyond accessors. Behaviour (zone authority, key
+    # Holds raw structural data (lanes, entries, audit_config, role_caps)
+    # but no behaviour beyond accessors. Behaviour (lane authority, key
     # resolution, rules) lives on Manifest::Policy / Resolver / Rules.
     class Data
       AUDIT_DEFAULTS = { max_size: 10_485_760, keep: 5 }.freeze
       WORKER_DEFAULTS = { pool: 4, poll: 5, lease_ttl: 60, max_attempts: 3 }.freeze
 
-      attr_reader :raw, :root, :entries, :declared_zone_kinds,
-                  :zone_descs, :zone_owners,
+      attr_reader :raw, :root, :entries, :declared_lane_kinds,
+                  :lane_descs, :lane_owners,
                   :audit_config, :worker_config, :role_caps, :policy
 
       def self.validate_key!(key)
@@ -27,7 +27,7 @@ module Textus
       def validate_key!(key) = self.class.validate_key!(key)
 
       def self.parse(raw, root:)
-        raise BadFrontmatter.new(File.join(root.to_s, "manifest.yaml"), "manifest must declare zones:") if Array(raw["zones"]).empty?
+        raise BadFrontmatter.new(File.join(root.to_s, "manifest.yaml"), "manifest must declare lanes:") if Array(raw["lanes"]).empty?
 
         Schema.validate!(raw)
         new(raw: raw, root: root)
@@ -36,17 +36,17 @@ module Textus
       def initialize(raw:, root:)
         @raw = raw
         @root = root
-        # Write authority is derived from capabilities × zone-kind (ADR 0030),
-        # not a per-zone writer list. "Which zones are declared" lives in the
-        # one kind-keyed map below (declared_zone_kinds); membership checks by
-        # read-side callers (boot, maintenance/zone_mv) use its keyset (ADR 0034).
-        @declared_zone_kinds = Array(raw["zones"]).to_h do |z|
+        # Write authority is derived from capabilities × lane-kind (ADR 0030),
+        # not a per-lane writer list. "Which lanes are declared" lives in the
+        # one kind-keyed map below (declared_lane_kinds); membership checks by
+        # read-side callers (boot, maintenance/data_mv) use its keyset (ADR 0034).
+        @declared_lane_kinds = Array(raw["lanes"]).to_h do |z|
           [z["name"], z["kind"]&.to_sym]
         end
-        @zone_descs  = Array(raw["zones"]).to_h { |z| [z["name"], z["desc"]] }
-        # Only zones that actually declare an owner — keep nil-tombstones out so a
-        # future `zone_owners.key?(name)` means "owner declared", not "zone exists".
-        @zone_owners = Array(raw["zones"]).to_h { |z| [z["name"], z["owner"]] }.compact
+        @lane_descs  = Array(raw["lanes"]).to_h { |z| [z["name"], z["desc"]] }
+        # Only lanes that actually declare an owner — keep nil-tombstones out so a
+        # future `lane_owners.key?(name)` means "owner declared", not "lane exists".
+        @lane_owners = Array(raw["lanes"]).to_h { |z| [z["name"], z["owner"]] }.compact
         @audit_config = build_audit_config(raw)
         @worker_config = build_worker_config(raw)
         @role_caps = Capabilities.resolve(raw["roles"])

data/lib/textus/manifest/entry/base.rb

@@ -2,14 +2,14 @@ module Textus
   class Manifest
     class Entry
       class Base < Entry
-        attr_reader :raw, :key, :path, :zone, :schema, :owner, :format, :publish_targets
+        attr_reader :raw, :key, :path, :lane, :schema, :owner, :format, :publish_targets
 
         # rubocop:disable Metrics/ParameterLists, Lint/MissingSuper
-        def initialize(raw:, key:, path:, zone:, schema:, owner:, format:, publish_targets: [])
+        def initialize(raw:, key:, path:, lane:, schema:, owner:, format:, publish_targets: [])
           @raw = raw
           @key = key
           @path = path
-          @zone = zone
+          @lane = lane
           @schema = schema
           @owner = owner
           @format = format
@@ -17,13 +17,14 @@ module Textus
         end
         # rubocop:enable Metrics/ParameterLists, Lint/MissingSuper
 
-        def zone_writers(policy)
-          policy.zone_writers(@zone)
+        def lane_writers(policy)
+          verb = policy.verb_for_lane(@lane)
+          policy.roles_with_capability(verb)
         rescue UsageError => e
           raise UsageError.new("entry '#{@key}': #{e.message}")
         end
 
-        def in_proposal_zone?(policy) = policy.queue_zone?(@zone)
+        def in_proposal_lane?(policy) = policy.queue_lane?(@lane)
 
         def nested?  = false
         def derived? = false
@@ -36,6 +37,9 @@ module Textus
         def external?   = false
         def projection? = false
 
+        alias zone lane
+        alias in_proposal_zone? in_proposal_lane?
+
         # Whether git should track this entry's file. Default true; an entry
         # marked `tracked: false` in the manifest stays protocol-readable but is
         # listed in the generated `.gitignore` (ADR 0043). Cross-cutting, so it
@@ -58,22 +62,15 @@ module Textus
         def ignored?(_rel_path) = false
 
         # Minimal context object passed into entry `publish_via` hooks.
-        # Everything beyond the three primitives is derived. Data.define
-        # instances are frozen, so we recompute per-call rather than
-        # memoizing — RoleScope/Hooks::Context construction is cheap.
+        # Everything beyond the three primitives is derived.
         PublishContext = ::Data.define(:container, :call, :reader) do
           def manifest   = container.manifest
           def root       = container.root
           def repo_root  = File.dirname(container.root)
-          def events     = container.events
-
-          def hook_context
-            Textus::Hooks::Context.new(scope: scope_for_hooks)
-          end
 
-          def emit(event, **payload)
-            events.publish(event, ctx: hook_context, **payload)
-          end
+          # No-op: event bus removed in workflow redesign; callers that fire
+          # :entry_published / :entry_produced remain unchanged in the source.
+          def emit(_event, **_payload) = nil
 
           # Read a named template from the store's templates/ directory.
           # Raises TemplateError when the file doesn't exist.
@@ -87,14 +84,6 @@ module Textus
             end
             File.read(path)
           end
-
-          private
-
-          def scope_for_hooks
-            Textus::RoleScope.new(
-              container: container, role: call.role, dry_run: call.dry_run,
-            )
-          end
         end
 
         # ADR 0049: an entry resolves, once, to one Publish::* mode that owns its

data/lib/textus/manifest/entry/parser.rb

@@ -4,8 +4,7 @@ module Textus
       module Parser
         def self.call(raw)
           key = raw["key"] or raise UsageError.new("manifest entry missing key")
-          path = raw["path"] or raise UsageError.new("manifest entry '#{key}' missing path")
-          zone = raw["zone"] or raise UsageError.new("manifest entry '#{key}' missing zone")
+          lane = raw["lane"] or raise UsageError.new("manifest entry '#{key}' missing lane")
 
           raw_kind = raw["kind"] or raise BadManifest.new("entry '#{key}' missing required `kind:` (#{Entry::REGISTRY.keys.join("|")})")
           kind = raw_kind.to_sym
@@ -15,11 +14,14 @@ module Textus
               "the produce method is `source.from` (#{kind == :intake ? "handler" : "project|command"})",
             )
           end
-          format = resolve_format(raw, path)
+
+          explicit_path = raw["path"]
+          format = resolve_format(raw, explicit_path)
+          path   = explicit_path || derive_path(key, kind, format)
 
           common = {
             raw: raw,
-            key: key, path: path, zone: zone,
+            key: key, path: path, lane: lane,
             schema: raw["schema"], owner: raw["owner"],
             format: format,
             publish_targets: publish_targets(raw)
@@ -30,13 +32,13 @@ module Textus
           klass.from_raw(common, raw)
         end
 
-        # ADR 0093: an entry's production block is the unified `source:`. Returns a
-        # Domain::Policy::Source; kind (intake/derived) is read from source.from.
-        def self.parse_source(raw, key)
-          block = raw["source"] or
-            raise BadManifest.new("entry '#{key}' requires a source: { from: project|handler|command, ... }")
+        # Parse the optional `source:` block. Returns nil when absent (workflow
+        # produced entries register their produce logic in .textus/workflows/).
+        def self.parse_source(raw, _key)
+          block = raw["source"]
+          return nil if block.nil?
 
-          Textus::Domain::Policy::Source.new(block)
+          Textus::Manifest::Policy::Source.new(block)
         end
 
         # ADR 0094: `publish:` is a LIST of target objects — to-targets
@@ -52,13 +54,16 @@ module Textus
               "[{to:, template:?} | {tree:}] (ADR 0094); the `publish: { … }` map form was retired",
             )
           end
-          block.map { |t| Textus::Domain::Policy::PublishTarget.new(t) }
+          block.map { |t| Textus::Manifest::Policy::PublishTarget.new(t) }
         end
 
         def self.resolve_format(raw, path)
           declared = raw["format"]
-          ext = File.extname(path)
-          inferred = Textus::Entry.infer_from_extension(ext)
+
+          return declared || "markdown" if path.nil? || path.empty?
+
+          ext      = File.extname(path)
+          inferred = Textus::Format.infer_from_extension(ext)
 
           if declared.nil?
             return inferred if inferred
@@ -66,7 +71,7 @@ module Textus
             return "markdown"
           end
 
-          raise UsageError.new("entry '#{raw["key"]}': unknown format #{declared.inspect}") unless Textus::Entry.formats.include?(declared)
+          raise UsageError.new("entry '#{raw["key"]}': unknown format #{declared.inspect}") unless Textus::Format.formats.include?(declared)
 
           if ext != "" && inferred && inferred != declared
             raise UsageError.new(
@@ -76,6 +81,18 @@ module Textus
 
           declared
         end
+
+        # Derives the manifest-relative path from key + kind + format.
+        # Key::Path.normalize_relative_path will prepend data/ at resolution time.
+        def self.derive_path(key, kind_sym, format)
+          dir_path = key.split(".").join("/")
+          return dir_path if kind_sym == :nested
+
+          ext = Textus::Format.for(format).extensions.first
+          "#{dir_path}#{ext}"
+        end
+
+        private_class_method :derive_path
       end
     end
   end

data/lib/textus/manifest/entry/produced.rb

@@ -1,52 +1,32 @@
 module Textus
   class Manifest
     class Entry
-      # A produced entry (ADR 0095) — anything with a `source:`. The produce
-      # method (intake/derived/external) is read from source.from; there is no
-      # separate kind for it. Merges the former Derived + Intake classes.
+      # A produced entry — `kind: produced` with an optional `source:` block.
+      # When `source:` is present it must be `from: external` (out-of-band
+      # generator; textus detects drift but never runs it). When absent the
+      # entry is produced by a workflow file in .textus/workflows/.
       class Produced < Base
-        attr_reader :source, :events
+        attr_reader :source
 
-        def initialize(source:, events: {}, **rest)
+        def initialize(source:, **rest)
           super(**rest)
           @source = source
-          @events = events || {}
         end
 
-        def intake?     = @source.kind == :intake
-        def derived?    = @source.kind == :derived
-        def external?   = @source.external?
-        def projection? = @source.projection?
-        def nested?     = !!@raw["nested"]
-        def handler     = @source.handler
-        def config      = @source.config
+        def external? = @source&.external? || false
+        def nested?   = !!@raw["nested"]
 
         KIND = :produced
 
-        # ADR 0094/0095: projection (from: project) sources build their DATA
-        # artifact here, then publish via the ONE shared mode (Publish::ToPaths).
-        # Intake bytes come from Produce::Acquire::Intake and command (external) bytes from the
-        # out-of-band runner — neither builds, but both still publish their
-        # existing store bytes through the same mode. A projection entry with no
-        # targets is a terminal data node: it produced data, so report :built
-        # even though nothing was emitted.
+        # Publish existing store bytes via the shared publish mode (Publish::ToPaths
+        # or Publish::None). Workflow runners handle the produce step; this method
+        # only publishes whatever bytes are already on disk.
         def publish_via(pctx, prefix: nil)
-          built = false
-          if projection?
-            Textus::Produce::Acquire::Projection.new(container: pctx.container, call: pctx.call).run(self)
-            built = true
-            pctx.emit(:entry_produced, key: @key, envelope: pctx.reader.call(@key), sources: Array(@source.select).compact)
-          end
-
-          emitted = publish_mode.publish(pctx, prefix: prefix)
-          return emitted if emitted
-          return nil unless built
-
-          { kind: :built, value: { "key" => @key, "path" => Key::Path.resolve(pctx.manifest.data, self), "published_to" => [] } }
+          publish_mode.publish(pctx, prefix: prefix)
         end
 
         def self.from_raw(common, raw)
-          new(source: Parser.parse_source(raw, common[:key]), events: raw["events"] || {}, **common)
+          new(source: Parser.parse_source(raw, common[:key]), **common)
         end
 
         Entry::REGISTRY[KIND] = self

data/lib/textus/manifest/entry/publish/mode.rb

@@ -36,7 +36,7 @@ module Textus
 
           # Store-side directory this entry's tree lives under.
           def store_base(pctx)
-            File.join(pctx.root, "zones", entry.path)
+            File.join(pctx.root, Textus::Key::Path.normalize_relative_path(entry.path))
           end
         end
       end

data/lib/textus/manifest/entry/publish/to_paths.rb

@@ -16,6 +16,7 @@ module Textus
 
           def publish(pctx, prefix: nil) # rubocop:disable Lint/UnusedMethodArgument,Metrics/AbcSize
             targets = entry.publish_targets.select(&:to_target?)
+
             return nil if targets.empty?
 
             data_path = pctx.manifest.resolver.resolve(entry.key).path
@@ -25,11 +26,11 @@ module Textus
 
             targets.each do |t|
               if t.renders?
-                content ||= Textus::Entry.for_format(entry.format).parse(File.read(data_path), path: data_path)["content"]
+                content ||= Textus::Format.for(entry.format).parse(File.read(data_path), path: data_path)["content"]
                 publish_bytes(render_bytes(t, content, renderer, pctx), entry.key, t, pctx, data_path, envelope)
               elsif strip_meta?(entry)
-                content ||= Textus::Entry.for_format(entry.format).parse(File.read(data_path), path: data_path)["content"]
-                bytes = Textus::Entry.for_format(entry.format).serialize(meta: {}, body: "", content: content)
+                content ||= Textus::Format.for(entry.format).parse(File.read(data_path), path: data_path)["content"]
+                bytes = Textus::Format.for(entry.format).serialize(meta: {}, body: "", content: content)
                 publish_bytes(bytes, entry.key, t, pctx, data_path, envelope)
               else
                 # opaque / command / non-structured — publish the stored file as-is