textus 0.53.0 → 0.54.0

data/lib/textus/jobs/materialize.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Textus
+  module Jobs
+    class Materialize < Base
+      TYPE = "materialize"
+
+      def initialize(key:)
+        super()
+        @key = key
+      end
+
+      def args = { key: @key }
+
+      def call(container:, call:)
+        Textus::Produce::Engine.converge(container: container, call: call, keys: [@key])
+      end
+    end
+  end
+end

data/lib/textus/{background → jobs}/plan.rb

@@ -1,5 +1,5 @@
 module Textus
-  module Background
+  module Jobs
     Plan = Data.define(:steps, :warnings) do
       def to_h
         { "steps" => steps, "warnings" => warnings }

data/lib/textus/jobs/planner.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Textus
+  module Jobs
+    class Planner
+      ACTIONS_BY_TRIGGER = {
+        "convergence" => %w[materialize sweep],
+        "entry.written" => %w[materialize],
+        "entry.deleted" => %w[materialize],
+        "entry.moved" => %w[materialize],
+        "proposal.accepted" => %w[materialize],
+        "proposal.rejected" => %w[materialize],
+      }.freeze
+
+      SCOPE_RESOLVERS = {
+        "materialize" => :producible_keys,
+        "sweep" => :lane_keys,
+      }.freeze
+
+      def self.seed(container:, queue:, role:)
+        jobs = new(container: container).plan(
+          trigger: { "type" => "convergence" },
+          role: role,
+        )
+        jobs.each { |j| queue.enqueue(j) }
+      end
+
+      def initialize(container:)
+        @container = container
+        @manifest  = container.manifest
+      end
+
+      def plan(trigger:, role:)
+        type = trigger["type"] || trigger[:type]
+        trigger["target"] || trigger[:target]
+        return [] if type.nil?
+
+        blocks_with_react = @manifest.rules.blocks.select(&:react)
+        if blocks_with_react.any?
+          plan_from_rules(blocks_with_react, type, role)
+        else
+          plan_from_defaults(type, role)
+        end
+      end
+
+      private
+
+      def plan_from_rules(blocks, type, role)
+        jobs = []
+        blocks
+          .select { |b| matches_trigger?(b.react, type) }
+          .each do |block|
+            do_action = block.react.raw["do"]
+            Array(do_action).each do |action|
+              if action == "sweep"
+                jobs << Textus::Ports::JobStore::Job.new(
+                  type: "sweep", args: { "scope" => {} }, enqueued_by: role,
+                )
+              else
+                resolver = SCOPE_RESOLVERS.fetch(action, :producible_keys)
+                keys = send(resolver, nil)
+                keys.each { |key| jobs << job(action, key, role) }
+              end
+            end
+          end
+        jobs
+      end
+
+      def plan_from_defaults(type, role)
+        actions = ACTIONS_BY_TRIGGER.fetch(type, [])
+        jobs = []
+        producible_keys(nil).each { |k| jobs << job("materialize", k, role) } if actions.include?("materialize")
+        if actions.include?("sweep")
+          jobs << Textus::Ports::JobStore::Job.new(
+            type: "sweep", args: { "scope" => {} }, enqueued_by: role,
+          )
+        end
+        jobs
+      end
+
+      def matches_trigger?(react, type)
+        on = react.raw["on"]
+        Array(on).include?(type)
+      end
+
+      def job(type, key, enqueued_by)
+        Textus::Ports::JobStore::Job.new(type: type, args: { "key" => key }, enqueued_by: enqueued_by)
+      end
+
+      def producible_keys(_target)
+        @manifest.data.entries
+                 .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
+                 .map(&:key)
+      end
+
+      def lane_keys(_target)
+        @manifest.data.entries.map(&:key)
+      end
+    end
+  end
+end

data/lib/textus/jobs/retention.rb

@@ -0,0 +1,48 @@
+require "fileutils"
+
+module Textus
+  module Jobs
+    class Retention
+      def initialize(container:, call:)
+        @container = container
+        @call = call
+      end
+
+      def call(rows)
+        out = { dropped: [], archived: [], failed: [] }
+        rows.each do |row|
+          key = row["key"]
+          begin
+            case row["action"]
+            when "drop"
+              delete(key)
+              out[:dropped] << key
+            when "archive"
+              archive_leaf(row)
+              delete(key)
+              out[:archived] << key
+            end
+          rescue Textus::Error => e
+            out[:failed] << { "key" => key, "error" => e.message }
+          end
+        end
+        out
+      end
+
+      private
+
+      def archive_leaf(row)
+        src  = row["path"]
+        root = @container.root.to_s
+        rel  = src.delete_prefix("#{root}/")
+        dest = File.join(root, "archive", rel)
+        FileUtils.mkdir_p(File.dirname(dest))
+        FileUtils.cp(src, dest)
+      end
+
+      def delete(key)
+        Textus::Action::KeyDelete.new(key: key).call(container: @container, call: @call)
+      end
+    end
+  end
+end

data/lib/textus/jobs/sweep.rb

@@ -0,0 +1,27 @@
+module Textus
+  module Jobs
+    class Sweep < Base
+      REQUIRED_ROLE = Textus::Role::AUTOMATION
+      TYPE = "sweep"
+
+      def initialize(scope: nil, key: nil)
+        super()
+        @scope = scope || {}
+        @key   = key
+      end
+
+      def args = { scope: @scope, key: @key }.compact
+
+      def call(container:, call:)
+        prefix = @key || (@scope.is_a?(Hash) ? @scope["prefix"] : nil)
+        lane   = @scope.is_a?(Hash) ? @scope["lane"] : nil
+        rows = Textus::Core::Retention::Sweep.new(
+          manifest: container.manifest,
+          file_stat: Textus::Ports::Storage::FileStat.new,
+          clock: Textus::Ports::Clock.new,
+        ).call(prefix: prefix, lane: lane)
+        Textus::Jobs::Retention.new(container: container, call: call).call(rows)
+      end
+    end
+  end
+end

data/lib/textus/{background → jobs}/worker.rb

@@ -1,5 +1,5 @@
 module Textus
-  module Background
+  module Jobs
     class Worker
       Summary = Struct.new(:completed, :failed, keyword_init: true)
 
@@ -46,7 +46,7 @@ module Textus
 
       def run_one(leased)
         job    = leased.job
-        klass  = Textus::Background::Job.fetch(job.type)
+        klass  = Textus::Jobs.fetch(job.type)
         action = if klass.instance_method(:initialize).parameters.any?
                    klass.new(**job.args.transform_keys(&:to_sym))
                  else

data/lib/textus/jobs.rb

@@ -0,0 +1,15 @@
+module Textus
+  module Jobs
+    @registry = {}
+
+    def self.registry = @registry
+
+    def self.register(klass)
+      @registry[klass::TYPE] = klass if klass.const_defined?(:TYPE, false)
+    end
+
+    def self.fetch(type)
+      @registry.fetch(type) { raise Textus::UsageError.new("unknown job type: #{type}") }
+    end
+  end
+end

data/lib/textus/key/path.rb

@@ -9,7 +9,7 @@ module Textus
       # callers historically passed the whole Manifest but should now pass
       # `manifest.data`.
       def self.resolve(data, mentry)
-        primary_ext = Entry.for_format(mentry.format).extensions.first
+        primary_ext = Format.for(mentry.format).extensions.first
         rel_path = normalize_relative_path(mentry.path)
         if File.extname(mentry.path) == ""
           File.join(data.root, rel_path + primary_ext)

data/lib/textus/manifest/entry/base.rb

@@ -62,22 +62,15 @@ module Textus
         def ignored?(_rel_path) = false
 
         # Minimal context object passed into entry `publish_via` hooks.
-        # Everything beyond the three primitives is derived. Data.define
-        # instances are frozen, so we recompute per-call rather than
-        # memoizing — RoleScope/Step::Context construction is cheap.
+        # Everything beyond the three primitives is derived.
         PublishContext = ::Data.define(:container, :call, :reader) do
           def manifest   = container.manifest
           def root       = container.root
           def repo_root  = File.dirname(container.root)
-          def steps      = container.steps
 
-          def hook_context
-            Textus::Step::Context.new(scope: scope_for_hooks)
-          end
-
-          def emit(event, **payload)
-            steps.publish(event, ctx: hook_context, **payload)
-          end
+          # No-op: event bus removed in workflow redesign; callers that fire
+          # :entry_published / :entry_produced remain unchanged in the source.
+          def emit(_event, **_payload) = nil
 
           # Read a named template from the store's templates/ directory.
           # Raises TemplateError when the file doesn't exist.
@@ -91,14 +84,6 @@ module Textus
             end
             File.read(path)
           end
-
-          private
-
-          def scope_for_hooks
-            Textus::Surfaces::RoleScope.new(
-              container: container, role: call.role, dry_run: call.dry_run,
-            )
-          end
         end
 
         # ADR 0049: an entry resolves, once, to one Publish::* mode that owns its

data/lib/textus/manifest/entry/parser.rb

@@ -4,7 +4,6 @@ module Textus
       module Parser
         def self.call(raw)
           key = raw["key"] or raise UsageError.new("manifest entry missing key")
-          path = raw["path"] or raise UsageError.new("manifest entry '#{key}' missing path")
           lane = raw["lane"] or raise UsageError.new("manifest entry '#{key}' missing lane")
 
           raw_kind = raw["kind"] or raise BadManifest.new("entry '#{key}' missing required `kind:` (#{Entry::REGISTRY.keys.join("|")})")
@@ -15,7 +14,10 @@ module Textus
               "the produce method is `source.from` (#{kind == :intake ? "handler" : "project|command"})",
             )
           end
-          format = resolve_format(raw, path)
+
+          explicit_path = raw["path"]
+          format = resolve_format(raw, explicit_path)
+          path   = explicit_path || derive_path(key, kind, format)
 
           common = {
             raw: raw,
@@ -30,11 +32,11 @@ module Textus
           klass.from_raw(common, raw)
         end
 
-        # ADR 0093: an entry's production block is the unified `source:`. Returns a
-        # Manifest::Policy::Source; kind (intake/derived) is read from source.from.
-        def self.parse_source(raw, key)
-          block = raw["source"] or
-            raise BadManifest.new("entry '#{key}' requires a source: { from: derive|fetch|external, ... }")
+        # Parse the optional `source:` block. Returns nil when absent (workflow
+        # produced entries register their produce logic in .textus/workflows/).
+        def self.parse_source(raw, _key)
+          block = raw["source"]
+          return nil if block.nil?
 
           Textus::Manifest::Policy::Source.new(block)
         end
@@ -57,8 +59,11 @@ module Textus
 
         def self.resolve_format(raw, path)
           declared = raw["format"]
-          ext = File.extname(path)
-          inferred = Textus::Entry.infer_from_extension(ext)
+
+          return declared || "markdown" if path.nil? || path.empty?
+
+          ext      = File.extname(path)
+          inferred = Textus::Format.infer_from_extension(ext)
 
           if declared.nil?
             return inferred if inferred
@@ -66,7 +71,7 @@ module Textus
             return "markdown"
           end
 
-          raise UsageError.new("entry '#{raw["key"]}': unknown format #{declared.inspect}") unless Textus::Entry.formats.include?(declared)
+          raise UsageError.new("entry '#{raw["key"]}': unknown format #{declared.inspect}") unless Textus::Format.formats.include?(declared)
 
           if ext != "" && inferred && inferred != declared
             raise UsageError.new(
@@ -76,6 +81,18 @@ module Textus
 
           declared
         end
+
+        # Derives the manifest-relative path from key + kind + format.
+        # Key::Path.normalize_relative_path will prepend data/ at resolution time.
+        def self.derive_path(key, kind_sym, format)
+          dir_path = key.split(".").join("/")
+          return dir_path if kind_sym == :nested
+
+          ext = Textus::Format.for(format).extensions.first
+          "#{dir_path}#{ext}"
+        end
+
+        private_class_method :derive_path
       end
     end
   end

data/lib/textus/manifest/entry/produced.rb

@@ -1,53 +1,32 @@
 module Textus
   class Manifest
     class Entry
-      # A produced entry (ADR 0095) — anything with a `source:`. The produce
-      # method (intake/derived/external) is read from source.from; there is no
-      # separate kind for it. Merges the former Derived + Intake classes.
+      # A produced entry — `kind: produced` with an optional `source:` block.
+      # When `source:` is present it must be `from: external` (out-of-band
+      # generator; textus detects drift but never runs it). When absent the
+      # entry is produced by a workflow file in .textus/workflows/.
       class Produced < Base
-        attr_reader :source, :events
+        attr_reader :source
 
-        def initialize(source:, events: {}, **rest)
+        def initialize(source:, **rest)
           super(**rest)
           @source = source
-          @events = events || {}
         end
 
-        def intake?     = @source.kind == :intake
-        def derived?    = @source.kind == :derived
-        def external?   = @source.external?
-        def projection? = @source.projection?
-        def fetch?      = @source.fetch?
-        def derive?     = @source.derive?
-        def nested?     = !!@raw["nested"]
-        def handler     = @source.handler
-        def config      = @source.config
+        def external? = @source&.external? || false
+        def nested?   = !!@raw["nested"]
 
         KIND = :produced
 
-        # ADR 0094/0095: projection (from: project) sources build their DATA
-        # artifact here, then publish via the ONE shared mode (Publish::ToPaths).
-        # Intake bytes come from Produce::Acquire::Intake and command (external) bytes from the
-        # out-of-band runner — neither builds, but both still publish their
-        # existing store bytes through the same mode. A projection entry with no
-        # targets is a terminal data node: it produced data, so report :built
-        # even though nothing was emitted.
+        # Publish existing store bytes via the shared publish mode (Publish::ToPaths
+        # or Publish::None). Workflow runners handle the produce step; this method
+        # only publishes whatever bytes are already on disk.
         def publish_via(pctx, prefix: nil)
-          built = false
-          if projection?
-            Textus::Pipeline::Acquire::Projection.new(container: pctx.container, call: pctx.call).run(self)
-            built = true
-            pctx.emit(:entry_produced, key: @key, envelope: pctx.reader.call(@key), sources: Array(@source.select).compact)
-          end
-          emitted = publish_mode.publish(pctx, prefix: prefix)
-          return emitted if emitted
-          return nil unless built
-
-          { kind: :built, value: { "key" => @key, "path" => Key::Path.resolve(pctx.manifest.data, self), "published_to" => [] } }
+          publish_mode.publish(pctx, prefix: prefix)
         end
 
         def self.from_raw(common, raw)
-          new(source: Parser.parse_source(raw, common[:key]), events: raw["events"] || {}, **common)
+          new(source: Parser.parse_source(raw, common[:key]), **common)
         end
 
         Entry::REGISTRY[KIND] = self

data/lib/textus/manifest/entry/publish/to_paths.rb

@@ -21,16 +21,16 @@ module Textus
 
             data_path = pctx.manifest.resolver.resolve(entry.key).path
             envelope  = pctx.reader.call(entry.key)
-            renderer  = Textus::Pipeline::Render.new(template_loader: ->(n) { pctx.read_template(n) })
+            renderer  = Textus::Produce::Render.new(template_loader: ->(n) { pctx.read_template(n) })
             content = nil # parsed lazily; the data's `content` (always _meta-free)
 
             targets.each do |t|
               if t.renders?
-                content ||= Textus::Entry.for_format(entry.format).parse(File.read(data_path), path: data_path)["content"]
+                content ||= Textus::Format.for(entry.format).parse(File.read(data_path), path: data_path)["content"]
                 publish_bytes(render_bytes(t, content, renderer, pctx), entry.key, t, pctx, data_path, envelope)
               elsif strip_meta?(entry)
-                content ||= Textus::Entry.for_format(entry.format).parse(File.read(data_path), path: data_path)["content"]
-                bytes = Textus::Entry.for_format(entry.format).serialize(meta: {}, body: "", content: content)
+                content ||= Textus::Format.for(entry.format).parse(File.read(data_path), path: data_path)["content"]
+                bytes = Textus::Format.for(entry.format).serialize(meta: {}, body: "", content: content)
                 publish_bytes(bytes, entry.key, t, pctx, data_path, envelope)
               else
                 # opaque / command / non-structured — publish the stored file as-is

data/lib/textus/manifest/entry/publish.rb

@@ -18,8 +18,6 @@ module Textus
         # the block groups the two but does not make exclusivity structural, so
         # this stays the one enforcement point (ADR 0052 D2).
         def self.resolve(entry)
-          reject_removed_publish_each(entry)
-
           set = []
           set << "publish.to"   unless Array(entry.publish_to).empty?
           set << "publish.tree" unless entry.publish_tree.nil?
@@ -33,16 +31,6 @@ module Textus
           mode_for(entry, set.first)
         end
 
-        def self.reject_removed_publish_each(entry)
-          return unless entry.raw["publish_each"]
-
-          raise Textus::UsageError.new(
-            "entry '#{entry.key}': publish_each was removed in 0.42.0 (ADR 0051) — " \
-            "mirror the subtree with `publish: { tree: \"...\" }`.",
-          )
-        end
-        private_class_method :reject_removed_publish_each
-
         def self.mode_for(entry, key)
           case key
           when "publish.to"   then ToPaths.new(entry)

data/lib/textus/manifest/entry/validators/format_matrix.rb

@@ -5,7 +5,7 @@ module Textus
         module FormatMatrix
           def self.call(entry, policy:) # rubocop:disable Lint/UnusedMethodArgument
             begin
-              Textus::Entry.for_format(entry.format).validate_path_extension(entry.path, entry.nested?)
+              Textus::Format.for(entry.format).validate_path_extension(entry.path, entry.nested?)
             rescue UsageError => e
               raise UsageError.new("entry '#{entry.key}': #{e.message}")
             end

data/lib/textus/manifest/entry/validators.rb

@@ -3,7 +3,6 @@ module Textus
     class Entry
       module Validators
         REGISTERED = [
-          Events,
           Publish,
           Ignore,
           FormatMatrix,

data/lib/textus/manifest/policy/source.rb

@@ -1,77 +1,29 @@
 module Textus
   class Manifest
     class Policy
-      # An entry's data-acquisition declaration (ADR 0094). `source:` says HOW the
-      # entry's data is acquired; rendering is a publish concern, so there are no
-      # template/render fields here. `from` is the acquire + staleness axis:
-      #   from: derive -> derived   (internal projection; observable -> rdeps staleness)
-      #   from: fetch -> intake     (external fetch; unobservable -> ttl staleness)
-      #   from: external -> external (out-of-band runner; staleness only, textus never runs it)
-      # Materialization is async-only (job-queue model): a write enqueues a
-      # `materialize` job, converged by a worker. There is no per-entry write
-      # trigger knob.
+      # An entry's external-generator declaration. `source:` now means ONLY
+      # `from: external` — out-of-band runner; textus never invokes it, only
+      # detects drift (doctor generator_drift check).
+      # `from: fetch` and `from: derive` are removed; those concerns now live
+      # in workflow files under .textus/workflows/.
       class Source
-        FROMS = %w[fetch derive external].freeze
-
-        attr_reader :from, :handler, :config, :command, :sources
+        attr_reader :command, :sources
 
         def initialize(raw)
-          @from = raw["from"].to_s
-          unless FROMS.include?(@from)
-            raise Textus::BadManifest.new("source.from must be one of #{FROMS.join("|")}, got #{raw["from"].inspect}")
+          from = raw["from"].to_s
+          unless from == "external"
+            raise Textus::BadManifest.new(
+              "from: #{from} is removed — use a workflow file under .textus/workflows/ instead",
+            )
           end
 
-          @ttl = raw["ttl"]
-          @projection = {}
-
-          case @from
-          when "fetch" then init_fetch(raw)
-          when "derive" then init_derive(raw)
-          when "external" then init_external(raw)
-          end
-        end
-
-        def kind
-          { "fetch" => :intake, "derive" => :derived, "external" => :external }.fetch(@from)
-        end
-
-        def fetch?      = @from == "fetch"
-        def derive?     = @from == "derive"
-        def external?   = @from == "external"
-        def projection? = derive?
-        def ttl_seconds = @ttl.nil? ? nil : Textus::Core::Duration.seconds(@ttl)
-
-        # Flattened projection accessors (ADR 0094) — read directly off the source
-        # block; nil when absent or not a projection source.
-        def select    = @projection["select"]
-        def pluck     = @projection["pluck"]
-        def sort_by   = @projection["sort_by"]
-        def transform = @projection["transform"]
-
-        # The projection spec hash fed to Textus::Projection (string keys, only the
-        # present fields). {} when not a projection.
-        def projection_spec = @projection.dup
-
-        private
-
-        def init_derive(raw)
-          %w[select pluck sort_by transform].each { |f| @projection[f] = raw[f] if raw.key?(f) }
-          return unless @projection["select"].nil? && @projection["transform"].nil?
-
-          raise Textus::BadManifest.new("source (from: derive) requires `select:` and/or `transform:`")
-        end
-
-        def init_fetch(raw)
-          @handler = raw["handler"] or
-            raise Textus::BadManifest.new("source (from: fetch) requires a `handler:` field")
-          @config = raw["config"] || {}
-        end
-
-        def init_external(raw)
           @command = raw["command"] or
             raise Textus::BadManifest.new("source (from: external) requires a `command:` field")
           @sources = raw["sources"] || []
         end
+
+        def external? = true
+        def kind      = :external
       end
     end
   end

data/lib/textus/manifest/policy.rb

@@ -62,11 +62,8 @@ module Textus
       # and ask it directly. Returns false if entries are not yet built
       # (validator phase during Data#initialize) — validators must not rely on
       # cross-entry state during construction.
-      def derived_entry?(key)
-        return false if @data.entries.nil?
-
-        entry = @data.entries.find { |e| e.key == key } or return false
-        entry.derived?
+      def derived_entry?(_key)
+        false
       end
 
       # The single lane declaring kind: machine, or nil.

data/lib/textus/manifest/resolver.rb

@@ -53,7 +53,7 @@ module Textus
         else
           raise UnknownKey.new(key, suggestions: suggestions_for(key)) unless nested_entry?(entry)
 
-          primary_ext = Textus::Entry.for_format(entry.format).extensions.first
+          primary_ext = Textus::Format.for(entry.format).extensions.first
           base = Textus::Key::Path.normalize_relative_path(entry.path)
           path = File.join(@data.root, base, *remaining) + primary_ext
           Resolution.new(entry: entry, path: path, remaining: remaining)
@@ -111,7 +111,7 @@ module Textus
       end
 
       def nested_glob(format)
-        Textus::Entry.for_format(format).nested_glob
+        Textus::Format.for(format).nested_glob
       end
     end
   end