textus 0.53.0 → 0.54.0

data/lib/textus/workflow/dsl.rb

@@ -0,0 +1,44 @@
+module Textus
+  module Workflow
+    module DSL
+      Step = Data.define(:name, :callable, :timeout)
+
+      class Definition
+        attr_reader :name, :steps, :match_pattern, :publish_block
+
+        def initialize(name)
+          @name          = name
+          @steps         = []
+          @match_pattern = nil
+          @publish_block = nil
+        end
+
+        def match(pattern)
+          @match_pattern = pattern
+        end
+
+        def step(name, callable_or_opt = nil, timeout: nil, &block)
+          callable = if callable_or_opt.respond_to?(:call)
+                       callable_or_opt
+                     elsif block
+                       block
+                     else
+                       raise ArgumentError.new("step :#{name} requires a block or a callable (got neither)")
+                     end
+          t = callable_or_opt.is_a?(Hash) ? callable_or_opt[:timeout] : timeout
+          @steps << Step.new(name: name, callable: callable, timeout: t)
+        end
+
+        def publish(&block)
+          @publish_block = block || :default
+        end
+
+        def match?(key)
+          return false unless @match_pattern
+
+          Pattern.match?(@match_pattern, key)
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/loader.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Workflow
+    class Loader
+      def self.load_all(root)
+        registry      = Registry.new
+        workflows_dir = File.join(root, "workflows")
+        return registry unless File.directory?(workflows_dir)
+
+        collector = Collector.new(registry)
+        Collector.with(collector) do
+          Dir.glob(File.join(workflows_dir, "**", "*.rb")).each { |path| load path }
+        end
+        registry
+      end
+    end
+  end
+end

data/lib/textus/workflow/pattern.rb

@@ -0,0 +1,18 @@
+module Textus
+  module Workflow
+    module Pattern
+      def self.match?(pattern, key)
+        if pattern.end_with?(".**")
+          prefix = pattern.delete_suffix(".**")
+          key.start_with?("#{prefix}.")
+        elsif pattern.end_with?(".*")
+          prefix = pattern.delete_suffix(".*")
+          suffix = key.delete_prefix("#{prefix}.")
+          key != suffix && !suffix.include?(".")
+        else
+          key == pattern
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/registry.rb

@@ -0,0 +1,21 @@
+module Textus
+  module Workflow
+    class Registry
+      def initialize
+        @definitions = []
+      end
+
+      def register(definition)
+        @definitions << definition
+      end
+
+      def for(key)
+        @definitions.find { |d| d.match?(key) }
+      end
+
+      def all
+        @definitions.dup
+      end
+    end
+  end
+end

data/lib/textus/workflow/runner.rb

@@ -0,0 +1,84 @@
+require "timeout"
+
+module Textus
+  module Workflow
+    class Runner
+      DEFAULT_TIMEOUT = 30
+
+      def initialize(definition, container:, call:)
+        @definition = definition
+        @container  = container
+        @call       = call
+      end
+
+      def run(key)
+        ctx  = build_context(key)
+        data = execute_steps(ctx)
+        publish(key, data, ctx)
+        data
+      end
+
+      private
+
+      def build_context(key)
+        res = @container.manifest.resolver.resolve(key)
+        Context.new(
+          key: key,
+          entry: res.entry,
+          config: {}.freeze,
+          lane: res.entry.lane.to_s,
+          container: @container,
+          call: @call,
+        )
+      end
+
+      def execute_steps(ctx)
+        data = nil
+        @definition.steps.each { |step| data = execute_one(step, data, ctx) }
+        data
+      end
+
+      def execute_one(step, data, ctx)
+        timeout = step.timeout || DEFAULT_TIMEOUT
+        Timeout.timeout(timeout) { step.callable.call(data, ctx) }
+      rescue Timeout::Error => e
+        raise StepFailed.new(step.name, e)
+      rescue Textus::Error
+        raise
+      rescue StandardError => e
+        raise StepFailed.new(step.name, e)
+      end
+
+      def publish(key, data, ctx)
+        blk = @definition.publish_block
+        return blk.call(data, ctx) if blk && blk != :default
+
+        built_in_publish(key, data, ctx)
+      end
+
+      def built_in_publish(key, data, ctx)
+        normalized = normalize(data, ctx.entry.format)
+        Gate::Auth.new(@container).check_action!(action: :converge, actor: @call.role, key: key)
+        Envelope::Writer.from(container: @container, call: @call).put(
+          key,
+          mentry: ctx.entry,
+          payload: Envelope::Writer::Payload.new(**normalized),
+        )
+      end
+
+      def normalize(data, format)
+        return { meta: {}, body: "", content: nil } if data.nil?
+
+        data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+        case format.to_s
+        when "markdown", "text"
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        when "json", "yaml"
+          { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+        else
+          { meta: {}, body: data.to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow.rb

@@ -0,0 +1,28 @@
+module Textus
+  module Workflow
+    class StepFailed < Textus::Error
+      attr_reader :step_name, :cause
+
+      def initialize(step_name, cause)
+        @step_name = step_name
+        @cause     = cause
+        super(:workflow_step_failed, "workflow step '#{step_name}' failed: #{cause.message}")
+      end
+    end
+
+    class NotFound < Textus::Error
+      def initialize(key)
+        super(:workflow_not_found, "no workflow matches key '#{key}'; add a .textus/workflows/*.rb file with match: '#{key}'")
+      end
+    end
+  end
+
+  def self.workflow(name, &)
+    collector = Workflow::Collector.current
+    raise "Textus.workflow called outside Workflow::Loader.load_all context" unless collector
+
+    defn = Workflow::DSL::Definition.new(name)
+    defn.instance_eval(&)
+    collector.register(defn)
+  end
+end

data/lib/textus.rb

@@ -9,17 +9,19 @@ loader.inflector.inflect(
   "cli" => "CLI",
   "json" => "Json",
   "yaml" => "Yaml",
-  "io" => "IO",
   "mcp" => "MCP",
   "mcp_serve" => "MCPServe",
+  "dsl" => "DSL",
 )
 loader.ignore(File.expand_path("textus/errors.rb", __dir__))
 loader.ignore(File.expand_path("textus/surfaces/mcp.rb", __dir__))
 loader.ignore(File.expand_path("textus/surfaces/mcp/errors.rb", __dir__))
+loader.ignore(File.expand_path("textus/workflow/errors.rb", __dir__))
 # Scaffold sources copied verbatim into user stores by `textus init`. They are
 # templates for user-owned step classes, not gem constants — Zeitwerk must not
 # manage or eager-load them.
 loader.ignore(File.expand_path("textus/init/templates", __dir__))
+loader.ignore(File.expand_path("textus/produce/acquire", __dir__))
 loader.setup
 loader.eager_load
 
@@ -33,6 +35,7 @@ Textus::Action::VERBS = {
   reject: Textus::Action::Reject,
   enqueue: Textus::Action::Enqueue,
   get: Textus::Action::Get,
+  ingest: Textus::Action::Ingest,
   list: Textus::Action::List,
   where: Textus::Action::Where,
   uid: Textus::Action::Uid,
@@ -87,4 +90,12 @@ Textus::Action::VERBS.each_key do |verb|
 end
 
 module Textus
+  def self.workflow(name, &)
+    collector = Workflow::Collector.current
+    raise "Textus.workflow called outside Workflow::Loader.load_all context" unless collector
+
+    defn = Workflow::DSL::Definition.new(name)
+    defn.instance_eval(&)
+    collector.register(defn)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: textus
 version: !ruby/object:Gem::Version
-  version: 0.53.0
+  version: 0.54.0
 platform: ruby
 authors:
 - Patrick
@@ -135,6 +135,7 @@ files:
 - lib/textus/action/drain.rb
 - lib/textus/action/enqueue.rb
 - lib/textus/action/get.rb
+- lib/textus/action/ingest.rb
 - lib/textus/action/jobs.rb
 - lib/textus/action/key_delete.rb
 - lib/textus/action/key_delete_prefix.rb
@@ -144,7 +145,6 @@ files:
 - lib/textus/action/propose.rb
 - lib/textus/action/published.rb
 - lib/textus/action/pulse.rb
-- lib/textus/action/pulse/scanner.rb
 - lib/textus/action/put.rb
 - lib/textus/action/rdeps.rb
 - lib/textus/action/reject.rb
@@ -155,15 +155,6 @@ files:
 - lib/textus/action/uid.rb
 - lib/textus/action/where.rb
 - lib/textus/action/write_verb.rb
-- lib/textus/background/job.rb
-- lib/textus/background/job/base.rb
-- lib/textus/background/job/materialize.rb
-- lib/textus/background/job/refresh.rb
-- lib/textus/background/job/sweep.rb
-- lib/textus/background/plan.rb
-- lib/textus/background/planner/plan.rb
-- lib/textus/background/retention/apply.rb
-- lib/textus/background/worker.rb
 - lib/textus/boot.rb
 - lib/textus/call.rb
 - lib/textus/command.rb
@@ -187,15 +178,14 @@ files:
 - lib/textus/doctor/check.rb
 - lib/textus/doctor/check/audit_log.rb
 - lib/textus/doctor/check/generator_drift.rb
-- lib/textus/doctor/check/handler_permit.rb
-- lib/textus/doctor/check/hooks.rb
 - lib/textus/doctor/check/illegal_keys.rb
-- lib/textus/doctor/check/intake_registration.rb
 - lib/textus/doctor/check/manifest_files.rb
+- lib/textus/doctor/check/notebook_sources.rb
 - lib/textus/doctor/check/orphaned_publish_targets.rb
 - lib/textus/doctor/check/proposal_targets.rb
 - lib/textus/doctor/check/protocol_version.rb
 - lib/textus/doctor/check/publish_tree_index_overlap.rb
+- lib/textus/doctor/check/raw_asset_paths.rb
 - lib/textus/doctor/check/rule_ambiguity.rb
 - lib/textus/doctor/check/schema_parse_error.rb
 - lib/textus/doctor/check/schema_violations.rb
@@ -204,23 +194,28 @@ files:
 - lib/textus/doctor/check/templates.rb
 - lib/textus/doctor/check/unowned_schema_fields.rb
 - lib/textus/doctor/validator.rb
-- lib/textus/entry.rb
-- lib/textus/entry/base.rb
-- lib/textus/entry/json.rb
-- lib/textus/entry/markdown.rb
-- lib/textus/entry/text.rb
-- lib/textus/entry/yaml.rb
 - lib/textus/envelope.rb
-- lib/textus/envelope/io/reader.rb
-- lib/textus/envelope/io/writer.rb
+- lib/textus/envelope/reader.rb
+- lib/textus/envelope/writer.rb
 - lib/textus/errors.rb
 - lib/textus/etag.rb
-- lib/textus/events.rb
+- lib/textus/format.rb
+- lib/textus/format/base.rb
+- lib/textus/format/json.rb
+- lib/textus/format/markdown.rb
+- lib/textus/format/text.rb
+- lib/textus/format/yaml.rb
 - lib/textus/gate.rb
 - lib/textus/gate/auth.rb
 - lib/textus/init.rb
-- lib/textus/init/templates/machine_intake.rb
-- lib/textus/init/templates/orientation_reducer.rb
+- lib/textus/jobs.rb
+- lib/textus/jobs/base.rb
+- lib/textus/jobs/materialize.rb
+- lib/textus/jobs/plan.rb
+- lib/textus/jobs/planner.rb
+- lib/textus/jobs/retention.rb
+- lib/textus/jobs/sweep.rb
+- lib/textus/jobs/worker.rb
 - lib/textus/key/distance.rb
 - lib/textus/key/grammar.rb
 - lib/textus/key/matching.rb
@@ -244,12 +239,10 @@ files:
 - lib/textus/manifest/entry/publish/to_paths.rb
 - lib/textus/manifest/entry/publish/tree.rb
 - lib/textus/manifest/entry/validators.rb
-- lib/textus/manifest/entry/validators/events.rb
 - lib/textus/manifest/entry/validators/format_matrix.rb
 - lib/textus/manifest/entry/validators/ignore.rb
 - lib/textus/manifest/entry/validators/publish.rb
 - lib/textus/manifest/policy.rb
-- lib/textus/manifest/policy/handler_permit.rb
 - lib/textus/manifest/policy/matcher.rb
 - lib/textus/manifest/policy/publish_target.rb
 - lib/textus/manifest/policy/react.rb
@@ -261,53 +254,23 @@ files:
 - lib/textus/manifest/schema/keys.rb
 - lib/textus/manifest/schema/validator.rb
 - lib/textus/manifest/schema/vocabulary.rb
-- lib/textus/pipeline/acquire/handler.rb
-- lib/textus/pipeline/acquire/intake.rb
-- lib/textus/pipeline/acquire/projection.rb
-- lib/textus/pipeline/acquire/serializer.rb
-- lib/textus/pipeline/acquire/serializer/json.rb
-- lib/textus/pipeline/acquire/serializer/text.rb
-- lib/textus/pipeline/acquire/serializer/yaml.rb
-- lib/textus/pipeline/engine.rb
-- lib/textus/pipeline/render.rb
 - lib/textus/ports/audit_log.rb
-- lib/textus/ports/audit_subscriber.rb
 - lib/textus/ports/build_lock.rb
 - lib/textus/ports/clock.rb
+- lib/textus/ports/job_store.rb
+- lib/textus/ports/job_store/job.rb
 - lib/textus/ports/publisher.rb
-- lib/textus/ports/queue.rb
-- lib/textus/ports/queue/job.rb
 - lib/textus/ports/sentinel_store.rb
 - lib/textus/ports/storage/file_stat.rb
 - lib/textus/ports/storage/file_store.rb
 - lib/textus/ports/watcher_lock.rb
-- lib/textus/projection.rb
+- lib/textus/produce/engine.rb
+- lib/textus/produce/render.rb
 - lib/textus/role.rb
 - lib/textus/schema.rb
 - lib/textus/schema/tools.rb
 - lib/textus/schemas.rb
 - lib/textus/session.rb
-- lib/textus/step.rb
-- lib/textus/step/base.rb
-- lib/textus/step/builtin.rb
-- lib/textus/step/builtin/csv_fetch.rb
-- lib/textus/step/builtin/ical_events_fetch.rb
-- lib/textus/step/builtin/json_fetch.rb
-- lib/textus/step/builtin/markdown_links_fetch.rb
-- lib/textus/step/builtin/rss_fetch.rb
-- lib/textus/step/catalog.rb
-- lib/textus/step/context.rb
-- lib/textus/step/discovery.rb
-- lib/textus/step/error_log.rb
-- lib/textus/step/event_bus.rb
-- lib/textus/step/fetch.rb
-- lib/textus/step/fire_report.rb
-- lib/textus/step/loader.rb
-- lib/textus/step/observe.rb
-- lib/textus/step/registry_store.rb
-- lib/textus/step/signature.rb
-- lib/textus/step/transform.rb
-- lib/textus/step/validate.rb
 - lib/textus/store.rb
 - lib/textus/surfaces/cli.rb
 - lib/textus/surfaces/cli/group.rb
@@ -330,6 +293,7 @@ files:
 - lib/textus/surfaces/mcp.rb
 - lib/textus/surfaces/mcp/catalog.rb
 - lib/textus/surfaces/mcp/errors.rb
+- lib/textus/surfaces/mcp/routing.rb
 - lib/textus/surfaces/mcp/server.rb
 - lib/textus/surfaces/mcp/session.rb
 - lib/textus/surfaces/mcp/tool_schemas.rb
@@ -337,6 +301,14 @@ files:
 - lib/textus/surfaces/watcher.rb
 - lib/textus/uid.rb
 - lib/textus/version.rb
+- lib/textus/workflow.rb
+- lib/textus/workflow/collector.rb
+- lib/textus/workflow/context.rb
+- lib/textus/workflow/dsl.rb
+- lib/textus/workflow/loader.rb
+- lib/textus/workflow/pattern.rb
+- lib/textus/workflow/registry.rb
+- lib/textus/workflow/runner.rb
 homepage: https://github.com/patrick204nqh/textus
 licenses:
 - MIT

data/lib/textus/action/pulse/scanner.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-require "time"
-
-module Textus
-  module Action
-    class Pulse
-      class Scanner
-        def initialize(prefix: nil, lane: nil)
-          @prefix = prefix
-          @lane   = lane
-        end
-
-        def call(container:, call:)
-          @container  = container
-          @call       = call
-          @manifest   = container.manifest
-          @file_store = container.file_store
-
-          rows = []
-          @manifest.data.entries.each do |mentry|
-            next if @prefix && !mentry.key.start_with?(@prefix)
-            next if @lane   && mentry.lane != @lane
-
-            rows << row_for(mentry)
-          end
-          rows
-        end
-
-        private
-
-        def row_for(mentry)
-          envelope = safe_get(mentry.key)
-          last = envelope&.meta&.dig("last_fetched_at")
-          ttl, action = policy_for(mentry)
-          return base_row(mentry, last).merge(status: :no_policy) if ttl.nil?
-
-          basis = basis_for(mentry)
-          expired = expired?(mentry, basis, ttl)
-          base_row(mentry, last).merge(
-            ttl_seconds: ttl,
-            action: action,
-            status: expired ? :expired : :fresh,
-            next_due_at: basis.nil? ? nil : (basis + ttl).utc.iso8601,
-          )
-        end
-
-        def policy_for(mentry)
-          if mentry.intake?
-            ttl = mentry.source.ttl_seconds
-            return [ttl, :refresh] unless ttl.nil?
-          end
-          ret = @manifest.rules.for(mentry.key).retention
-          return [ret.ttl_seconds, ret.action] unless ret.nil?
-
-          [nil, nil]
-        end
-
-        def basis_for(mentry)
-          return evaluator.intake_basis(mentry) if mentry.intake? && mentry.source.ttl_seconds
-
-          mtime_for(mentry.key)
-        end
-
-        def expired?(mentry, basis, ttl)
-          if mentry.intake? && mentry.source.ttl_seconds
-            evaluator.verdict(mentry).stale
-          else
-            basis.nil? || Textus::Core::Retention::Sweep.expired?(ttl_seconds: ttl, mtime: basis, now: @call.now)
-          end
-        end
-
-        def evaluator
-          @evaluator ||= Textus::Core::Freshness::Evaluator.new(
-            manifest: @manifest,
-            file_stat: Textus::Ports::Storage::FileStat.new,
-            clock: @call,
-          )
-        end
-
-        def mtime_for(key)
-          path = @manifest.resolver.resolve(key).path
-          @file_store.exists?(path) ? Textus::Ports::Storage::FileStat.new.mtime(path) : nil
-        rescue Textus::Error
-          nil
-        end
-
-        def base_row(mentry, last)
-          {
-            key: mentry.key,
-            lane: mentry.lane,
-            last_fetched_at: last,
-            age_seconds: last ? (@call.now - Time.parse(last)).to_i : nil,
-          }
-        end
-
-        def safe_get(key)
-          res = @manifest.resolver.resolve(key)
-          return nil unless @file_store.exists?(res.path)
-
-          raw = @file_store.read(res.path)
-          parsed = Textus::Entry.for_format(res.entry.format).parse(raw, path: res.path)
-          Textus::Envelope.build(
-            key: key,
-            mentry: res.entry,
-            path: res.path,
-            meta: parsed["_meta"],
-            body: parsed["body"],
-            etag: Textus::Etag.for_bytes(raw),
-            content: parsed["content"],
-          )
-        rescue Textus::Error
-          nil
-        end
-      end
-    end
-  end
-end

data/lib/textus/background/job/base.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Background
-    module Job
-      class Base
-        def self.inherited(subclass)
-          super
-          return unless subclass.name
-
-          TracePoint.new(:end) do |tp|
-            if tp.self == subclass
-              Textus::Background::Job.register(subclass)
-              tp.disable
-            end
-          end.enable
-        end
-
-        def call(**)
-          raise NotImplementedError.new("#{self.class}#call")
-        end
-
-        def args = {}
-      end
-    end
-  end
-end

data/lib/textus/background/job/materialize.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Background
-    module Job
-      class Materialize < Base
-        TYPE = "materialize"
-
-        def initialize(key:)
-          super()
-          @key = key
-        end
-
-        def args = { key: @key }
-
-        def call(container:, call:)
-          result = Textus::Pipeline::Engine.converge(container: container, call: call, keys: [@key])
-          return unless result.is_a?(Hash)
-
-          Array(result[:failed]).each do |failure|
-            container.steps.publish(
-              :produce_failed,
-              ctx: Textus::Step::Context.for(container: container, call: call),
-              keys: [failure["key"]], error: failure["error"]
-            )
-          end
-        end
-      end
-    end
-  end
-end

data/lib/textus/background/job/refresh.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Background
-    module Job
-      class Refresh < Base
-        TYPE = "refresh"
-
-        def initialize(key:)
-          super()
-          @key = key
-        end
-
-        def args = { key: @key }
-
-        def call(container:, call:)
-          Textus::Pipeline::Engine.converge(container: container, call: call, keys: [@key])
-        end
-      end
-    end
-  end
-end

data/lib/textus/background/job/sweep.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  module Background
-    module Job
-      class Sweep < Base
-        REQUIRED_ROLE = Textus::Role::AUTOMATION
-        TYPE = "sweep"
-
-        def initialize(scope: nil, key: nil)
-          super()
-          @scope = scope || {}
-          @key   = key
-        end
-
-        def args = { scope: @scope, key: @key }.compact
-
-        def call(container:, call:)
-          prefix = @key || (@scope.is_a?(Hash) ? @scope["prefix"] : nil)
-          lane   = @scope.is_a?(Hash) ? @scope["lane"] : nil
-          rows = Textus::Core::Retention::Sweep.new(
-            manifest: container.manifest,
-            file_stat: Textus::Ports::Storage::FileStat.new,
-            clock: Textus::Ports::Clock.new,
-          ).call(prefix: prefix, lane: lane)
-          Textus::Background::Retention::Apply.new(container: container, call: call).call(rows)
-        end
-      end
-    end
-  end
-end