textus 0.55.1 → 0.56.0

data/lib/textus/workflow/consumer.rb

@@ -0,0 +1,39 @@
+module Textus
+  module Workflow
+    class Consumer
+      BACKOFF_DELAYS = [10, 30, 60].freeze
+
+      def initialize(registry:, container:, call:)
+        @registry = registry
+        @container = container
+        @call = call
+      end
+
+      def consume(row, workflow_queue)
+        workflow_queue.mark_processing(row[:id])
+        workflow = @registry.for(row[:workflow_type])
+        Runner.new(workflow, container: @container, call: @call).run(row[:key])
+        workflow_queue.complete(row[:id])
+        row.merge(state: "done")
+      rescue StandardError => e
+        handle_failure(row, workflow_queue, e)
+      end
+
+      private
+
+      def handle_failure(row, workflow_queue, error)
+        retry_count = (row[:retry_count] || 0) + 1
+        max_attempts = row[:max_attempts] || 3
+
+        if retry_count >= max_attempts
+          workflow_queue.fail(row[:id], retry_count:, error: error.message)
+          row.merge(state: "failed", error: error.message)
+        else
+          index = [retry_count - 1, BACKOFF_DELAYS.length - 1].min
+          workflow_queue.reschedule(row[:id], retry_count:, delay: BACKOFF_DELAYS[index], error: error.message)
+          row.merge(state: "retrying")
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/dsl/lifecycle.rb

@@ -0,0 +1,32 @@
+module Textus
+  module Workflow
+    module DSL
+      module Lifecycle
+        VALID_EXPIRE_ACTIONS = %i[drop archive].freeze
+
+        def ttl_value(cadence)
+          return nil if cadence == :__no_arg__
+
+          Textus::Value::Duration.seconds(cadence)
+        end
+
+        def expire_action(action = :__no_arg__)
+          return nil if action == :__no_arg__
+
+          sym = action.to_s.to_sym
+          raise ArgumentError.new("on_expire must be one of drop|archive, got #{action.inspect}") unless VALID_EXPIRE_ACTIONS.include?(sym)
+
+          sym
+        end
+
+        def max_attempts_value(n)
+          n.to_i
+        end
+
+        def priority_value(n)
+          n.to_i
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/dsl/pattern_matcher.rb

@@ -0,0 +1,20 @@
+module Textus
+  module Workflow
+    module DSL
+      module PatternMatcher
+        def match_pattern?(pattern, key)
+          if pattern.end_with?(".**")
+            prefix = pattern.delete_suffix(".**")
+            key.start_with?("#{prefix}.")
+          elsif pattern.end_with?(".*")
+            prefix = pattern.delete_suffix(".*")
+            suffix = key.delete_prefix("#{prefix}.")
+            key != suffix && !suffix.include?(".")
+          else
+            key == pattern
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/dsl/step_builder.rb

@@ -0,0 +1,56 @@
+module Textus
+  module Workflow
+    module DSL
+      module StepBuilder
+        def build_step(name, callable = nil, timeout: nil, &block)
+          if callable.is_a?(Hash)
+            raise ArgumentError.new(
+              "step :#{name} no longer accepts options hash; pass timeout: keyword instead",
+            )
+          end
+
+          resolved = if callable.respond_to?(:call)
+                       callable
+                     elsif block
+                       block
+                     else
+                       raise ArgumentError.new("step :#{name} requires a block or a callable (got neither)")
+                     end
+
+          Step.new(name: name, callable: resolved, timeout: timeout)
+        end
+
+        def build_check(name, naming: nil, scope: nil, &block)
+          merge = lambda { |data, new_issues|
+            existing = Array(data&.dig("content", "issues"))
+            all = existing + new_issues
+            { "content" => { "ok" => all.empty?, "issues" => all, "count" => all.size } }
+          }
+
+          if naming
+            raise ArgumentError.new("check :#{name} requires a scope: for naming checks") unless scope
+
+            lambda { |data, ctx|
+              entries = ctx.container.manifest.resolver.enumerate.select do |row|
+                match_pattern?(scope, row[:key].to_s)
+              end
+              new_issues = check_naming(
+                entries, pattern: naming, code: "check.#{name}",
+                         fix: "rename to match #{naming.inspect}"
+              )
+              merge.call(data, new_issues)
+            }
+          elsif block
+            ->(data, ctx) { merge.call(data, block.call(ctx) || []) }
+          else
+            raise ArgumentError.new("check :#{name} requires naming: or a block")
+          end
+        end
+
+        def build_parallel(steps)
+          Parallel.new(steps: steps)
+        end
+      end
+    end
+  end
+end

data/lib/textus/workflow/dsl.rb

@@ -2,41 +2,131 @@ module Textus
   module Workflow
     module DSL
       Step = Data.define(:name, :callable, :timeout)
+      Parallel = Data.define(:steps)
+      PublishTarget = Data.define(:to, :template, :tree, :inject_boot)
 
       class Definition
-        attr_reader :name, :steps, :match_pattern, :publish_block
+        include StepBuilder
+        include PatternMatcher
+        include Lifecycle
+
+        attr_reader :name, :steps, :on_patterns, :save_to, :notify_keys,
+                    :publish_block, :publish_targets,
+                    :handled_events, :workflow_type
+
+        def self.register(defn)
+          @definitions ||= []
+          @definitions << defn
+        end
+
+        def self.definitions
+          @definitions || []
+        end
+
+        def self.clear_registered
+          @definitions = []
+        end
 
         def initialize(name)
-          @name          = name
-          @steps         = []
-          @match_pattern = nil
-          @publish_block = nil
+          @name                = name
+          @steps               = []
+          @on_patterns         = []
+          @save_to             = nil
+          @notify_keys         = []
+          @publish_block       = nil
+          @publish_targets     = []
+          @priority            = 0
+          @handled_events      = []
+          @max_attempts        = 3
+          @workflow_type       = name
         end
 
-        def match(pattern)
-          @match_pattern = pattern
+        # ── Event handling (internal) ──
+        def handles(*event_types)
+          @handled_events.concat(event_types.map(&:to_s).freeze)
         end
 
-        def step(name, callable_or_opt = nil, timeout: nil, &block)
-          callable = if callable_or_opt.respond_to?(:call)
-                       callable_or_opt
-                     elsif block
-                       block
-                     else
-                       raise ArgumentError.new("step :#{name} requires a block or a callable (got neither)")
-                     end
-          t = callable_or_opt.is_a?(Hash) ? callable_or_opt[:timeout] : timeout
-          @steps << Step.new(name: name, callable: callable, timeout: t)
+        def type
+          @workflow_type
         end
 
-        def publish(&block)
-          @publish_block = block || :default
+        def max_attempts(n = nil)
+          return @max_attempts if n.nil?
+
+          @max_attempts = max_attempts_value(n)
         end
 
+        # ── Trigger ──
+        def on(*patterns)
+          @on_patterns = patterns
+        end
+
+        # ── Steps ──
+        def step(name, callable = nil, timeout: nil, &)
+          @steps << build_step(name, callable, timeout:, &)
+        end
+
+        def check(name, naming: nil, scope: nil, &)
+          callable = build_check(name, naming:, scope:, &)
+          @steps << Step.new(name: :"check_#{name}", callable: callable, timeout: nil)
+        end
+
+        def parallel(&)
+          saved = @steps
+          @steps = []
+          instance_eval(&)
+          parallel_steps = @steps.dup
+          @steps = saved << build_parallel(parallel_steps)
+        end
+
+        # ── Output ──
+        def publish(to: nil, template: nil, tree: nil, inject_boot: nil, &block)
+          if block
+            @publish_block = block
+          elsif to || tree
+            @publish_targets << PublishTarget.new(to: to, template: template, tree: tree, inject_boot: inject_boot)
+          else
+            raise ArgumentError.new("publish requires to: (file), tree:, or a block; use save to: for store keys")
+          end
+        end
+
+        def save(to:)
+          @save_to = to
+        end
+
+        def notify(key)
+          @notify_keys << key
+        end
+
+        # ── Ordering ──
+        def priority(n = nil)
+          return @priority if n.nil?
+
+          @priority = priority_value(n)
+        end
+
+        # ── Lifecycle ──
+        def every(cadence = :__no_arg__)
+          return @ttl if cadence == :__no_arg__
+
+          @ttl = ttl_value(cadence)
+        end
+
+        alias ttl every
+
+        def on_expire(action = :__no_arg__)
+          @on_expire = expire_action(action)
+        end
+
+        # ── Matching (internal) ──
         def match?(key)
-          return false unless @match_pattern
+          return false unless @on_patterns&.any?
+
+          @on_patterns.any? { |p| match_pattern?(p, key) }
+        end
 
-          Pattern.match?(@match_pattern, key)
+        def multi_match?
+          @on_patterns&.any? { |p| p.end_with?("**") }
         end
       end
     end

data/lib/textus/workflow/event_emitter.rb

@@ -0,0 +1,21 @@
+module Textus
+  module Workflow
+    class EventEmitter
+      def initialize(workflow_queue, registry:)
+        @workflow_queue = workflow_queue
+        @registry = registry
+      end
+
+      def emit(event_type:, key:, lane: "artifacts")
+        workflows = @registry.workflows_for(event_type)
+        workflows.each do |wf|
+          @workflow_queue.push(
+            event_type: event_type.to_s, workflow_type: wf.type,
+            key:, lane:, max_attempts: wf.max_attempts
+          )
+        end
+        workflows.size
+      end
+    end
+  end
+end

data/lib/textus/workflow/publisher.rb

@@ -0,0 +1,59 @@
+module Textus
+  module Workflow
+    class Publisher
+      def initialize(container:, call:)
+        @container    = container
+        @call         = call
+        @store_engine = container.store_engine
+      end
+
+      def publish(key, data, ctx, definition)
+        definition.publish_targets.each { |t| publish_one(key, data, ctx, target: t) }
+
+        write_to_key(definition.save_to, data, ctx) if definition.save_to
+
+        definition.notify_keys.each { |nk| notify_key(nk, ctx) }
+
+        definition.publish_block&.call(data, ctx)
+      end
+
+      private
+
+      def write_to_key(target_key, data, ctx) # rubocop:disable Lint/UnusedMethodArgument
+        res = @container.manifest.resolver.resolve(target_key)
+        normalized = Textus::Protocol::Format.data_to_payload(data, res.entry.format)
+        meta = stamped_meta(normalized[:meta])
+        @store_engine.put(key: target_key, meta:, body: normalized[:body], content: normalized[:content], call: @call)
+      end
+
+      def notify_key(target_key, ctx) # rubocop:disable Lint/UnusedMethodArgument
+        @store_engine.put(key: target_key, meta: {}, body: nil, content: {}, call: @call)
+      end
+
+      def write_and_pctx(key, data, ctx)
+        normalized = Textus::Protocol::Format.data_to_payload(data, ctx.entry.format)
+        meta = stamped_meta(normalized[:meta])
+        @store_engine.put(key:, meta:, body: normalized[:body], content: normalized[:content], call: @call)
+        Textus::Protocol::Manifest::Entry::Base::PublishContext.new(
+          container: @container,
+          reader: ->(key:) { @store_engine.read(key:) },
+        )
+      end
+
+      def publish_one(key, data, ctx, target:)
+        pctx = write_and_pctx(key, data, ctx)
+        attrs = { "to" => target.to, "template" => target.template,
+                  "tree" => target.tree }
+        attrs["inject_boot"] = true if target.inject_boot
+        pub_target = Textus::Protocol::Manifest::Policy::PublishTarget.new(attrs)
+        entry = ctx.entry.with_publish_targets([pub_target])
+        entry.publish_via(pctx)
+      end
+
+      def stamped_meta(meta)
+        now = Time.now.utc.iso8601
+        (meta || {}).merge("generated_at" => now)
+      end
+    end
+  end
+end

data/lib/textus/workflow/registry.rb

@@ -1,20 +1,43 @@
+# frozen_string_literal: true
+
 module Textus
   module Workflow
     class Registry
-      def initialize
-        @definitions = []
+      class UnknownWorkflow < KeyError
+      end
+
+      def self.load_all(root)
+        geometry = Textus::Protocol::Layout.new(root)
+        return [] unless File.directory?(geometry.workflow_dir)
+
+        Workflow::DSL::Definition.clear_registered
+        Dir.glob(File.join(geometry.workflow_dir, "**", "*.rb")).each { |path| load path }
+        Workflow::DSL::Definition.definitions
+      end
+
+      attr_reader :definitions
+
+      def initialize(workflow_definitions)
+        @definitions = workflow_definitions
+        @by_event = {}
+        @by_type  = {}
+
+        workflow_definitions.each do |wf|
+          @by_type[wf.type] = wf
+          wf.handled_events.each { |ev| (@by_event[ev] ||= []) << wf }
+        end
       end
 
-      def register(definition)
-        @definitions << definition
+      def workflows_for(event_type)
+        @by_event.fetch(event_type.to_s, [])
       end
 
-      def for(key)
-        @definitions.find { |d| d.match?(key) }
+      def for(workflow_type)
+        @by_type[workflow_type.to_s] or raise(UnknownWorkflow.new("Unknown workflow type: #{workflow_type}"))
       end
 
-      def all
-        @definitions.dup
+      def match_for_key(key)
+        @definitions.find { |wf| wf.match?(key) }
       end
     end
   end

data/lib/textus/workflow/runner.rb

@@ -1,28 +1,38 @@
-require "timeout"
-
 module Textus
   module Workflow
+    class ParallelStepFailed < Textus::Error
+      attr_reader :failures
+
+      def initialize(failures)
+        @failures = failures
+        msgs = failures.map { |f| "#{f[:step]}: #{f[:error]}" }.join("; ")
+        super(:workflow_parallel_step_failed, "parallel step(s) failed: #{msgs}")
+      end
+    end
+
     class Runner
-      DEFAULT_TIMEOUT = 30
+      Context = Data.define(:key, :entry, :config, :lane, :container, :call) do
+        def store_engine = container.store_engine
+        def manifest     = container.manifest
+        def layout       = container.layout
+        def file_system  = container.store.file_system
+        def schemas      = container.schemas
+        def read(key)    = store_engine.read(key:)
+        def repo_root    = File.dirname(container.root)
+        def store_root   = container.root
+      end
 
       def initialize(definition, container:, call:)
         @definition = definition
         @container  = container
         @call       = call
+        @publisher  = Publisher.new(container: container, call: call)
+        @step_executor = StepExecutor.new
       end
 
       def run(key)
-        ctx  = build_context(key)
-        data = execute_steps(ctx)
-        publish(key, data, ctx)
-        data
-      end
-
-      private
-
-      def build_context(key)
         res = @container.manifest.resolver.resolve(key)
-        Context.new(
+        ctx = Context.new(
           key: key,
           entry: res.entry,
           config: {}.freeze,
@@ -30,56 +40,10 @@ module Textus
           container: @container,
           call: @call,
         )
-      end
-
-      def execute_steps(ctx)
-        data = nil
-        @definition.steps.each { |step| data = execute_one(step, data, ctx) }
+        data = @step_executor.execute(@definition.steps, ctx)
+        @publisher.publish(key, data, ctx, @definition) if data
         data
       end
-
-      def execute_one(step, data, ctx)
-        timeout = step.timeout || DEFAULT_TIMEOUT
-        Timeout.timeout(timeout) { step.callable.call(data, ctx) }
-      rescue Timeout::Error => e
-        raise StepFailed.new(step.name, e)
-      rescue Textus::Error
-        raise
-      rescue StandardError => e
-        raise StepFailed.new(step.name, e)
-      end
-
-      def publish(key, data, ctx)
-        blk = @definition.publish_block
-        return blk.call(data, ctx) if blk && blk != :default
-
-        built_in_publish(key, data, ctx)
-      end
-
-      def built_in_publish(key, data, ctx)
-        normalized = Textus::Format.data_to_payload(data, ctx.entry.format)
-        Gate::Auth.new(@container).check_action!(action: :converge, actor: @call.role, key: key)
-        Textus::Store::Envelope::Writer.from(container: @container, call: @call).put(
-          key,
-          mentry: ctx.entry,
-          payload: Textus::Store::Envelope::Writer::Payload.new(**normalized),
-        )
-        publish_external(key, ctx)
-      end
-
-      def publish_external(key, ctx)
-        entry = ctx.entry
-        return unless entry.publish_tree || !Array(entry.publish_to).empty?
-
-        entry_path = @container.manifest.resolver.resolve(key).path
-        return unless entry.publish_tree || File.exist?(entry_path)
-
-        reader = Textus::Store::Envelope::Reader.from(container: @container)
-        pctx   = Textus::Manifest::Entry::Base::PublishContext.new(
-          container: @container, call: @call, reader: reader.method(:read),
-        )
-        entry.publish_via(pctx)
-      end
     end
   end
 end

data/lib/textus/workflow/scheduler.rb

@@ -0,0 +1,45 @@
+module Textus
+  module Workflow
+    class Scheduler
+      def initialize(registry:, workflow_queue:, reader:)
+        @registry = registry
+        @workflow_queue = workflow_queue
+        @reader = reader
+      end
+
+      def seed_expired
+        seeded = 0
+        @registry.definitions.each do |wf|
+          next unless wf.ttl && wf.save_to
+
+          lane = wf.save_to.split(".").first
+          next if @workflow_queue.pending?(event_type: "workflow", key: wf.save_to)
+
+          entry = @reader.call(key: wf.save_to)
+          unless entry
+            @workflow_queue.push(
+              event_type: "workflow", workflow_type: wf.type,
+              key: wf.save_to, lane: lane
+            )
+            seeded += 1
+            next
+          end
+
+          meta = entry.respond_to?(:meta) ? entry.meta : (entry["_meta"] || {})
+          generated_at = meta["generated_at"]
+          next unless generated_at
+
+          age = Time.now - Time.parse(generated_at)
+          next unless age > wf.ttl
+
+          @workflow_queue.push(
+            event_type: "workflow", workflow_type: wf.type,
+            key: wf.save_to, lane: lane
+          )
+          seeded += 1
+        end
+        seeded
+      end
+    end
+  end
+end

data/lib/textus/workflow/step_executor.rb

@@ -0,0 +1,78 @@
+require "timeout"
+require "concurrent"
+
+module Textus
+  module Workflow
+    class StepExecutor
+      DEFAULT_TIMEOUT = 30
+      CONCURRENCY_ADAPTER = Textus::Infra::Adapters::ConcurrencyAdapter.new
+
+      class StepScope
+        include Textus::Workflow::StepHelpers
+
+        def initialize(data, ctx)
+          @data = data
+          @ctx  = ctx
+        end
+
+        def call(callable)
+          instance_exec(@data, @ctx, &callable)
+        end
+      end
+
+      def execute(steps, ctx)
+        data = nil
+        steps.each { |step| data = execute_one(step, data, ctx) }
+        data
+      end
+
+      private
+
+      def execute_one(step, data, ctx)
+        case step
+        when DSL::Step then execute_single(step, data, ctx)
+        when DSL::Parallel then execute_parallel(step, ctx)
+        else raise ArgumentError.new("unknown step type: #{step.class}")
+        end
+      end
+
+      def execute_single(step, data, ctx)
+        scope = StepScope.new(data, ctx)
+        timeout = step.timeout || DEFAULT_TIMEOUT
+        Timeout.timeout(timeout) { scope.call(step.callable) }
+      rescue Timeout::Error => e
+        raise StepFailed.new(step.name, e)
+      rescue Textus::Error
+        raise
+      rescue StandardError => e
+        raise StepFailed.new(step.name, e)
+      end
+
+      def execute_parallel(parallel, ctx)
+        scope = StepScope.new(nil, ctx)
+        promises = parallel.steps.map do |step|
+          CONCURRENCY_ADAPTER.future do
+            timeout = step.timeout || DEFAULT_TIMEOUT
+            Timeout.timeout(timeout) { scope.call(step.callable) }
+          end
+        end
+
+        results = CONCURRENCY_ADAPTER.zip_futures(*promises).value!
+        failures = []
+        outputs = {}
+        results.each_with_index do |result, i|
+          step = parallel.steps[i]
+          if result.fulfilled?
+            outputs[step.name] = result.value!
+          else
+            failures << { step: step.name, error: result.reason.message }
+          end
+        end
+
+        raise ParallelStepFailed.new(failures) unless failures.empty?
+
+        outputs
+      end
+    end
+  end
+end