igniter 0.4.5 → 0.5.1

data/packages/igniter-core/lib/igniter/core/dataflow/aggregate_state.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Dataflow
+    # Mutable state for a maintained aggregate node.
+    #
+    # Stores per-item contributions so that the aggregate can be updated
+    # incrementally using only the diff (added/changed/removed) from the upstream
+    # incremental collection — not the full item set.
+    #
+    # One AggregateState instance lives on the Execution (keyed by node name) and
+    # persists across update_inputs calls for the lifetime of the contract execution.
+    #
+    # == Update semantics
+    #
+    #   added   → contribute! (project + add into @accum)
+    #   removed → retract!    (subtract from @accum using stored contribution)
+    #   changed → retract!(old) then contribute!(new) — true differential update
+    #   unchanged → no-op
+    #
+    class AggregateState
+      def initialize(operator)
+        @operator      = operator
+        @contributions = {} # key => contribution (what the item "contributes")
+        @accum         = operator.initial
+      end
+
+      # Apply a diff from an IncrementalCollectionResult, updating the aggregate
+      # state in O(changed + added + removed) time.
+      #
+      # @param diff              [Igniter::Dataflow::Diff]
+      # @param collection_result [Igniter::Dataflow::IncrementalCollectionResult]
+      def apply_diff!(diff, collection_result)
+        diff.changed.each do |key|
+          retract!(key)
+          contribute!(key, collection_result[key])
+        end
+        diff.added.each   { |key| contribute!(key, collection_result[key]) }
+        diff.removed.each { |key| retract!(key) }
+      end
+
+      # Current aggregate value (finalized from accumulated state).
+      def value
+        if @operator.recompute
+          @operator.finalize.call(nil, @contributions)
+        else
+          @operator.finalize.call(@accum, @contributions.size)
+        end
+      end
+
+      # Number of items currently tracked in the aggregate.
+      def item_count
+        @contributions.size
+      end
+
+      private
+
+      def contribute!(key, item)
+        contribution = @operator.project.call(item)
+        return if contribution.nil?
+
+        @contributions[key] = contribution
+        return if @operator.recompute
+
+        @accum = @operator.add.call(@accum, contribution)
+      end
+
+      def retract!(key)
+        old_contribution = @contributions.delete(key)
+        return unless old_contribution
+        return if @operator.recompute
+
+        @accum = @operator.remove.call(@accum, old_contribution)
+      end
+    end
+  end
+end

data/packages/igniter-core/lib/igniter/core/dataflow/diff.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Dataflow
+    # Immutable record of what changed in an incremental collection resolve.
+    #
+    # @attr added     [Array<Object>] keys of items added since the last resolve
+    # @attr removed   [Array<Object>] keys of items removed since the last resolve
+    # @attr changed   [Array<Object>] keys of items whose content changed
+    # @attr unchanged [Array<Object>] keys of items that were identical to the last resolve
+    Diff = Struct.new(:added, :removed, :changed, :unchanged, keyword_init: true) do
+      # Returns true if any items were added, removed, or changed.
+      def any_changes?
+        added.any? || removed.any? || changed.any?
+      end
+
+      # Returns the total number of items that needed processing (added + changed).
+      def processed_count
+        added.size + changed.size
+      end
+
+      # Returns a compact human-readable summary.
+      def explain # rubocop:disable Metrics/AbcSize
+        parts = []
+        parts << "added(#{added.size}): #{added.inspect}" unless added.empty?
+        parts << "removed(#{removed.size}): #{removed.inspect}" unless removed.empty?
+        parts << "changed(#{changed.size}): #{changed.inspect}" unless changed.empty?
+        parts << "unchanged(#{unchanged.size})" unless unchanged.empty?
+        parts.empty? ? "(no changes)" : parts.join(", ")
+      end
+
+      def to_h
+        { added: added, removed: removed, changed: changed, unchanged: unchanged }
+      end
+    end
+  end
+end

data/packages/igniter-core/lib/igniter/core/dataflow/diff_state.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Dataflow
+    # Mutable state for an incremental collection node.
+    #
+    # Stores fingerprints of previous items and their resolved CollectionResult::Items
+    # so the resolver can skip unchanged items on subsequent updates.
+    #
+    # One DiffState instance lives on the Execution (keyed by node name) and persists
+    # across update_inputs calls for the lifetime of the contract execution.
+    class DiffState
+      attr_reader :cached_items
+
+      def initialize
+        @snapshots    = {}  # key => fingerprint string
+        @cached_items = {}  # key => Runtime::CollectionResult::Item
+      end
+
+      # Compute a Diff between the current normalized items and the previous state.
+      #
+      # @param current_items [Array<Hash>] normalized item input hashes
+      # @param key_fn        [Proc]        extracts the item key from an item hash
+      # @return [Diff]
+      def compute_diff(current_items, key_fn)
+        current_keys = current_items.to_h { |i| [key_fn.call(i), i] }
+        added, changed, unchanged = partition_items(current_items, key_fn)
+        removed = @snapshots.keys.reject { |k| current_keys.key?(k) }
+        build_diff(added, removed, changed, unchanged, key_fn)
+      end
+
+      # Record a resolved item (after running its child contract).
+      def update!(key, item_inputs, collection_result_item)
+        @snapshots[key]    = fingerprint(item_inputs)
+        @cached_items[key] = collection_result_item
+      end
+
+      # Remove a previously tracked item (on removal from the input).
+      def retract!(key)
+        @snapshots.delete(key)
+        @cached_items.delete(key)
+      end
+
+      # Returns the cached CollectionResult::Item for a key, or nil.
+      def cached_item_for(key)
+        @cached_items[key]
+      end
+
+      private
+
+      def partition_items(items, key_fn)
+        items.each_with_object([[], [], []]) do |item, (added, changed, unchanged)|
+          k = key_fn.call(item)
+          if !@snapshots.key?(k)
+            added << item
+          elsif @snapshots[k] != fingerprint(item)
+            changed << item
+          else
+            unchanged << item
+          end
+        end
+      end
+
+      def build_diff(added, removed, changed, unchanged, key_fn)
+        Diff.new(
+          added: added.map { |i| key_fn.call(i) },
+          removed: removed,
+          changed: changed.map { |i| key_fn.call(i) },
+          unchanged: unchanged.map { |i| key_fn.call(i) }
+        )
+      end
+
+      # Stable fingerprint for change detection — order-independent for Hash items.
+      def fingerprint(item)
+        return item.hash.to_s unless item.is_a?(Hash)
+
+        item.sort_by { |k, _| k.to_s }.map { |k, v| "#{k}:#{v.inspect}" }.hash.to_s
+      end
+    end
+  end
+end

data/packages/igniter-core/lib/igniter/core/dataflow/incremental_collection_result.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Dataflow
+    # A CollectionResult that carries a Diff describing what changed in the last resolve.
+    #
+    # Inherits all CollectionResult behaviour (successes, failures, summary, to_h, etc.)
+    # and adds:
+    #
+    #   result.diff          → Igniter::Dataflow::Diff
+    #   result.diff.added    → keys added in the last update
+    #   result.diff.removed  → keys removed in the last update
+    #   result.diff.changed  → keys whose item content changed
+    #   result.diff.unchanged → keys that were identical — no child contract was re-run
+    #
+    class IncrementalCollectionResult < Runtime::CollectionResult
+      attr_reader :diff
+
+      def initialize(items:, diff:)
+        super(items: items, mode: :incremental)
+        @diff = diff
+      end
+
+      # Extends the base summary with incremental diff counters.
+      def summary
+        super.merge(
+          added: diff.added.size,
+          removed: diff.removed.size,
+          changed: diff.changed.size,
+          unchanged: diff.unchanged.size
+        )
+      end
+
+      def as_json(*)
+        super.merge(diff: diff.to_h)
+      end
+    end
+  end
+end

data/packages/igniter-core/lib/igniter/core/dataflow/window_filter.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Dataflow
+    # Applies a sliding window filter to an items array before incremental diff computation.
+    #
+    # Two window modes are supported:
+    #
+    #   window: { last: 100 }
+    #     Keep only the last 100 items (by position in the array — most recent last).
+    #
+    #   window: { seconds: 300, field: :received_at }
+    #     Keep only items where item[field] >= Time.now - seconds.
+    #     The field value must respond to `>=` with a Time (e.g., Time, DateTime).
+    class WindowFilter
+      def initialize(options)
+        @options = options
+      end
+
+      # @param items [Array<Hash>] normalized item input hashes
+      # @return [Array<Hash>] filtered items
+      def apply(items)
+        return items unless @options
+
+        if @options.key?(:last)
+          apply_last_n(items)
+        elsif @options.key?(:seconds)
+          apply_time_window(items)
+        else
+          items
+        end
+      end
+
+      private
+
+      def apply_last_n(items)
+        n = @options[:last]
+        items.last(n)
+      end
+
+      def apply_time_window(items)
+        field  = @options[:field].to_sym
+        cutoff = Time.now - @options[:seconds]
+        items.select { |item| item[field] >= cutoff }
+      end
+    end
+  end
+end

data/packages/igniter-core/lib/igniter/core/dataflow.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative "dataflow/diff"
+require_relative "dataflow/diff_state"
+require_relative "dataflow/window_filter"
+require_relative "dataflow/incremental_collection_result"
+require_relative "dataflow/aggregate_operators"
+require_relative "dataflow/aggregate_state"
+require_relative "model/aggregate_node"
+
+module Igniter
+  # Incremental Dataflow — Phase 1: differential collection processing.
+  #
+  # Adds `mode: :incremental` to the `collection` DSL node. In this mode the
+  # resolver tracks per-item state between `update_inputs` calls and re-runs
+  # child contracts only for items that were added or changed. Removed items
+  # are retracted automatically. Unchanged items reuse their cached results
+  # with zero re-computation cost.
+  #
+  # Optional `window:` filter limits the active item set before diff computation:
+  #
+  #   window: { last: 100 }
+  #     Keep only the last 100 items (most recent last in the array).
+  #
+  #   window: { seconds: 300, field: :received_at }
+  #     Keep items where item[:received_at] >= Time.now - 300.
+  #
+  # == Usage
+  #
+  #   require "igniter/extensions/dataflow"
+  #
+  #   class SensorPipeline < Igniter::Contract
+  #     define do
+  #       input :readings, type: :array
+  #
+  #       collection :processed,
+  #                  with: :readings,
+  #                  each: SensorContract,
+  #                  key: :sensor_id,
+  #                  mode: :incremental,
+  #                  window: { last: 500 }
+  #
+  #       output :processed
+  #     end
+  #   end
+  #
+  #   contract = SensorPipeline.new(readings: initial_readings)
+  #   contract.resolve_all
+  #
+  #   contract.update_inputs(readings: updated_readings)
+  #   contract.resolve_all
+  #
+  #   result = contract.result.processed
+  #   result.diff.added     # => [:sensor_42]   — child contract was re-run
+  #   result.diff.changed   # => [:sensor_7]    — child contract was re-run
+  #   result.diff.removed   # => [:sensor_3]    — retracted from result
+  #   result.diff.unchanged # => [:sensor_1, :sensor_2]  — result reused
+  #
+  #   # Convenience: push events as a diff instead of replacing the full array
+  #   contract.feed_diff(:readings, add: [new_reading], remove: [:sensor_3])
+  #
+  module Dataflow
+    class DataflowError < Igniter::Error; end
+  end
+end

data/packages/igniter-core/lib/igniter/core/diagnostics/agent_contributor.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Diagnostics
+    module AgentContributor
+      class << self
+        def augment(report:, execution:)
+          agents = summarize_agents(execution)
+          return report if agents[:entries].empty?
+
+          report[:agents] = agents
+          augment_outputs(report: report, execution: execution, agents: agents)
+          augment_errors(report: report, agents: agents)
+          report
+        end
+
+        def append_text(report:, lines:)
+          agents = report[:agents]
+          return unless agents
+
+          summaries = agents[:entries].map do |entry|
+            summary = "#{entry[:node_name]}(#{entry[:status]} #{entry[:agent_trace_summary]})"
+            summary += " token=#{entry[:token]}" if entry[:token]
+            summary += " reply=#{entry[:reply_mode]}" if entry[:reply_mode]
+            summary += " phase=#{entry.dig(:agent_session, :phase)}" if entry.dig(:agent_session, :phase)
+            summary += " turn=#{entry.dig(:agent_session, :turn)}" if entry.dig(:agent_session, :turn)
+            summary
+          end
+
+          lines << "Agents: #{agents_overview(agents)} #{summaries.join(', ')}"
+        end
+
+        def append_markdown_summary(report:, lines:)
+          agents = report[:agents]
+          return unless agents
+
+          lines << "- Agents: #{agents_overview(agents)}"
+        end
+
+        def append_markdown_sections(report:, lines:)
+          agents = report[:agents]
+          return unless agents
+
+          lines << ""
+          lines << "## Agents"
+          agents[:entries].each do |entry|
+            line = "- `#{entry[:node_name]}` `#{entry[:status]}`: `#{entry[:agent_trace_summary]}`"
+            line += " token=`#{entry[:token]}`" if entry[:token]
+            line += " reply=`#{entry[:reply_mode]}`" if entry[:reply_mode]
+            line += " phase=`#{entry.dig(:agent_session, :phase)}`" if entry.dig(:agent_session, :phase)
+            line += " turn=`#{entry.dig(:agent_session, :turn)}`" if entry.dig(:agent_session, :turn)
+            line += " error=`#{entry[:error][:message]}`" if entry[:error]
+            lines << line
+          end
+        end
+
+        private
+
+        def summarize_agents(execution)
+          entries = execution.cache.values.filter_map { |state| agent_entry_for(state, execution) }
+
+          {
+            total: entries.size,
+            succeeded: entries.count { |entry| entry[:status] == :succeeded },
+            pending: entries.count { |entry| entry[:status] == :pending },
+            failed: entries.count { |entry| entry[:status] == :failed },
+            facets: summarize_agent_facets(entries),
+            entries: entries
+          }
+        end
+
+        def agent_entry_for(state, execution)
+          return unless state.node.kind == :agent
+
+          trace = extract_agent_trace(state)
+          return unless trace
+
+          events = summarize_node_events(execution, state.node.name)
+          entry = {
+            node_name: state.node.name,
+            path: state.node.path,
+            status: state.status,
+            reply_mode: state.node.reply_mode,
+            events: events,
+            agent_trace: trace,
+            agent_trace_summary: summarize_agent_trace(trace)
+          }
+
+          entry[:agent_session] = state.details[:agent_session] if state.details[:agent_session]
+
+          if state.pending? && state.value.is_a?(Igniter::Runtime::DeferredResult)
+            entry[:token] = state.value.token
+            entry[:waiting_on] = state.value.waiting_on
+            entry[:source_node] = state.value.source_node
+            entry[:agent_session] = state.value.agent_session_data if state.value.agent_session_data
+          end
+
+          if state.failed?
+            entry[:error] = {
+              type: state.error.class.name,
+              message: state.error.message
+            }
+          end
+
+          entry
+        end
+
+        def extract_agent_trace(state)
+          return state.details[:agent_trace] if state.details[:agent_trace]
+          return state.value.agent_trace if state.pending? && state.value.is_a?(Igniter::Runtime::DeferredResult) && state.value.agent_trace
+          return nil unless state.failed?
+
+          context = state.error.respond_to?(:context) ? state.error.context : {}
+          context[:agent_trace] || context["agent_trace"]
+        end
+
+        def augment_outputs(report:, execution:, agents:)
+          agent_by_node = agents[:entries].each_with_object({}) do |entry, memo|
+            memo[entry[:node_name].to_sym] = entry
+          end
+
+          execution.compiled_graph.outputs.each do |output_node|
+            entry = agent_by_node[output_node.source_root.to_sym]
+            value = report[:outputs][output_node.name]
+            next unless entry && value.is_a?(Hash)
+
+            report[:outputs][output_node.name] = value.merge(
+              agent_trace: entry[:agent_trace],
+              agent_trace_summary: entry[:agent_trace_summary],
+              agent_session: entry[:agent_session]
+            )
+          end
+        end
+
+        def augment_errors(report:, agents:)
+          agent_by_node = agents[:entries].each_with_object({}) do |entry, memo|
+            memo[entry[:node_name].to_sym] = entry
+          end
+
+          report[:errors].map! do |error|
+            entry = agent_by_node[error[:node_name].to_sym]
+            next error unless entry
+
+            error.merge(
+              agent_trace: entry[:agent_trace],
+              agent_trace_summary: entry[:agent_trace_summary]
+            )
+          end
+        end
+
+        def summarize_agent_facets(entries)
+          {
+            by_status: count_by(entries) { |entry| entry[:status] },
+            by_reply_mode: count_by(entries) { |entry| entry[:reply_mode] },
+            by_mode: count_by(entries) { |entry| hash_value(entry[:agent_trace], :mode) },
+            by_adapter: count_by(entries) { |entry| hash_value(entry[:agent_trace], :adapter) },
+            by_outcome: count_by(entries) { |entry| hash_value(entry[:agent_trace], :outcome) },
+            by_reason: count_by(entries) { |entry| hash_value(entry[:agent_trace], :reason) },
+            by_latest_event: count_by(entries) { |entry| entry.dig(:events, :latest_type) }
+          }
+        end
+
+        def summarize_node_events(execution, node_name)
+          events = execution.events.events.select { |event| event.node_name == node_name.to_sym }
+
+          {
+            total: events.size,
+            latest_type: events.last&.type,
+            types: events.map(&:type).uniq
+          }
+        end
+
+        def summarize_agent_trace(trace)
+          parts = []
+          adapter = hash_value(trace, :adapter)
+          mode = hash_value(trace, :mode)
+          via = hash_value(trace, :via)
+          message = hash_value(trace, :message)
+          local = hash_value(trace, :local)
+          registered = hash_value(trace, :registered)
+          alive = hash_value(trace, :alive)
+          outcome = hash_value(trace, :outcome)
+          reason = hash_value(trace, :reason)
+
+          parts << "adapter=#{adapter}" if adapter
+          parts << "mode=#{mode}" if mode
+          parts << "via=#{via}" if via
+          parts << "message=#{message}" if message
+          parts << "local=#{local}" unless local.nil?
+          parts << "registered=#{registered}" unless registered.nil?
+          parts << "alive=#{alive}" unless alive.nil?
+          parts << "outcome=#{outcome}" if outcome
+          parts << "reason=#{reason}" if reason
+          parts.join(" ")
+        end
+
+        def agents_overview(agents)
+          facets = agents[:facets] || {}
+          parts = [
+            "total=#{agents[:total]}",
+            "succeeded=#{agents[:succeeded]}",
+            "pending=#{agents[:pending]}",
+            "failed=#{agents[:failed]}"
+          ]
+          parts << "modes=#{inline_counts(facets[:by_mode])}" if present_counts?(facets[:by_mode])
+          parts << "replies=#{inline_counts(facets[:by_reply_mode])}" if present_counts?(facets[:by_reply_mode])
+          parts << "adapters=#{inline_counts(facets[:by_adapter])}" if present_counts?(facets[:by_adapter])
+          parts << "outcomes=#{inline_counts(facets[:by_outcome])}" if present_counts?(facets[:by_outcome])
+          parts << "reasons=#{inline_counts(facets[:by_reason])}" if present_counts?(facets[:by_reason])
+          parts.join(", ")
+        end
+
+        def count_by(entries)
+          entries.each_with_object(Hash.new(0)) do |entry, memo|
+            key = yield(entry)
+            next if key.nil?
+
+            memo[key.to_sym] += 1
+          end
+        end
+
+        def inline_counts(counts)
+          counts.map { |key, value| "#{key}=#{value}" }.join(", ")
+        end
+
+        def present_counts?(counts)
+          counts && !counts.empty?
+        end
+
+        def hash_value(hash, key)
+          return nil unless hash.is_a?(Hash)
+
+          return hash[key] if hash.key?(key)
+          return hash[key.to_s] if hash.key?(key.to_s)
+
+          nil
+        end
+      end
+    end
+  end
+end