igniter 0.4.3 → 0.5.0

data/lib/igniter/agents/ai/alert_agent.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative "../proactive_agent"
+
+module Igniter
+  module Agents
+    # Threshold-based alerting agent.
+    #
+    # AlertAgent extends ProactiveAgent with an opinionated DSL for declaring
+    # numeric thresholds.  A single +monitor+ / +threshold+ pair registers both
+    # the watcher and the trigger automatically.
+    #
+    # @example
+    #   class ApiAlerts < Igniter::Agents::AlertAgent
+    #     intent "Alert when API error rate or latency spikes"
+    #     scan_interval 15.0
+    #
+    #     monitor :error_rate,  source: -> { Metrics.error_rate }
+    #     monitor :p99_latency, source: -> { Metrics.p99 }
+    #
+    #     threshold :error_rate,  above: 0.05
+    #     threshold :p99_latency, above: 500
+    #   end
+    #
+    #   ref    = ApiAlerts.start
+    #   alerts = ref.call(:alerts)   # => Array<AlertRecord>
+    class AlertAgent < ProactiveAgent
+      # Immutable record created when a threshold is breached.
+      AlertRecord = Struct.new(:metric, :value, :kind, :threshold,
+                               :fired_at, keyword_init: true)
+
+      proactive_initial_state alerts: [], silenced: false
+
+      class << self
+        # Register a polling source for a named metric.
+        # Usually called before +threshold+ for the same metric name.
+        #
+        # @param metric [Symbol, String]
+        # @param source [#call]  — zero-argument callable returning a Numeric
+        def monitor(metric, source:)
+          watch(metric.to_sym, poll: source)
+        end
+
+        # Declare a threshold for a watched metric.
+        # Automatically registers a trigger that fires when the condition is met.
+        #
+        # @param metric [Symbol, String]
+        # @param above  [Numeric, nil]  — breach if value > above
+        # @param below  [Numeric, nil]  — breach if value < below
+        def threshold(metric, above: nil, below: nil)
+          name = metric.to_sym
+
+          trigger(:"threshold_#{name}",
+            condition: ->(ctx) {
+              val = ctx[name]
+              return false if val.nil?
+
+              (above && val.to_f > above.to_f) ||
+                (below && val.to_f < below.to_f)
+            },
+            action: ->(state:, context:) {
+              next state if state[:silenced]
+
+              val  = context[name]
+              kind = above && val.to_f > above.to_f ? :above : :below
+              rec  = AlertRecord.new(
+                metric:    name,
+                value:     val,
+                kind:      kind,
+                threshold: kind == :above ? above : below,
+                fired_at:  Time.now
+              )
+              state.merge(alerts: (state[:alerts] + [rec]).last(200))
+            }
+          )
+        end
+      end
+
+      # ── Inheritance ────────────────────────────────────────────────────────
+      # Re-inject AlertAgent-specific handlers into every subclass so that
+      # anonymous test classes (Class.new(AlertAgent)) also have them.
+      def self.inherited(subclass)
+        super  # ProactiveAgent.inherited → resets @handlers, injects proactive ones
+        inject_alert_handlers!(subclass)
+      end
+
+      private_class_method def self.inject_alert_handlers!(klass)
+        klass.on(:silence)      { |state:, **| state.merge(silenced: true)  }
+        klass.on(:unsilence)    { |state:, **| state.merge(silenced: false) }
+        klass.on(:alerts)       { |state:, **| state.fetch(:alerts, []).dup }
+        klass.on(:clear_alerts) { |state:, **| state.merge(alerts: []) }
+      end
+
+      # Suppress alert creation (scans and condition checks still run).
+      on :silence   do |state:, **| state.merge(silenced: true)  end
+      on :unsilence do |state:, **| state.merge(silenced: false) end
+
+      # Sync query — all recorded AlertRecord objects.
+      #
+      # @return [Array<AlertRecord>]
+      on :alerts do |state:, **|
+        state.fetch(:alerts, []).dup
+      end
+
+      # Clear alert history.
+      on :clear_alerts do |state:, **|
+        state.merge(alerts: [])
+      end
+    end
+  end
+end

data/lib/igniter/agents/ai/chain_agent.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Agents
+    # Executes a sequence of callables where each step's output becomes the
+    # next step's input.
+    #
+    # Each step callable receives:
+    #   input:   [Object]         — output from the previous step (or initial input)
+    #   context: [Hash]           — shared context unchanged throughout the run
+    #   results: [Array<StepResult>]  — results from all preceding steps
+    #
+    # On error the chain stops immediately by default. Pass +stop_on_error: false+
+    # to continue with the error message as the next step's input.
+    #
+    # @example Summarise → translate → format
+    #   ref = ChainAgent.start
+    #   ref.send(:set_chain, steps: [
+    #     { name: :summarise,  callable: ->(input:, **) { SummariseSkill.call(text: input) } },
+    #     { name: :translate,  callable: ->(input:, **) { TranslateSkill.call(text: input) } },
+    #     { name: :format,     callable: ->(input:, **) { FormatSkill.call(content: input)  } }
+    #   ])
+    #   ref.send(:run, input: long_article, context: { target_lang: "es" })
+    #   steps = ref.call(:results)
+    class ChainAgent < Igniter::Agent
+      # Immutable record for one completed step.
+      StepResult = Struct.new(:name, :input, :output, :status, keyword_init: true)
+
+      initial_state chain: [], results: [], context: {}
+
+      # Append a step to the end of the chain.
+      #
+      # Payload keys:
+      #   name     [String, Symbol]  — step identifier
+      #   callable [#call]           — receives (input:, context:, results:)
+      on :add_step do |state:, payload:|
+        step = { name: payload.fetch(:name).to_s, callable: payload.fetch(:callable) }
+        state.merge(chain: state[:chain] + [step])
+      end
+
+      # Replace the entire chain.
+      #
+      # Payload keys:
+      #   steps [Array<Hash>]  — each element must have :name and :callable keys
+      on :set_chain do |state:, payload:|
+        steps = Array(payload.fetch(:steps)).map do |s|
+          { name: s.fetch(:name).to_s, callable: s.fetch(:callable) }
+        end
+        state.merge(chain: steps)
+      end
+
+      # Remove a step by name.
+      #
+      # Payload keys:
+      #   name [String, Symbol]
+      on :remove_step do |state:, payload:|
+        name = payload.fetch(:name).to_s
+        state.merge(chain: state[:chain].reject { |s| s[:name] == name })
+      end
+
+      # Execute the chain with an initial input.
+      #
+      # Payload keys:
+      #   input         [Object]   — starting value for the first step
+      #   context       [Hash]     — shared context passed to every step (default: {})
+      #   stop_on_error [Boolean]  — halt on first error (default: true)
+      on :run do |state:, payload:|
+        agent = new
+        agent.send(:run_chain, state, payload)
+      end
+
+      # Sync query — step results from the most recent run.
+      #
+      # @return [Array<StepResult>]
+      on :results do |state:, **|
+        state[:results]
+      end
+
+      # Sync query — list registered step names.
+      #
+      # @return [Array<String>]
+      on :steps do |state:, **|
+        state[:chain].map { |s| s[:name] }
+      end
+
+      # Clear results and context from the last run (chain is preserved).
+      on :reset do |state:, **|
+        state.merge(results: [], context: {})
+      end
+
+      private
+
+      def run_chain(state, payload)
+        input         = payload.fetch(:input)
+        context       = payload.fetch(:context, state[:context])
+        stop_on_error = payload.fetch(:stop_on_error, true)
+        results       = []
+        current       = input
+
+        state[:chain].each do |step|
+          output, status = invoke_step(step[:callable], current, context, results)
+
+          results << StepResult.new(
+            name:   step[:name],
+            input:  current,
+            output: output,
+            status: status
+          )
+
+          break if status == :error && stop_on_error
+
+          current = output
+        end
+
+        state.merge(results: results, context: context)
+      end
+
+      # @return [[output, status]]
+      def invoke_step(callable, input, context, results)
+        output = callable.call(input: input, context: context, results: results)
+        [output, :ok]
+      rescue StandardError => e
+        [e.message, :error]
+      end
+    end
+  end
+end

data/lib/igniter/agents/ai/critic_agent.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Agents
+    # Evaluates output quality and optionally retries generation until a score
+    # threshold is met.
+    #
+    # Two evaluation modes:
+    # * **Rule-based** (default) — heuristics based on length and emptiness;
+    #   no external dependencies.
+    # * **LLM-assisted** — delegates to any callable that accepts +output:+ and
+    #   +criteria:+ and returns +{ score: Float, feedback: String }+ or a
+    #   String that will be parsed for a numeric score.
+    #
+    # Scores are on a 0–10 scale. The default passing threshold is 7.0.
+    #
+    # @example Rule-based gate
+    #   ref = CriticAgent.start
+    #   ref.send(:evaluate, output: "Short answer", criteria: "completeness")
+    #   ev = ref.call(:last_evaluation)
+    #   puts ev.passed  # => false (too short)
+    #
+    # @example LLM-assisted with retry
+    #   evaluator = ->(output:, criteria:) {
+    #     result = MyGraderSkill.call(output: output, criteria: criteria)
+    #     { score: result.score, feedback: result.feedback }
+    #   }
+    #   ref = CriticAgent.start(initial_state: { evaluator: evaluator, threshold: 8.0 })
+    #   ref.send(:evaluate_and_retry,
+    #     output:         first_draft,
+    #     criteria:       "accuracy, completeness",
+    #     max_retries:    2,
+    #     generator:      ->(draft:) { improve_draft(draft) },
+    #     generator_args: { draft: first_draft }
+    #   )
+    class CriticAgent < Igniter::Agent
+      # Immutable evaluation result.
+      Evaluation = Struct.new(:score, :feedback, :passed, :criteria, keyword_init: true)
+
+      initial_state evaluator: nil, threshold: 7.0, evaluations: []
+
+      # Evaluate a single output.
+      #
+      # Payload keys:
+      #   output    [String, Object]  — required; the artifact to evaluate
+      #   criteria  [String]          — evaluation criteria (default: "quality, relevance")
+      #   evaluator [#call, nil]      — override state evaluator for this call
+      #   threshold [Float, nil]      — override state threshold for this call
+      on :evaluate do |state:, payload:|
+        agent = new
+        ev    = agent.send(:run_evaluation, payload, state)
+        state.merge(evaluations: state[:evaluations] + [ev])
+      end
+
+      # Evaluate and re-generate until the score passes or retries are exhausted.
+      #
+      # Payload keys:
+      #   output          [String]   — initial output to evaluate
+      #   criteria        [String]   — evaluation criteria
+      #   generator       [#call]    — required; called with **generator_args to produce a new output
+      #   generator_args  [Hash]     — arguments forwarded to generator (default: {})
+      #   max_retries     [Integer]  — maximum re-generation attempts (default: 3)
+      #   evaluator       [#call]    — override state evaluator
+      #   threshold       [Float]    — override state threshold
+      on :evaluate_and_retry do |state:, payload:|
+        agent = new
+        agent.send(:run_evaluate_and_retry, state, payload)
+      end
+
+      # Set default evaluator and/or threshold.
+      #
+      # Payload keys:
+      #   evaluator [#call]  — new default evaluator
+      #   threshold [Float]  — new default threshold
+      on :configure do |state:, payload:|
+        state.merge(
+          evaluator: payload.fetch(:evaluator, state[:evaluator]),
+          threshold: payload.fetch(:threshold, state[:threshold]).to_f
+        )
+      end
+
+      # Sync query — returns the most recent Evaluation, or nil.
+      on :last_evaluation do |state:, **|
+        state[:evaluations].last
+      end
+
+      # Sync query — returns all recorded Evaluation structs.
+      on :evaluations do |state:, **|
+        state[:evaluations]
+      end
+
+      # Clear evaluation history.
+      on :clear do |state:, **|
+        state.merge(evaluations: [])
+      end
+
+      private
+
+      # @return [Evaluation]
+      def run_evaluation(payload, state)
+        output    = payload.fetch(:output)
+        criteria  = payload.fetch(:criteria, "quality, relevance")
+        evaluator = payload.fetch(:evaluator, state[:evaluator])
+        threshold = payload.fetch(:threshold, state[:threshold]).to_f
+
+        score, feedback = evaluator ? llm_score(evaluator, output, criteria)
+                                    : rule_score(output)
+
+        Evaluation.new(
+          score:    score.to_f,
+          feedback: feedback.to_s,
+          passed:   score.to_f >= threshold,
+          criteria: criteria
+        )
+      end
+
+      # @return [Hash] updated state with all evaluations recorded
+      def run_evaluate_and_retry(state, payload)
+        max_retries    = payload.fetch(:max_retries, 3)
+        generator      = payload.fetch(:generator)
+        generator_args = payload.fetch(:generator_args, {})
+        output         = payload.fetch(:output)
+        all_evals      = []
+
+        (max_retries + 1).times do |attempt|
+          output = generator.call(**generator_args) if attempt.positive?
+          ev     = run_evaluation(payload.merge(output: output), state)
+          all_evals << ev
+          break if ev.passed
+        end
+
+        state.merge(evaluations: state[:evaluations] + all_evals)
+      end
+
+      # Call the user-supplied evaluator. Accepts two return shapes:
+      #   Hash  with :score / :feedback keys
+      #   String  (we scan for the first number in the text as the score)
+      def llm_score(evaluator, output, criteria)
+        result = evaluator.call(output: output, criteria: criteria)
+        case result
+        when Hash
+          [result.fetch(:score, 5.0), result.fetch(:feedback, "")]
+        else
+          text  = result.to_s
+          score = text.match(/\b(\d+(?:\.\d+)?)\b/)&.captures&.first&.to_f || 5.0
+          [score, text]
+        end
+      rescue StandardError => e
+        [0.0, "Evaluator error: #{e.message}"]
+      end
+
+      # Minimal rule-based heuristic (no LLM required).
+      def rule_score(output)
+        text = output.to_s.strip
+        return [0.0, "Output is empty"]          if text.empty?
+        return [3.0, "Output is very short"]     if text.length < 50
+        return [5.5, "Output is below average length"] if text.length < 200
+
+        [7.5, "Output meets basic length criteria"]
+      end
+    end
+  end
+end

data/lib/igniter/agents/ai/evaluator_agent.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Agents
+    # Tracks performance metrics for named subjects (agents, services,
+    # contracts), computes weighted aggregate scores, and compares subjects
+    # against each other or against stored baselines.
+    #
+    # Scoring:
+    # * Metrics are grouped by name; the last 20 readings are averaged.
+    # * Per-subject or global weights scale each metric's contribution.
+    # * When a baseline is set the score is normalised to 0–100 relative to it;
+    #   without a baseline the raw weighted average is returned.
+    # * Grades: A ≥ 90, B ≥ 75, C ≥ 60, D otherwise.
+    #
+    # @example Track two services and compare
+    #   ref = EvaluatorAgent.start
+    #   ref.send(:record_metric, subject: :api, name: :throughput, value: 850)
+    #   ref.send(:record_metric, subject: :api, name: :error_rate,  value: 2.1)
+    #   ref.send(:set_baseline,  subject: :api, baseline: 800)
+    #   ref.send(:evaluate,      subject: :api)
+    #   ev = ref.call(:evaluations, subject: :api).last
+    #   puts ev.grade   # => "A"
+    class EvaluatorAgent < Igniter::Agent
+      MetricRecord = Struct.new(:name, :value, :recorded_at, keyword_init: true)
+      Evaluation   = Struct.new(:subject, :score, :grade, :metrics,
+                                :recorded_at, keyword_init: true)
+      Comparison   = Struct.new(:subject_a, :subject_b, :winner, :delta,
+                                keyword_init: true)
+
+      GRADES = [
+        [90.0, "A"],
+        [75.0, "B"],
+        [60.0, "C"],
+        [  0.0, "D"]
+      ].freeze
+
+      # subjects: Hash<String, { metrics: Array<MetricRecord>, baseline: Float?, weights: Hash }>
+      initial_state \
+        subjects:    {},
+        evaluations: [],
+        weights:     {}
+
+      # Record a metric reading for a subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol]  — subject identifier
+      #   name    [String, Symbol]  — metric name
+      #   value   [Numeric]         — metric value
+      on :record_metric do |state:, payload:|
+        subject = payload.fetch(:subject).to_s
+        metric  = MetricRecord.new(
+          name:        payload.fetch(:name).to_s,
+          value:       payload.fetch(:value).to_f,
+          recorded_at: Time.now
+        )
+        entry   = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
+        updated = entry.merge(metrics: (entry[:metrics] + [metric]).last(200))
+        state.merge(subjects: state[:subjects].merge(subject => updated))
+      end
+
+      # Set the reference baseline value for a subject.
+      # Scores will be expressed as a percentage of this baseline.
+      #
+      # Payload keys:
+      #   subject  [String, Symbol]
+      #   baseline [Numeric]
+      on :set_baseline do |state:, payload:|
+        subject  = payload.fetch(:subject).to_s
+        baseline = payload.fetch(:baseline).to_f
+        entry    = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
+        updated  = entry.merge(baseline: baseline)
+        state.merge(subjects: state[:subjects].merge(subject => updated))
+      end
+
+      # Set per-metric weights for a subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol]
+      #   weights [Hash<String, Numeric>]  — metric name → weight
+      on :set_weights do |state:, payload:|
+        subject = payload.fetch(:subject).to_s
+        weights = payload.fetch(:weights).transform_keys(&:to_s)
+        entry   = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
+        updated = entry.merge(weights: weights)
+        state.merge(subjects: state[:subjects].merge(subject => updated))
+      end
+
+      # Compute and store an Evaluation for a subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol]
+      on :evaluate do |state:, payload:|
+        agent    = new
+        ev       = agent.send(:compute_evaluation, payload.fetch(:subject).to_s, state)
+        next state unless ev
+
+        state.merge(evaluations: state[:evaluations] + [ev])
+      end
+
+      # Sync query — compare the most recent evaluations of two subjects.
+      #
+      # Payload keys:
+      #   a [String, Symbol]  — first subject
+      #   b [String, Symbol]  — second subject
+      #
+      # @return [Comparison, nil]
+      on :compare do |state:, payload:|
+        a = state[:evaluations].select { |e| e.subject == payload.fetch(:a).to_s }.last
+        b = state[:evaluations].select { |e| e.subject == payload.fetch(:b).to_s }.last
+        next nil unless a && b
+
+        delta  = (a.score - b.score).round(4)
+        winner = if delta > 0 then a.subject
+                 elsif delta < 0 then b.subject
+                 else :tie
+                 end
+        Comparison.new(subject_a: a.subject, subject_b: b.subject,
+                       winner: winner, delta: delta.abs)
+      end
+
+      # Sync query — all evaluations, optionally filtered by subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol, nil]
+      #
+      # @return [Array<Evaluation>]
+      on :evaluations do |state:, payload:|
+        filter = payload&.fetch(:subject, nil)
+        evs    = state[:evaluations]
+        evs    = evs.select { |e| e.subject == filter.to_s } if filter
+        evs.dup
+      end
+
+      # Sync query — list registered subject names.
+      #
+      # @return [Array<String>]
+      on :subjects do |state:, **|
+        state[:subjects].keys
+      end
+
+      # Set global default weights (applied when a subject has no per-metric weight).
+      #
+      # Payload keys:
+      #   weights [Hash<String, Numeric>]
+      on :configure do |state:, payload:|
+        state.merge(payload.slice(:weights).compact)
+      end
+
+      # Clear all subjects and evaluations.
+      on :reset do |state:, **|
+        state.merge(subjects: {}, evaluations: [])
+      end
+
+      private
+
+      def compute_evaluation(name, state)
+        data = state[:subjects][name]
+        return nil unless data && data[:metrics].any?
+
+        weights  = state[:weights].merge(data[:weights] || {})
+        metrics  = data[:metrics]
+        baseline = data[:baseline]
+
+        grouped = metrics.group_by(&:name)
+        score_parts = grouped.map do |mname, records|
+          avg    = records.last(20).sum(&:value) / [records.last(20).size, 1].max
+          weight = (weights[mname] || 1.0).to_f
+          [avg * weight, weight]
+        end
+
+        total_weight = [score_parts.sum { |_, w| w }, 0.001].max
+        raw          = score_parts.sum { |v, _| v } / total_weight
+
+        score = if baseline && baseline > 0
+          [(raw / baseline * 100).round(4), 100.0].min
+        else
+          raw.round(4)
+        end
+
+        grade = GRADES.find { |threshold, _| score.to_f >= threshold }&.last || "D"
+
+        Evaluation.new(
+          subject:     name,
+          score:       score,
+          grade:       grade,
+          metrics:     grouped.keys,
+          recorded_at: Time.now
+        )
+      end
+    end
+  end
+end