igniter 0.4.5 → 0.5.1

data/packages/igniter-agents/lib/igniter/ai/agents/critic_agent.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Igniter
+  module AI
+    module Agents
+    # Evaluates output quality and optionally retries generation until a score
+    # threshold is met.
+    #
+    # Two evaluation modes:
+    # * **Rule-based** (default) — heuristics based on length and emptiness;
+    #   no external dependencies.
+    # * **LLM-assisted** — delegates to any callable that accepts +output:+ and
+    #   +criteria:+ and returns +{ score: Float, feedback: String }+ or a
+    #   String that will be parsed for a numeric score.
+    #
+    # Scores are on a 0–10 scale. The default passing threshold is 7.0.
+    #
+    # @example Rule-based gate
+    #   ref = CriticAgent.start
+    #   ref.send(:evaluate, output: "Short answer", criteria: "completeness")
+    #   ev = ref.call(:last_evaluation)
+    #   puts ev.passed  # => false (too short)
+    #
+    # @example LLM-assisted with retry
+    #   evaluator = ->(output:, criteria:) {
+    #     result = MyGraderSkill.call(output: output, criteria: criteria)
+    #     { score: result.score, feedback: result.feedback }
+    #   }
+    #   ref = CriticAgent.start(initial_state: { evaluator: evaluator, threshold: 8.0 })
+    #   ref.send(:evaluate_and_retry,
+    #     output:         first_draft,
+    #     criteria:       "accuracy, completeness",
+    #     max_retries:    2,
+    #     generator:      ->(draft:) { improve_draft(draft) },
+    #     generator_args: { draft: first_draft }
+    #   )
+      class CriticAgent < Igniter::Agent
+      # Immutable evaluation result.
+      Evaluation = Struct.new(:score, :feedback, :passed, :criteria, keyword_init: true)
+
+      initial_state evaluator: nil, threshold: 7.0, evaluations: []
+
+      # Evaluate a single output.
+      #
+      # Payload keys:
+      #   output    [String, Object]  — required; the artifact to evaluate
+      #   criteria  [String]          — evaluation criteria (default: "quality, relevance")
+      #   evaluator [#call, nil]      — override state evaluator for this call
+      #   threshold [Float, nil]      — override state threshold for this call
+      on :evaluate do |state:, payload:|
+        agent = new
+        ev    = agent.send(:run_evaluation, payload, state)
+        state.merge(evaluations: state[:evaluations] + [ev])
+      end
+
+      # Evaluate and re-generate until the score passes or retries are exhausted.
+      #
+      # Payload keys:
+      #   output          [String]   — initial output to evaluate
+      #   criteria        [String]   — evaluation criteria
+      #   generator       [#call]    — required; called with **generator_args to produce a new output
+      #   generator_args  [Hash]     — arguments forwarded to generator (default: {})
+      #   max_retries     [Integer]  — maximum re-generation attempts (default: 3)
+      #   evaluator       [#call]    — override state evaluator
+      #   threshold       [Float]    — override state threshold
+      on :evaluate_and_retry do |state:, payload:|
+        agent = new
+        agent.send(:run_evaluate_and_retry, state, payload)
+      end
+
+      # Set default evaluator and/or threshold.
+      #
+      # Payload keys:
+      #   evaluator [#call]  — new default evaluator
+      #   threshold [Float]  — new default threshold
+      on :configure do |state:, payload:|
+        state.merge(
+          evaluator: payload.fetch(:evaluator, state[:evaluator]),
+          threshold: payload.fetch(:threshold, state[:threshold]).to_f
+        )
+      end
+
+      # Sync query — returns the most recent Evaluation, or nil.
+      on :last_evaluation do |state:, **|
+        state[:evaluations].last
+      end
+
+      # Sync query — returns all recorded Evaluation structs.
+      on :evaluations do |state:, **|
+        state[:evaluations]
+      end
+
+      # Clear evaluation history.
+      on :clear do |state:, **|
+        state.merge(evaluations: [])
+      end
+
+      private
+
+      # @return [Evaluation]
+      def run_evaluation(payload, state)
+        output    = payload.fetch(:output)
+        criteria  = payload.fetch(:criteria, "quality, relevance")
+        evaluator = payload.fetch(:evaluator, state[:evaluator])
+        threshold = payload.fetch(:threshold, state[:threshold]).to_f
+
+        score, feedback = evaluator ? llm_score(evaluator, output, criteria)
+                                    : rule_score(output)
+
+        Evaluation.new(
+          score:    score.to_f,
+          feedback: feedback.to_s,
+          passed:   score.to_f >= threshold,
+          criteria: criteria
+        )
+      end
+
+      # @return [Hash] updated state with all evaluations recorded
+      def run_evaluate_and_retry(state, payload)
+        max_retries    = payload.fetch(:max_retries, 3)
+        generator      = payload.fetch(:generator)
+        generator_args = payload.fetch(:generator_args, {})
+        output         = payload.fetch(:output)
+        all_evals      = []
+
+        (max_retries + 1).times do |attempt|
+          output = generator.call(**generator_args) if attempt.positive?
+          ev     = run_evaluation(payload.merge(output: output), state)
+          all_evals << ev
+          break if ev.passed
+        end
+
+        state.merge(evaluations: state[:evaluations] + all_evals)
+      end
+
+      # Call the user-supplied evaluator. Accepts two return shapes:
+      #   Hash  with :score / :feedback keys
+      #   String  (we scan for the first number in the text as the score)
+      def llm_score(evaluator, output, criteria)
+        result = evaluator.call(output: output, criteria: criteria)
+        case result
+        when Hash
+          [result.fetch(:score, 5.0), result.fetch(:feedback, "")]
+        else
+          text  = result.to_s
+          score = text.match(/\b(\d+(?:\.\d+)?)\b/)&.captures&.first&.to_f || 5.0
+          [score, text]
+        end
+      rescue StandardError => e
+        [0.0, "Evaluator error: #{e.message}"]
+      end
+
+      # Minimal rule-based heuristic (no LLM required).
+      def rule_score(output)
+        text = output.to_s.strip
+        return [0.0, "Output is empty"]          if text.empty?
+        return [3.0, "Output is very short"]     if text.length < 50
+        return [5.5, "Output is below average length"] if text.length < 200
+
+        [7.5, "Output meets basic length criteria"]
+      end
+      end
+    end
+  end
+end

data/packages/igniter-agents/lib/igniter/ai/agents/evaluator_agent.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Igniter
+  module AI
+    module Agents
+    # Tracks performance metrics for named subjects (agents, services,
+    # contracts), computes weighted aggregate scores, and compares subjects
+    # against each other or against stored baselines.
+    #
+    # Scoring:
+    # * Metrics are grouped by name; the last 20 readings are averaged.
+    # * Per-subject or global weights scale each metric's contribution.
+    # * When a baseline is set the score is normalised to 0–100 relative to it;
+    #   without a baseline the raw weighted average is returned.
+    # * Grades: A ≥ 90, B ≥ 75, C ≥ 60, D otherwise.
+    #
+    # @example Track two services and compare
+    #   ref = EvaluatorAgent.start
+    #   ref.send(:record_metric, subject: :api, name: :throughput, value: 850)
+    #   ref.send(:record_metric, subject: :api, name: :error_rate,  value: 2.1)
+    #   ref.send(:set_baseline,  subject: :api, baseline: 800)
+    #   ref.send(:evaluate,      subject: :api)
+    #   ev = ref.call(:evaluations, subject: :api).last
+    #   puts ev.grade   # => "A"
+      class EvaluatorAgent < Igniter::Agent
+      MetricRecord = Struct.new(:name, :value, :recorded_at, keyword_init: true)
+      Evaluation   = Struct.new(:subject, :score, :grade, :metrics,
+                                :recorded_at, keyword_init: true)
+      Comparison   = Struct.new(:subject_a, :subject_b, :winner, :delta,
+                                keyword_init: true)
+
+      GRADES = [
+        [90.0, "A"],
+        [75.0, "B"],
+        [60.0, "C"],
+        [  0.0, "D"]
+      ].freeze
+
+      # subjects: Hash<String, { metrics: Array<MetricRecord>, baseline: Float?, weights: Hash }>
+      initial_state \
+        subjects:    {},
+        evaluations: [],
+        weights:     {}
+
+      # Record a metric reading for a subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol]  — subject identifier
+      #   name    [String, Symbol]  — metric name
+      #   value   [Numeric]         — metric value
+      on :record_metric do |state:, payload:|
+        subject = payload.fetch(:subject).to_s
+        metric  = MetricRecord.new(
+          name:        payload.fetch(:name).to_s,
+          value:       payload.fetch(:value).to_f,
+          recorded_at: Time.now
+        )
+        entry   = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
+        updated = entry.merge(metrics: (entry[:metrics] + [metric]).last(200))
+        state.merge(subjects: state[:subjects].merge(subject => updated))
+      end
+
+      # Set the reference baseline value for a subject.
+      # Scores will be expressed as a percentage of this baseline.
+      #
+      # Payload keys:
+      #   subject  [String, Symbol]
+      #   baseline [Numeric]
+      on :set_baseline do |state:, payload:|
+        subject  = payload.fetch(:subject).to_s
+        baseline = payload.fetch(:baseline).to_f
+        entry    = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
+        updated  = entry.merge(baseline: baseline)
+        state.merge(subjects: state[:subjects].merge(subject => updated))
+      end
+
+      # Set per-metric weights for a subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol]
+      #   weights [Hash<String, Numeric>]  — metric name → weight
+      on :set_weights do |state:, payload:|
+        subject = payload.fetch(:subject).to_s
+        weights = payload.fetch(:weights).transform_keys(&:to_s)
+        entry   = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
+        updated = entry.merge(weights: weights)
+        state.merge(subjects: state[:subjects].merge(subject => updated))
+      end
+
+      # Compute and store an Evaluation for a subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol]
+      on :evaluate do |state:, payload:|
+        agent    = new
+        ev       = agent.send(:compute_evaluation, payload.fetch(:subject).to_s, state)
+        next state unless ev
+
+        state.merge(evaluations: state[:evaluations] + [ev])
+      end
+
+      # Sync query — compare the most recent evaluations of two subjects.
+      #
+      # Payload keys:
+      #   a [String, Symbol]  — first subject
+      #   b [String, Symbol]  — second subject
+      #
+      # @return [Comparison, nil]
+      on :compare do |state:, payload:|
+        a = state[:evaluations].select { |e| e.subject == payload.fetch(:a).to_s }.last
+        b = state[:evaluations].select { |e| e.subject == payload.fetch(:b).to_s }.last
+        next nil unless a && b
+
+        delta  = (a.score - b.score).round(4)
+        winner = if delta > 0 then a.subject
+                 elsif delta < 0 then b.subject
+                 else :tie
+                 end
+        Comparison.new(subject_a: a.subject, subject_b: b.subject,
+                       winner: winner, delta: delta.abs)
+      end
+
+      # Sync query — all evaluations, optionally filtered by subject.
+      #
+      # Payload keys:
+      #   subject [String, Symbol, nil]
+      #
+      # @return [Array<Evaluation>]
+      on :evaluations do |state:, payload:|
+        filter = payload&.fetch(:subject, nil)
+        evs    = state[:evaluations]
+        evs    = evs.select { |e| e.subject == filter.to_s } if filter
+        evs.dup
+      end
+
+      # Sync query — list registered subject names.
+      #
+      # @return [Array<String>]
+      on :subjects do |state:, **|
+        state[:subjects].keys
+      end
+
+      # Set global default weights (applied when a subject has no per-metric weight).
+      #
+      # Payload keys:
+      #   weights [Hash<String, Numeric>]
+      on :configure do |state:, payload:|
+        state.merge(payload.slice(:weights).compact)
+      end
+
+      # Clear all subjects and evaluations.
+      on :reset do |state:, **|
+        state.merge(subjects: {}, evaluations: [])
+      end
+
+      private
+
+      def compute_evaluation(name, state)
+        data = state[:subjects][name]
+        return nil unless data && data[:metrics].any?
+
+        weights  = state[:weights].merge(data[:weights] || {})
+        metrics  = data[:metrics]
+        baseline = data[:baseline]
+
+        grouped = metrics.group_by(&:name)
+        score_parts = grouped.map do |mname, records|
+          avg    = records.last(20).sum(&:value) / [records.last(20).size, 1].max
+          weight = (weights[mname] || 1.0).to_f
+          [avg * weight, weight]
+        end
+
+        total_weight = [score_parts.sum { |_, w| w }, 0.001].max
+        raw          = score_parts.sum { |v, _| v } / total_weight
+
+        score = if baseline && baseline > 0
+          [(raw / baseline * 100).round(4), 100.0].min
+        else
+          raw.round(4)
+        end
+
+        grade = GRADES.find { |threshold, _| score.to_f >= threshold }&.last || "D"
+
+        Evaluation.new(
+          subject:     name,
+          score:       score,
+          grade:       grade,
+          metrics:     grouped.keys,
+          recorded_at: Time.now
+        )
+      end
+      end
+    end
+  end
+end

data/packages/igniter-agents/lib/igniter/ai/agents/evolution_agent.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+module Igniter
+  module AI
+    module Agents
+    # Mission: become better.
+    #
+    # An evolutionary self-improvement agent that maintains a population of
+    # strategies (prompt variants, parameter configs, tool selections — any
+    # Hash), evaluates their fitness, selects survivors, generates mutations,
+    # and tracks lineage across generations.
+    #
+    # The agent is intentionally domain-agnostic. Strategies are plain Hashes
+    # whose semantics are defined by the caller's +fitness_fn+.
+    #
+    # Life-cycle of one generation:
+    #   seed → evaluate_population → evolve   (or: seed → run_generation × N)
+    #
+    # Mutation:
+    # * **Rule-based** (default) — numeric values are scaled ±20 %, booleans
+    #   are flipped, arrays are sub-sampled. Strings are kept unchanged.
+    # * **LLM-assisted** — delegates to any callable that accepts
+    #   +strategy:+ and +generation:+ and returns a Hash.
+    #
+    # @example Evolve a prompt configuration
+    #   fitness = ->(config) { MyEvaluator.score(config) }
+    #   ref = EvolutionAgent.start(initial_state: {
+    #     fitness_fn: fitness,
+    #     population_size: 4
+    #   })
+    #   ref.send(:seed, strategies: [
+    #     { temperature: 0.7, max_tokens: 512 },
+    #     { temperature: 0.3, max_tokens: 256 }
+    #   ])
+    #   3.times { ref.send(:run_generation) }
+    #   best = ref.call(:best)   # => Strategy struct
+      class EvolutionAgent < Igniter::Agent
+      # An individual strategy in the population.
+      Strategy = Struct.new(:id, :config, :fitness, :generation,
+                            :parent_ids, :mutations, keyword_init: true)
+
+      # Summary record for one completed generation.
+      GenerationReport = Struct.new(:generation, :population_size, :best_fitness,
+                                    :mean_fitness, :best_id, keyword_init: true)
+
+      initial_state \
+        population:       [],
+        generation:       0,
+        history:          [],
+        best_strategy:    nil,
+        fitness_fn:       nil,
+        llm:              nil,
+        population_size:  6,
+        mutation_rate:    0.3,
+        elite_fraction:   0.5
+
+      # Seed the population with an initial set of strategy configs.
+      # Resets generation counter and history.
+      #
+      # Payload keys:
+      #   strategies [Array<Hash>]  — initial strategy configs
+      on :seed do |state:, payload:|
+        pop = Array(payload.fetch(:strategies)).each_with_index.map do |cfg, i|
+          Strategy.new(
+            id:         "gen0_#{i}",
+            config:     cfg,
+            fitness:    nil,
+            generation: 0,
+            parent_ids: [],
+            mutations:  []
+          )
+        end
+        state.merge(population: pop, generation: 0, history: [], best_strategy: nil)
+      end
+
+      # Score every strategy in the population using the fitness function.
+      # Strategies that raise are assigned fitness 0.0.
+      #
+      # Payload keys:
+      #   fitness_fn [#call, nil]  — override the state fitness function
+      on :evaluate_population do |state:, payload:|
+        fitness_fn = (payload && payload[:fitness_fn]) || state[:fitness_fn]
+        next state unless fitness_fn
+
+        agent     = new
+        evaluated = agent.send(:score_population, state[:population], fitness_fn)
+        best      = evaluated.max_by { |s| s.fitness || -Float::INFINITY }
+        state.merge(population: evaluated, best_strategy: best)
+      end
+
+      # Select elite survivors, generate children by mutation, increment generation.
+      # Requires the population to be already evaluated (see +:evaluate_population+).
+      on :evolve do |state:, payload:|
+        agent          = new
+        new_pop, report = agent.send(:run_evolution, state)
+        next state unless new_pop
+
+        best = new_pop.max_by { |s| s.fitness || -Float::INFINITY }
+        state.merge(
+          population:    new_pop,
+          generation:    state[:generation] + 1,
+          history:       state[:history] + [report],
+          best_strategy: best
+        )
+      end
+
+      # Convenience: evaluate the current population then evolve — one full cycle.
+      #
+      # Payload keys:
+      #   fitness_fn [#call, nil]  — override the state fitness function
+      on :run_generation do |state:, payload:|
+        fitness_fn = (payload && payload[:fitness_fn]) || state[:fitness_fn]
+        next state unless fitness_fn
+
+        agent = new
+
+        # evaluate
+        evaluated_pop = agent.send(:score_population, state[:population], fitness_fn)
+        with_scores   = state.merge(population: evaluated_pop)
+
+        # evolve
+        new_pop, report = agent.send(:run_evolution, with_scores)
+        next with_scores unless new_pop
+
+        best = new_pop.max_by { |s| s.fitness || -Float::INFINITY }
+        with_scores.merge(
+          population:    new_pop,
+          generation:    state[:generation] + 1,
+          history:       state[:history] + [report],
+          best_strategy: best
+        )
+      end
+
+      # Add a single mutated child of a given strategy to the population.
+      #
+      # Payload keys:
+      #   id [String]  — id of the parent strategy
+      on :mutate_strategy do |state:, payload:|
+        parent = state[:population].find { |s| s.id == payload.fetch(:id) }
+        next state unless parent
+
+        agent = new
+        child = agent.send(:mutate_one, parent, state, state[:population].size)
+        state.merge(population: state[:population] + [child])
+      end
+
+      # Sync query — best evaluated strategy so far.
+      #
+      # @return [Strategy, nil]
+      on :best do |state:, **|
+        state[:best_strategy]
+      end
+
+      # Sync query — current population.
+      #
+      # @return [Array<Strategy>]
+      on :population do |state:, **|
+        state[:population].dup
+      end
+
+      # Sync query — evolution history (one GenerationReport per generation).
+      #
+      # @return [Array<GenerationReport>]
+      on :history do |state:, **|
+        state[:history].dup
+      end
+
+      # Sync query — current generation number.
+      #
+      # @return [Integer]
+      on :generation do |state:, **|
+        state[:generation]
+      end
+
+      # Update agent configuration.
+      #
+      # Payload keys:
+      #   fitness_fn      [#call]
+      #   llm             [#call]
+      #   population_size [Integer]
+      #   mutation_rate   [Float]   — 0.0–1.0
+      #   elite_fraction  [Float]   — 0.0–1.0
+      on :configure do |state:, payload:|
+        state.merge(
+          payload.slice(:fitness_fn, :llm, :population_size,
+                        :mutation_rate, :elite_fraction).compact
+        )
+      end
+
+      # Clear population and history (configuration is preserved).
+      on :reset do |state:, **|
+        state.merge(population: [], generation: 0, history: [], best_strategy: nil)
+      end
+
+      private
+
+      def score_population(population, fitness_fn)
+        population.map do |s|
+          score = begin
+            fitness_fn.call(s.config).to_f
+          rescue StandardError
+            0.0
+          end
+          Strategy.new(
+            id: s.id, config: s.config, fitness: score,
+            generation: s.generation, parent_ids: s.parent_ids, mutations: s.mutations
+          )
+        end
+      end
+
+      def run_evolution(state)
+        evaluated = state[:population].reject { |s| s.fitness.nil? }
+        return [nil, nil] if evaluated.empty?
+
+        elite_count = [(evaluated.size * state[:elite_fraction]).ceil, 1].max
+        elite       = evaluated.sort_by { |s| -(s.fitness || 0) }.first(elite_count)
+
+        fitnesses = elite.map { |s| s.fitness.to_f }
+        report    = GenerationReport.new(
+          generation:      state[:generation],
+          population_size: state[:population].size,
+          best_fitness:    fitnesses.max.round(6),
+          mean_fitness:    (fitnesses.sum / fitnesses.size).round(6),
+          best_id:         elite.first.id
+        )
+
+        target   = [state[:population_size], elite.size].max
+        children = []
+        idx      = 0
+        until elite.size + children.size >= target
+          parent = elite[idx % elite.size]
+          child  = mutate_one(parent, state, elite.size + idx)
+          children << child
+          idx += 1
+        end
+
+        [elite + children, report]
+      end
+
+      def mutate_one(parent, state, idx)
+        next_gen = state[:generation] + 1
+        config   = if state[:llm]
+          mutate_with_llm(state[:llm], parent.config, next_gen)
+        else
+          mutate_rule_based(parent.config, state[:mutation_rate])
+        end
+
+        Strategy.new(
+          id:         "gen#{next_gen}_#{idx}",
+          config:     config,
+          fitness:    nil,
+          generation: next_gen,
+          parent_ids: [parent.id],
+          mutations:  detect_mutations(parent.config, config)
+        )
+      end
+
+      def mutate_rule_based(config, rate)
+        config.each_with_object({}) do |(k, v), acc|
+          acc[k] = rand < rate ? mutate_value(v) : v
+        end
+      end
+
+      def mutate_value(value)
+        case value
+        when Numeric           then (value * (0.8 + rand * 0.4)).round(6)
+        when TrueClass, FalseClass then !value
+        when Array             then value.empty? ? value : value.sample([1, rand(value.size)].max)
+        else value
+        end
+      end
+
+      def mutate_with_llm(llm, config, generation)
+        result = llm.call(strategy: config, generation: generation)
+        result.is_a?(Hash) ? result : mutate_rule_based(config, 0.3)
+      rescue StandardError
+        mutate_rule_based(config, 0.3)
+      end
+
+      def detect_mutations(original, mutated)
+        original.keys.filter_map do |k|
+          "#{k}: #{original[k].inspect} → #{mutated[k].inspect}" if original[k] != mutated[k]
+        end
+      end
+      end
+    end
+  end
+end