igniter 0.5.1 → 0.5.2

data/packages/igniter-agents/lib/igniter/ai/agents/critic_agent.rb

@@ -1,165 +0,0 @@
-# frozen_string_literal: true
-
-module Igniter
-  module AI
-    module Agents
-    # Evaluates output quality and optionally retries generation until a score
-    # threshold is met.
-    #
-    # Two evaluation modes:
-    # * **Rule-based** (default) — heuristics based on length and emptiness;
-    #   no external dependencies.
-    # * **LLM-assisted** — delegates to any callable that accepts +output:+ and
-    #   +criteria:+ and returns +{ score: Float, feedback: String }+ or a
-    #   String that will be parsed for a numeric score.
-    #
-    # Scores are on a 0–10 scale. The default passing threshold is 7.0.
-    #
-    # @example Rule-based gate
-    #   ref = CriticAgent.start
-    #   ref.send(:evaluate, output: "Short answer", criteria: "completeness")
-    #   ev = ref.call(:last_evaluation)
-    #   puts ev.passed  # => false (too short)
-    #
-    # @example LLM-assisted with retry
-    #   evaluator = ->(output:, criteria:) {
-    #     result = MyGraderSkill.call(output: output, criteria: criteria)
-    #     { score: result.score, feedback: result.feedback }
-    #   }
-    #   ref = CriticAgent.start(initial_state: { evaluator: evaluator, threshold: 8.0 })
-    #   ref.send(:evaluate_and_retry,
-    #     output:         first_draft,
-    #     criteria:       "accuracy, completeness",
-    #     max_retries:    2,
-    #     generator:      ->(draft:) { improve_draft(draft) },
-    #     generator_args: { draft: first_draft }
-    #   )
-      class CriticAgent < Igniter::Agent
-      # Immutable evaluation result.
-      Evaluation = Struct.new(:score, :feedback, :passed, :criteria, keyword_init: true)
-
-      initial_state evaluator: nil, threshold: 7.0, evaluations: []
-
-      # Evaluate a single output.
-      #
-      # Payload keys:
-      #   output    [String, Object]  — required; the artifact to evaluate
-      #   criteria  [String]          — evaluation criteria (default: "quality, relevance")
-      #   evaluator [#call, nil]      — override state evaluator for this call
-      #   threshold [Float, nil]      — override state threshold for this call
-      on :evaluate do |state:, payload:|
-        agent = new
-        ev    = agent.send(:run_evaluation, payload, state)
-        state.merge(evaluations: state[:evaluations] + [ev])
-      end
-
-      # Evaluate and re-generate until the score passes or retries are exhausted.
-      #
-      # Payload keys:
-      #   output          [String]   — initial output to evaluate
-      #   criteria        [String]   — evaluation criteria
-      #   generator       [#call]    — required; called with **generator_args to produce a new output
-      #   generator_args  [Hash]     — arguments forwarded to generator (default: {})
-      #   max_retries     [Integer]  — maximum re-generation attempts (default: 3)
-      #   evaluator       [#call]    — override state evaluator
-      #   threshold       [Float]    — override state threshold
-      on :evaluate_and_retry do |state:, payload:|
-        agent = new
-        agent.send(:run_evaluate_and_retry, state, payload)
-      end
-
-      # Set default evaluator and/or threshold.
-      #
-      # Payload keys:
-      #   evaluator [#call]  — new default evaluator
-      #   threshold [Float]  — new default threshold
-      on :configure do |state:, payload:|
-        state.merge(
-          evaluator: payload.fetch(:evaluator, state[:evaluator]),
-          threshold: payload.fetch(:threshold, state[:threshold]).to_f
-        )
-      end
-
-      # Sync query — returns the most recent Evaluation, or nil.
-      on :last_evaluation do |state:, **|
-        state[:evaluations].last
-      end
-
-      # Sync query — returns all recorded Evaluation structs.
-      on :evaluations do |state:, **|
-        state[:evaluations]
-      end
-
-      # Clear evaluation history.
-      on :clear do |state:, **|
-        state.merge(evaluations: [])
-      end
-
-      private
-
-      # @return [Evaluation]
-      def run_evaluation(payload, state)
-        output    = payload.fetch(:output)
-        criteria  = payload.fetch(:criteria, "quality, relevance")
-        evaluator = payload.fetch(:evaluator, state[:evaluator])
-        threshold = payload.fetch(:threshold, state[:threshold]).to_f
-
-        score, feedback = evaluator ? llm_score(evaluator, output, criteria)
-                                    : rule_score(output)
-
-        Evaluation.new(
-          score:    score.to_f,
-          feedback: feedback.to_s,
-          passed:   score.to_f >= threshold,
-          criteria: criteria
-        )
-      end
-
-      # @return [Hash] updated state with all evaluations recorded
-      def run_evaluate_and_retry(state, payload)
-        max_retries    = payload.fetch(:max_retries, 3)
-        generator      = payload.fetch(:generator)
-        generator_args = payload.fetch(:generator_args, {})
-        output         = payload.fetch(:output)
-        all_evals      = []
-
-        (max_retries + 1).times do |attempt|
-          output = generator.call(**generator_args) if attempt.positive?
-          ev     = run_evaluation(payload.merge(output: output), state)
-          all_evals << ev
-          break if ev.passed
-        end
-
-        state.merge(evaluations: state[:evaluations] + all_evals)
-      end
-
-      # Call the user-supplied evaluator. Accepts two return shapes:
-      #   Hash  with :score / :feedback keys
-      #   String  (we scan for the first number in the text as the score)
-      def llm_score(evaluator, output, criteria)
-        result = evaluator.call(output: output, criteria: criteria)
-        case result
-        when Hash
-          [result.fetch(:score, 5.0), result.fetch(:feedback, "")]
-        else
-          text  = result.to_s
-          score = text.match(/\b(\d+(?:\.\d+)?)\b/)&.captures&.first&.to_f || 5.0
-          [score, text]
-        end
-      rescue StandardError => e
-        [0.0, "Evaluator error: #{e.message}"]
-      end
-
-      # Minimal rule-based heuristic (no LLM required).
-      def rule_score(output)
-        text = output.to_s.strip
-        return [0.0, "Output is empty"]          if text.empty?
-        return [3.0, "Output is very short"]     if text.length < 50
-        return [5.5, "Output is below average length"] if text.length < 200
-
-        [7.5, "Output meets basic length criteria"]
-      end
-      end
-    end
-  end
-end

data/packages/igniter-agents/lib/igniter/ai/agents/evaluator_agent.rb

@@ -1,195 +0,0 @@
-# frozen_string_literal: true
-
-module Igniter
-  module AI
-    module Agents
-    # Tracks performance metrics for named subjects (agents, services,
-    # contracts), computes weighted aggregate scores, and compares subjects
-    # against each other or against stored baselines.
-    #
-    # Scoring:
-    # * Metrics are grouped by name; the last 20 readings are averaged.
-    # * Per-subject or global weights scale each metric's contribution.
-    # * When a baseline is set the score is normalised to 0–100 relative to it;
-    #   without a baseline the raw weighted average is returned.
-    # * Grades: A ≥ 90, B ≥ 75, C ≥ 60, D otherwise.
-    #
-    # @example Track two services and compare
-    #   ref = EvaluatorAgent.start
-    #   ref.send(:record_metric, subject: :api, name: :throughput, value: 850)
-    #   ref.send(:record_metric, subject: :api, name: :error_rate,  value: 2.1)
-    #   ref.send(:set_baseline,  subject: :api, baseline: 800)
-    #   ref.send(:evaluate,      subject: :api)
-    #   ev = ref.call(:evaluations, subject: :api).last
-    #   puts ev.grade   # => "A"
-      class EvaluatorAgent < Igniter::Agent
-      MetricRecord = Struct.new(:name, :value, :recorded_at, keyword_init: true)
-      Evaluation   = Struct.new(:subject, :score, :grade, :metrics,
-                                :recorded_at, keyword_init: true)
-      Comparison   = Struct.new(:subject_a, :subject_b, :winner, :delta,
-                                keyword_init: true)
-
-      GRADES = [
-        [90.0, "A"],
-        [75.0, "B"],
-        [60.0, "C"],
-        [  0.0, "D"]
-      ].freeze
-
-      # subjects: Hash<String, { metrics: Array<MetricRecord>, baseline: Float?, weights: Hash }>
-      initial_state \
-        subjects:    {},
-        evaluations: [],
-        weights:     {}
-
-      # Record a metric reading for a subject.
-      #
-      # Payload keys:
-      #   subject [String, Symbol]  — subject identifier
-      #   name    [String, Symbol]  — metric name
-      #   value   [Numeric]         — metric value
-      on :record_metric do |state:, payload:|
-        subject = payload.fetch(:subject).to_s
-        metric  = MetricRecord.new(
-          name:        payload.fetch(:name).to_s,
-          value:       payload.fetch(:value).to_f,
-          recorded_at: Time.now
-        )
-        entry   = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
-        updated = entry.merge(metrics: (entry[:metrics] + [metric]).last(200))
-        state.merge(subjects: state[:subjects].merge(subject => updated))
-      end
-
-      # Set the reference baseline value for a subject.
-      # Scores will be expressed as a percentage of this baseline.
-      #
-      # Payload keys:
-      #   subject  [String, Symbol]
-      #   baseline [Numeric]
-      on :set_baseline do |state:, payload:|
-        subject  = payload.fetch(:subject).to_s
-        baseline = payload.fetch(:baseline).to_f
-        entry    = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
-        updated  = entry.merge(baseline: baseline)
-        state.merge(subjects: state[:subjects].merge(subject => updated))
-      end
-
-      # Set per-metric weights for a subject.
-      #
-      # Payload keys:
-      #   subject [String, Symbol]
-      #   weights [Hash<String, Numeric>]  — metric name → weight
-      on :set_weights do |state:, payload:|
-        subject = payload.fetch(:subject).to_s
-        weights = payload.fetch(:weights).transform_keys(&:to_s)
-        entry   = state[:subjects].fetch(subject, { metrics: [], baseline: nil, weights: {} })
-        updated = entry.merge(weights: weights)
-        state.merge(subjects: state[:subjects].merge(subject => updated))
-      end
-
-      # Compute and store an Evaluation for a subject.
-      #
-      # Payload keys:
-      #   subject [String, Symbol]
-      on :evaluate do |state:, payload:|
-        agent    = new
-        ev       = agent.send(:compute_evaluation, payload.fetch(:subject).to_s, state)
-        next state unless ev
-
-        state.merge(evaluations: state[:evaluations] + [ev])
-      end
-
-      # Sync query — compare the most recent evaluations of two subjects.
-      #
-      # Payload keys:
-      #   a [String, Symbol]  — first subject
-      #   b [String, Symbol]  — second subject
-      #
-      # @return [Comparison, nil]
-      on :compare do |state:, payload:|
-        a = state[:evaluations].select { |e| e.subject == payload.fetch(:a).to_s }.last
-        b = state[:evaluations].select { |e| e.subject == payload.fetch(:b).to_s }.last
-        next nil unless a && b
-
-        delta  = (a.score - b.score).round(4)
-        winner = if delta > 0 then a.subject
-                 elsif delta < 0 then b.subject
-                 else :tie
-                 end
-        Comparison.new(subject_a: a.subject, subject_b: b.subject,
-                       winner: winner, delta: delta.abs)
-      end
-
-      # Sync query — all evaluations, optionally filtered by subject.
-      #
-      # Payload keys:
-      #   subject [String, Symbol, nil]
-      #
-      # @return [Array<Evaluation>]
-      on :evaluations do |state:, payload:|
-        filter = payload&.fetch(:subject, nil)
-        evs    = state[:evaluations]
-        evs    = evs.select { |e| e.subject == filter.to_s } if filter
-        evs.dup
-      end
-
-      # Sync query — list registered subject names.
-      #
-      # @return [Array<String>]
-      on :subjects do |state:, **|
-        state[:subjects].keys
-      end
-
-      # Set global default weights (applied when a subject has no per-metric weight).
-      #
-      # Payload keys:
-      #   weights [Hash<String, Numeric>]
-      on :configure do |state:, payload:|
-        state.merge(payload.slice(:weights).compact)
-      end
-
-      # Clear all subjects and evaluations.
-      on :reset do |state:, **|
-        state.merge(subjects: {}, evaluations: [])
-      end
-
-      private
-
-      def compute_evaluation(name, state)
-        data = state[:subjects][name]
-        return nil unless data && data[:metrics].any?
-
-        weights  = state[:weights].merge(data[:weights] || {})
-        metrics  = data[:metrics]
-        baseline = data[:baseline]
-
-        grouped = metrics.group_by(&:name)
-        score_parts = grouped.map do |mname, records|
-          avg    = records.last(20).sum(&:value) / [records.last(20).size, 1].max
-          weight = (weights[mname] || 1.0).to_f
-          [avg * weight, weight]
-        end
-
-        total_weight = [score_parts.sum { |_, w| w }, 0.001].max
-        raw          = score_parts.sum { |v, _| v } / total_weight
-
-        score = if baseline && baseline > 0
-          [(raw / baseline * 100).round(4), 100.0].min
-        else
-          raw.round(4)
-        end
-
-        grade = GRADES.find { |threshold, _| score.to_f >= threshold }&.last || "D"
-
-        Evaluation.new(
-          subject:     name,
-          score:       score,
-          grade:       grade,
-          metrics:     grouped.keys,
-          recorded_at: Time.now
-        )
-      end
-      end
-    end
-  end
-end

data/packages/igniter-agents/lib/igniter/ai/agents/evolution_agent.rb

@@ -1,288 +0,0 @@
-# frozen_string_literal: true
-
-module Igniter
-  module AI
-    module Agents
-    # Mission: become better.
-    #
-    # An evolutionary self-improvement agent that maintains a population of
-    # strategies (prompt variants, parameter configs, tool selections — any
-    # Hash), evaluates their fitness, selects survivors, generates mutations,
-    # and tracks lineage across generations.
-    #
-    # The agent is intentionally domain-agnostic. Strategies are plain Hashes
-    # whose semantics are defined by the caller's +fitness_fn+.
-    #
-    # Life-cycle of one generation:
-    #   seed → evaluate_population → evolve   (or: seed → run_generation × N)
-    #
-    # Mutation:
-    # * **Rule-based** (default) — numeric values are scaled ±20 %, booleans
-    #   are flipped, arrays are sub-sampled. Strings are kept unchanged.
-    # * **LLM-assisted** — delegates to any callable that accepts
-    #   +strategy:+ and +generation:+ and returns a Hash.
-    #
-    # @example Evolve a prompt configuration
-    #   fitness = ->(config) { MyEvaluator.score(config) }
-    #   ref = EvolutionAgent.start(initial_state: {
-    #     fitness_fn: fitness,
-    #     population_size: 4
-    #   })
-    #   ref.send(:seed, strategies: [
-    #     { temperature: 0.7, max_tokens: 512 },
-    #     { temperature: 0.3, max_tokens: 256 }
-    #   ])
-    #   3.times { ref.send(:run_generation) }
-    #   best = ref.call(:best)   # => Strategy struct
-      class EvolutionAgent < Igniter::Agent
-      # An individual strategy in the population.
-      Strategy = Struct.new(:id, :config, :fitness, :generation,
-                            :parent_ids, :mutations, keyword_init: true)
-
-      # Summary record for one completed generation.
-      GenerationReport = Struct.new(:generation, :population_size, :best_fitness,
-                                    :mean_fitness, :best_id, keyword_init: true)
-
-      initial_state \
-        population:       [],
-        generation:       0,
-        history:          [],
-        best_strategy:    nil,
-        fitness_fn:       nil,
-        llm:              nil,
-        population_size:  6,
-        mutation_rate:    0.3,
-        elite_fraction:   0.5
-
-      # Seed the population with an initial set of strategy configs.
-      # Resets generation counter and history.
-      #
-      # Payload keys:
-      #   strategies [Array<Hash>]  — initial strategy configs
-      on :seed do |state:, payload:|
-        pop = Array(payload.fetch(:strategies)).each_with_index.map do |cfg, i|
-          Strategy.new(
-            id:         "gen0_#{i}",
-            config:     cfg,
-            fitness:    nil,
-            generation: 0,
-            parent_ids: [],
-            mutations:  []
-          )
-        end
-        state.merge(population: pop, generation: 0, history: [], best_strategy: nil)
-      end
-
-      # Score every strategy in the population using the fitness function.
-      # Strategies that raise are assigned fitness 0.0.
-      #
-      # Payload keys:
-      #   fitness_fn [#call, nil]  — override the state fitness function
-      on :evaluate_population do |state:, payload:|
-        fitness_fn = (payload && payload[:fitness_fn]) || state[:fitness_fn]
-        next state unless fitness_fn
-
-        agent     = new
-        evaluated = agent.send(:score_population, state[:population], fitness_fn)
-        best      = evaluated.max_by { |s| s.fitness || -Float::INFINITY }
-        state.merge(population: evaluated, best_strategy: best)
-      end
-
-      # Select elite survivors, generate children by mutation, increment generation.
-      # Requires the population to be already evaluated (see +:evaluate_population+).
-      on :evolve do |state:, payload:|
-        agent          = new
-        new_pop, report = agent.send(:run_evolution, state)
-        next state unless new_pop
-
-        best = new_pop.max_by { |s| s.fitness || -Float::INFINITY }
-        state.merge(
-          population:    new_pop,
-          generation:    state[:generation] + 1,
-          history:       state[:history] + [report],
-          best_strategy: best
-        )
-      end
-
-      # Convenience: evaluate the current population then evolve — one full cycle.
-      #
-      # Payload keys:
-      #   fitness_fn [#call, nil]  — override the state fitness function
-      on :run_generation do |state:, payload:|
-        fitness_fn = (payload && payload[:fitness_fn]) || state[:fitness_fn]
-        next state unless fitness_fn
-
-        agent = new
-
-        # evaluate
-        evaluated_pop = agent.send(:score_population, state[:population], fitness_fn)
-        with_scores   = state.merge(population: evaluated_pop)
-
-        # evolve
-        new_pop, report = agent.send(:run_evolution, with_scores)
-        next with_scores unless new_pop
-
-        best = new_pop.max_by { |s| s.fitness || -Float::INFINITY }
-        with_scores.merge(
-          population:    new_pop,
-          generation:    state[:generation] + 1,
-          history:       state[:history] + [report],
-          best_strategy: best
-        )
-      end
-
-      # Add a single mutated child of a given strategy to the population.
-      #
-      # Payload keys:
-      #   id [String]  — id of the parent strategy
-      on :mutate_strategy do |state:, payload:|
-        parent = state[:population].find { |s| s.id == payload.fetch(:id) }
-        next state unless parent
-
-        agent = new
-        child = agent.send(:mutate_one, parent, state, state[:population].size)
-        state.merge(population: state[:population] + [child])
-      end
-
-      # Sync query — best evaluated strategy so far.
-      #
-      # @return [Strategy, nil]
-      on :best do |state:, **|
-        state[:best_strategy]
-      end
-
-      # Sync query — current population.
-      #
-      # @return [Array<Strategy>]
-      on :population do |state:, **|
-        state[:population].dup
-      end
-
-      # Sync query — evolution history (one GenerationReport per generation).
-      #
-      # @return [Array<GenerationReport>]
-      on :history do |state:, **|
-        state[:history].dup
-      end
-
-      # Sync query — current generation number.
-      #
-      # @return [Integer]
-      on :generation do |state:, **|
-        state[:generation]
-      end
-
-      # Update agent configuration.
-      #
-      # Payload keys:
-      #   fitness_fn      [#call]
-      #   llm             [#call]
-      #   population_size [Integer]
-      #   mutation_rate   [Float]   — 0.0–1.0
-      #   elite_fraction  [Float]   — 0.0–1.0
-      on :configure do |state:, payload:|
-        state.merge(
-          payload.slice(:fitness_fn, :llm, :population_size,
-                        :mutation_rate, :elite_fraction).compact
-        )
-      end
-
-      # Clear population and history (configuration is preserved).
-      on :reset do |state:, **|
-        state.merge(population: [], generation: 0, history: [], best_strategy: nil)
-      end
-
-      private
-
-      def score_population(population, fitness_fn)
-        population.map do |s|
-          score = begin
-            fitness_fn.call(s.config).to_f
-          rescue StandardError
-            0.0
-          end
-          Strategy.new(
-            id: s.id, config: s.config, fitness: score,
-            generation: s.generation, parent_ids: s.parent_ids, mutations: s.mutations
-          )
-        end
-      end
-
-      def run_evolution(state)
-        evaluated = state[:population].reject { |s| s.fitness.nil? }
-        return [nil, nil] if evaluated.empty?
-
-        elite_count = [(evaluated.size * state[:elite_fraction]).ceil, 1].max
-        elite       = evaluated.sort_by { |s| -(s.fitness || 0) }.first(elite_count)
-
-        fitnesses = elite.map { |s| s.fitness.to_f }
-        report    = GenerationReport.new(
-          generation:      state[:generation],
-          population_size: state[:population].size,
-          best_fitness:    fitnesses.max.round(6),
-          mean_fitness:    (fitnesses.sum / fitnesses.size).round(6),
-          best_id:         elite.first.id
-        )
-
-        target   = [state[:population_size], elite.size].max
-        children = []
-        idx      = 0
-        until elite.size + children.size >= target
-          parent = elite[idx % elite.size]
-          child  = mutate_one(parent, state, elite.size + idx)
-          children << child
-          idx += 1
-        end
-
-        [elite + children, report]
-      end
-
-      def mutate_one(parent, state, idx)
-        next_gen = state[:generation] + 1
-        config   = if state[:llm]
-          mutate_with_llm(state[:llm], parent.config, next_gen)
-        else
-          mutate_rule_based(parent.config, state[:mutation_rate])
-        end
-
-        Strategy.new(
-          id:         "gen#{next_gen}_#{idx}",
-          config:     config,
-          fitness:    nil,
-          generation: next_gen,
-          parent_ids: [parent.id],
-          mutations:  detect_mutations(parent.config, config)
-        )
-      end
-
-      def mutate_rule_based(config, rate)
-        config.each_with_object({}) do |(k, v), acc|
-          acc[k] = rand < rate ? mutate_value(v) : v
-        end
-      end
-
-      def mutate_value(value)
-        case value
-        when Numeric           then (value * (0.8 + rand * 0.4)).round(6)
-        when TrueClass, FalseClass then !value
-        when Array             then value.empty? ? value : value.sample([1, rand(value.size)].max)
-        else value
-        end
-      end
-
-      def mutate_with_llm(llm, config, generation)
-        result = llm.call(strategy: config, generation: generation)
-        result.is_a?(Hash) ? result : mutate_rule_based(config, 0.3)
-      rescue StandardError
-        mutate_rule_based(config, 0.3)
-      end
-
-      def detect_mutations(original, mutated)
-        original.keys.filter_map do |k|
-          "#{k}: #{original[k].inspect} → #{mutated[k].inspect}" if original[k] != mutated[k]
-        end
-      end
-      end
-    end
-  end
-end