active_harness 0.1.0 → 0.2.1

data/lib/active_harness/tribunal.rb

@@ -0,0 +1,216 @@
+begin
+  require "concurrent"
+rescue LoadError
+  raise LoadError,
+    "ActiveHarness::Tribunal requires the 'concurrent-ruby' gem. " \
+    "Add `gem 'concurrent-ruby'` to your Gemfile."
+end
+
+module ActiveHarness
+  # Can be used directly or subclassed with a class-level DSL.
+  #
+  # Direct usage:
+  #   tribunal = ActiveHarness::Tribunal.new(
+  #     input:   "Is this message toxic?",
+  #     context: { user_id: 42 },
+  #     agents:  [ToxicityAgent, BiasAgent, SpamAgent],
+  #     timeout: 7
+  #   )
+  #   tribunal.on(:after_agent) { |result| puts result.model }
+  #   tribunal.process { |results| results.all? { |r| r.parsed["result"] == true } }
+  #   tribunal.call
+  #
+  # Subclass with DSL:
+  #   class ContentQualityTribunal < ActiveHarness::Tribunal
+  #     agents PolitenessAgent, ConstructivenessAgent
+  #     on(:after_agent) { |result| puts result.model }
+  #     process { |results| results.all? { |r| r.parsed["result"] == true } }
+  #   end
+  #   ContentQualityTribunal.new(input: "...").call
+  #
+  class Tribunal
+    VALID_HOOKS = %i[
+      before_call
+      after_agent
+      agent_error
+      after_call
+      before_verdict
+      after_verdict
+    ].freeze
+
+    # -------------------------------------------------------------------------
+    # Class-level DSL — used when subclassing ActiveHarness::Tribunal
+    # -------------------------------------------------------------------------
+    class << self
+      # Declare agents at the class level.
+      #   agents PolitenessAgent, ConstructivenessAgent
+      def agents(*list)
+        tribunal_config[:agents] = list.flatten
+      end
+
+      # Class-level hook registration.
+      #   on(:after_agent) { |result| puts result.model }
+      def on(event, &block)
+        unless VALID_HOOKS.include?(event)
+          raise ArgumentError, "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.join(", ")}"
+        end
+
+        tribunal_config[:hooks][event] = block
+      end
+
+      # Rails-style aliases for +on+:
+      #
+      #   before :call                     do ... end   # → on :before_call
+      #   before :agent                    do ... end   # → on :before_agent (not used yet)
+      #   before :verdict                  do |r| end   # → on :before_verdict
+      #   after  :call                     do ... end   # → on :after_call
+      #   after  :agent                    do |r| end   # → on :after_agent
+      #   after  :verdict                  do |v| end   # → on :after_verdict
+      #   callback :agent_error            do |n,e| end # → on :agent_error
+      def before(event, &block)
+        on(:"before_#{event}", &block)
+      end
+
+      def after(event, &block)
+        on(:"after_#{event}", &block)
+      end
+
+      def callback(event, &block)
+        on(event, &block)
+      end
+
+      # Class-level process block.
+      #   process { |results| results.all? { |r| r.parsed["result"] == true } }
+      def process(&block)
+        tribunal_config[:process] = block
+      end
+
+      def tribunal_config
+        @tribunal_config ||= { agents: [], hooks: {} }
+      end
+
+      def inherited(subclass)
+        subclass.instance_variable_set(:@tribunal_config, { agents: [], hooks: {} })
+      end
+    end
+
+    attr_accessor :input
+    attr_reader :results, :errors, :verdict, :execution_time, :agent_execution_times
+
+    def initialize(input: nil, context: {}, agents: nil, timeout: 7)
+      config = self.class.tribunal_config
+
+      @input         = input
+      @context       = context
+      @agents        = agents || config[:agents]
+      @timeout       = timeout
+      @process_block = config[:process]
+      @hooks         = config[:hooks].dup
+      @results       = []
+      @errors        = []
+      @verdict       = nil
+      @execution_time       = nil
+      @agent_execution_times = []
+    end
+
+    # Instance-level hook registration — overrides class-level hooks.
+    # :before_verdict is a transform hook: its return value replaces the results array.
+    def on(event, &block)
+      unless VALID_HOOKS.include?(event)
+        raise ArgumentError, "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.join(", ")}"
+      end
+
+      @hooks[event] = block
+      self
+    end
+
+    # Instance-level process block — overrides class-level block.
+    def process(&block)
+      @process_block = block
+      self
+    end
+
+    # Run all agents in parallel, then compute the verdict.
+    # Returns self so calls can be chained: tribunal.call.verdict
+    #
+    # Behaviour on failure:
+    #   - If some agents fail/timeout, their errors are in #errors and
+    #     #results contains only successful results.
+    #   - If ALL agents fail/timeout, raises Errors::AllAgentsFailed.
+    def call
+      agents = resolve_agents
+      run_hook(:before_call)
+
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      futures = agents.map do |agent|
+        t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        future = Concurrent::Future.execute { agent.call }
+        [future, t0]
+      end
+
+      @results              = []
+      @errors               = []
+      @agent_execution_times = []
+
+      futures.each_with_index do |(future, t0), index|
+        future.wait(@timeout)
+        elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0).round(3)
+        @agent_execution_times << { agent: agents[index].class.name, time: elapsed }
+
+        if future.fulfilled?
+          @results << future.value
+          run_hook(:after_agent, future.value)
+        elsif future.incomplete?
+          error = Errors::TimeoutError.new(
+            "Agent #{agents[index].class.name} timed out after #{@timeout}s"
+          )
+          @errors << { agent: agents[index].class.name, error: error }
+          run_hook(:agent_error, agents[index].class.name, error)
+        else
+          @errors << { agent: agents[index].class.name, error: future.reason }
+          run_hook(:agent_error, agents[index].class.name, future.reason)
+        end
+      end
+
+      @execution_time = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at).round(3)
+
+      run_hook(:after_call, @results, @errors)
+
+      if @results.empty?
+        messages = @errors.map { |e| "#{e[:agent]}: #{e[:error].message}" }.join("; ")
+        raise Errors::AllAgentsFailed, "All agents failed — #{messages}"
+      end
+
+      verdict_input = transform_hook(:before_verdict, @results)
+      @verdict = @process_block ? @process_block.call(verdict_input) : nil
+      run_hook(:after_verdict, @verdict)
+
+      self
+    end
+
+    private
+
+    def run_hook(event, *args)
+      @hooks[event]&.call(*args)
+    end
+
+    # Like run_hook but uses the return value to replace the passed value.
+    def transform_hook(event, value)
+      return value unless @hooks[event]
+
+      @hooks[event].call(value)
+    end
+
+    def resolve_agents
+      @agents.map do |agent|
+        if agent.is_a?(Class)
+          agent.new(input: @input, context: @context.dup)
+        else
+          agent.input = @input if @input
+          agent
+        end
+      end
+    end
+  end
+end

data/lib/active_harness.rb

@@ -1,49 +1,20 @@
-require "active_harness/core/version"
-require "active_harness/core/errors"
-require "active_harness/core/configuration"
-require "active_harness/payload"
-
-require "active_harness/http/client"
-require "active_harness/http/retry_policy"
-
-require "active_harness/rate_limit/request_limiter"
-require "active_harness/rate_limit/risk_holdback"
-
-require "active_harness/models/model_request"
-require "active_harness/models/model_response"
-require "active_harness/results/input_result"
-require "active_harness/results/debug_result"
-require "active_harness/results/result"
-
-require "active_harness/providers/base"
-require "active_harness/providers/openai"
-require "active_harness/providers/openrouter"
-require "active_harness/providers/anthropic"
-require "active_harness/providers/google"
-
-require "active_harness/prompts/guard_system_prompt"
-
-require "active_harness/pipeline/provider_registry"
-require "active_harness/pipeline/prompt_builder"
-require "active_harness/pipeline/output_parser"
-require "active_harness/pipeline/fallback_runner"
-require "active_harness/pipeline/guard_runner"
-require "active_harness/pipeline/engine"
-require "active_harness/agent"
+require_relative "active_harness/core/errors"
+require_relative "active_harness/result"
+require_relative "active_harness/http/client"
+require_relative "active_harness/http/streaming_client"
+require_relative "active_harness/providers/base"
+require_relative "active_harness/providers/openai"
+require_relative "active_harness/providers/openrouter"
+require_relative "active_harness/providers/groq"
+require_relative "active_harness/providers/gemini"
+require_relative "active_harness/providers/anthropic"
+require_relative "active_harness/memory"
+require_relative "active_harness/agent"
+require_relative "active_harness/tribunal"
+require_relative "active_harness/pipeline"
+
+require_relative "active_harness/railtie" if defined?(Rails::Railtie)
 
 module ActiveHarness
-  class << self
-    def configure
-      yield configuration
-    end
-
-    def configuration
-      @configuration ||= Configuration.new
-    end
-    alias config configuration
-
-    def reset!
-      @configuration = nil
-    end
-  end
+  VERSION = "0.2.0"
 end

data/lib/generators/active_harness/agent/agent_generator.rb

@@ -0,0 +1,16 @@
+require "rails/generators"
+
+module ActiveHarness
+  module Generators
+    class AgentGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates an ActiveHarness agent in app/ai/agents/"
+
+      def create_agent
+        template "agent.rb.tt",
+                 "app/ai/agents/#{file_name}_agent.rb"
+      end
+    end
+  end
+end

data/lib/generators/active_harness/agent/templates/agent.rb.tt

@@ -0,0 +1,8 @@
+class <%= class_name %>Agent < ActiveHarness::Agent
+  system_prompt <%= class_name %>Prompt
+
+  model do
+    use      provider: :openrouter, model: "mistralai/mistral-nemo"
+    fallback provider: :openrouter, model: "meta-llama/llama-3.1-8b-instruct"
+  end
+end

data/lib/generators/active_harness/install/install_generator.rb

@@ -0,0 +1,54 @@
+require "rails/generators"
+
+module ActiveHarness
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      desc "Creates the app/ai/ directory structure for ActiveHarness"
+
+      source_root File.expand_path("templates", __dir__)
+
+      APP_AI_DIRS = %w[agents prompts tribunals pipelines memory].freeze
+
+      def create_structure
+        APP_AI_DIRS.each do |dir|
+          empty_directory "app/ai/#{dir}"
+          copy_templates_if_empty(dir)
+        end
+      end
+
+      def copy_controller
+        target = File.join(destination_root, "app", "controllers", "ai_test_support_controller.rb")
+        return if File.exist?(target)
+
+        copy_file "controllers/ai_controller.rb",
+                  "app/controllers/ai_test_support_controller.rb"
+      end
+
+      def inject_routes
+        route <<~ROUTES.strip
+          # ActiveHarness — AI test support endpoints
+          post "ai/agent",        to: "ai_test_support#agent"
+          post "ai/agent_memory", to: "ai_test_support#agent_memory"
+          post "ai/tribunal",     to: "ai_test_support#tribunal"
+          post "ai/pipeline",     to: "ai_test_support#pipeline"
+          get  "ai/agent_stream", to: "ai_test_support#agent_stream"
+        ROUTES
+      end
+
+      private
+
+      def copy_templates_if_empty(dir)
+        target_dir = File.join(destination_root, "app", "ai", dir)
+        return if Dir.glob(File.join(target_dir, "*.rb")).any?
+
+        template_dir = File.join(self.class.source_root, dir)
+        return unless Dir.exist?(template_dir)
+
+        Dir[File.join(template_dir, "*.rb")].sort.each do |tpl|
+          copy_file File.join(dir, File.basename(tpl)),
+                    File.join("app", "ai", dir, File.basename(tpl))
+        end
+      end
+    end
+  end
+end

data/lib/generators/active_harness/install/templates/agents/test_support_agent.rb

@@ -0,0 +1,10 @@
+require_relative "../prompts/test_support_prompt"
+
+class TestSupportAgent < ActiveHarness::Agent
+  system_prompt TestSupportPrompt
+
+  model do
+    use      provider: :openrouter, model: "mistralai/mistral-nemo"
+    fallback provider: :openrouter, model: "meta-llama/llama-3.1-8b-instruct"
+  end
+end

data/lib/generators/active_harness/install/templates/agents/test_support_guard_agent.rb

@@ -0,0 +1,11 @@
+require_relative "../prompts/test_support_guard_prompt"
+
+class TestSupportGuardAgent < ActiveHarness::Agent
+  system_prompt TestSupportGuardPrompt
+  format :json
+
+  model do
+    use      provider: :openrouter, model: "meta-llama/llama-3.1-8b-instruct"
+    fallback provider: :openrouter, model: "mistralai/mistral-nemo"
+  end
+end

data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb

@@ -0,0 +1,105 @@
+class AiTestSupportController < ApplicationController
+  include ActionController::Live
+
+  # ---------------------------------------------------------------------------
+  # POST /ai/agent
+  # body: { input: "What is your return policy?" }
+  # ---------------------------------------------------------------------------
+  def agent
+    result = TestSupportAgent.call(input: params.require(:input))
+
+    render json: {
+      output: result.output,
+      model:  result.model,
+      time:   result.execution_time
+    }
+  end
+
+  # ---------------------------------------------------------------------------
+  # POST /ai/agent_memory
+  # body: { input: "Does that apply to accessories?", session_id: "user_42" }
+  #
+  # Uses AppMemory so the same session keeps conversational context
+  # across multiple requests.
+  # ---------------------------------------------------------------------------
+  def agent_memory
+    memory = TestSupportMemory.new(session_id: params.require(:session_id))
+    result = TestSupportAgent.call(input: params.require(:input), memory: memory)
+
+    render json: {
+      output: result.output,
+      model:  result.model,
+      time:   result.execution_time,
+      turns:  memory.size
+    }
+  end
+
+  # ---------------------------------------------------------------------------
+  # POST /ai/tribunal
+  # body: { input: "Buy cheap pills now!!!" }
+  #
+  # Returns verdict: true (safe) or false (rejected).
+  # ---------------------------------------------------------------------------
+  def tribunal
+    result = TestSupportGuardTribunal.call(input: params.require(:input))
+
+    render json: {
+      verdict: result.verdict,
+      time:    result.execution_time
+    }
+  end
+
+  # ---------------------------------------------------------------------------
+  # POST /ai/pipeline
+  # body: { input: "What is your return policy?" }
+  #
+  # Runs the full TestSupportPipeline.
+  # If a guard step stops the pipeline early, stopped: true is returned.
+  # ---------------------------------------------------------------------------
+  def pipeline
+    pipe = TestSupportPipeline.new(input: params.require(:input))
+    pipe.call
+
+    if pipe.stopped?
+      render json: { stopped: true, stopped_at: pipe.stopped_at }
+    else
+      render json: { stopped: false, output: pipe.output }
+    end
+  end
+
+  # ---------------------------------------------------------------------------
+  # GET /ai/agent_stream?input=What+is+your+return+policy%3F
+  #
+  # Streams the response token by token using Server-Sent Events.
+  # Each token arrives as:   data: {"token":"..."}
+  # End of stream is marked: data: {"done":true}
+  #
+  # JavaScript client example:
+  #   const es = new EventSource('/ai/agent_stream?input=Hello');
+  #   es.onmessage = ({ data }) => {
+  #     const { token, done } = JSON.parse(data);
+  #     if (done) { es.close(); return; }
+  #     document.querySelector('#output').insertAdjacentText('beforeend', token);
+  #   };
+  # ---------------------------------------------------------------------------
+  def agent_stream
+    input = params.require(:input)
+
+    response.headers["Content-Type"]  = "text/event-stream"
+    response.headers["Cache-Control"] = "no-cache"
+    response.headers["X-Accel-Buffering"] = "no"  # disable nginx buffering
+
+    sse = ActionController::Live::SSE.new(response.stream, event: "message")
+
+    TestSupportAgent.call(
+      input:  input,
+      stream: ->(token) { sse.write({ token: token }.to_json) }
+    )
+
+    sse.write({ done: true }.to_json)
+  rescue ActionController::Live::ClientDisconnected
+    # client closed the connection — normal, nothing to do
+  ensure
+    sse.close
+  end
+end

data/lib/generators/active_harness/install/templates/memory/test_support_memory.rb

@@ -0,0 +1,16 @@
+class TestSupportMemory < ActiveHarness::Memory
+  # Usage: TestSupportMemory.new(session_id: "user_42")
+  #
+  # Wraps ActiveHarness::Memory with project defaults so callers
+  # only need to pass a session_id.
+  def initialize(session_id:)
+    super(
+      session_id:   session_id,
+      depth:        10,
+      adapter:      :file,
+      path:         Rails.root.join("storage", "ai", "memory").to_s,
+      storage_size: 200,
+      pretty:       Rails.env.development?
+    )
+  end
+end

data/lib/generators/active_harness/install/templates/pipelines/test_support_pipeline.rb

@@ -0,0 +1,31 @@
+require_relative "../tribunals/test_support_guard_tribunal"
+require_relative "../agents/test_support_agent"
+
+# Two-step pipeline: spam guard → answer.
+# Add more steps between them as needed.
+class TestSupportPipeline < ActiveHarness::Pipeline
+  # Step 1 — GUARD: reject spam before spending tokens on an answer
+  step :spam_guard do
+    use TestSupportGuardTribunal
+    stop_if ->(result) { result.verdict == false }
+  end
+
+  # Step 2 — RESPOND: generate the actual answer
+  step :respond, TestSupportAgent
+
+  before :step do |step_name, _payload|
+    puts "[pipeline] → :#{step_name}"
+  end
+
+  after :step do |step_name, _result|
+    puts "[pipeline] ✓ :#{step_name}"
+  end
+
+  callback :stopped do |step_name, _result|
+    puts "[pipeline] ✗ STOPPED at :#{step_name}"
+  end
+
+  callback :complete do |_last_result|
+    puts "[pipeline] ✓ complete"
+  end
+end

data/lib/generators/active_harness/install/templates/prompts/test_support_guard_prompt.rb

@@ -0,0 +1,9 @@
+class TestSupportGuardPrompt
+  def call
+    <<~PROMPT.strip
+      You are a spam detection filter.
+      Analyze the message and reply ONLY with valid JSON, no markdown:
+      {"spam": true, "reason": "..."} or {"spam": false, "reason": "..."}
+    PROMPT
+  end
+end

data/lib/generators/active_harness/install/templates/prompts/test_support_prompt.rb

@@ -0,0 +1,5 @@
+class TestSupportPrompt
+  def call
+    "You are a concise and helpful assistant. Answer in 1-2 sentences."
+  end
+end

data/lib/generators/active_harness/install/templates/tribunals/test_support_guard_tribunal.rb

@@ -0,0 +1,11 @@
+require_relative "../agents/test_support_guard_agent"
+
+# Runs TestSupportGuardAgent in parallel (single agent here, extendable).
+# Verdict is true (safe) when no spam is detected.
+class TestSupportGuardTribunal < ActiveHarness::Tribunal
+  agents TestSupportGuardAgent
+
+  process do |results|
+    results.none? { |r| r.parsed["spam"] == true }
+  end
+end

data/lib/generators/active_harness/memory/memory_generator.rb

@@ -0,0 +1,16 @@
+require "rails/generators"
+
+module ActiveHarness
+  module Generators
+    class MemoryGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates an ActiveHarness memory class in app/ai/memory/"
+
+      def create_memory
+        template "memory.rb.tt",
+                 "app/ai/memory/#{file_name}_memory.rb"
+      end
+    end
+  end
+end

data/lib/generators/active_harness/memory/templates/memory.rb.tt

@@ -0,0 +1,12 @@
+class <%= class_name %>Memory < ActiveHarness::Memory
+  def initialize(session_id:)
+    super(
+      session_id:   session_id,
+      depth:        10,
+      adapter:      :file,
+      path:         Rails.root.join("storage", "ai", "memory").to_s,
+      storage_size: 200,
+      pretty:       Rails.env.development?
+    )
+  end
+end

data/lib/generators/active_harness/pipeline/pipeline_generator.rb

@@ -0,0 +1,16 @@
+require "rails/generators"
+
+module ActiveHarness
+  module Generators
+    class PipelineGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates an ActiveHarness pipeline in app/ai/pipelines/"
+
+      def create_pipeline
+        template "pipeline.rb.tt",
+                 "app/ai/pipelines/#{file_name}_pipeline.rb"
+      end
+    end
+  end
+end

data/lib/generators/active_harness/pipeline/templates/pipeline.rb.tt

@@ -0,0 +1,19 @@
+class <%= class_name %>Pipeline < ActiveHarness::Pipeline
+  step :respond, <%= class_name %>Agent
+
+  before :step do |step_name, _payload|
+    puts "[pipeline] → :#{step_name}"
+  end
+
+  after :step do |step_name, _result|
+    puts "[pipeline] ✓ :#{step_name}"
+  end
+
+  callback :stopped do |step_name, _result|
+    puts "[pipeline] ✗ STOPPED at :#{step_name}"
+  end
+
+  callback :complete do |_last_result|
+    puts "[pipeline] ✓ complete"
+  end
+end

data/lib/generators/active_harness/prompt/prompt_generator.rb

@@ -0,0 +1,16 @@
+require "rails/generators"
+
+module ActiveHarness
+  module Generators
+    class PromptGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates an ActiveHarness prompt in app/ai/prompts/"
+
+      def create_prompt
+        template "prompt.rb.tt",
+                 "app/ai/prompts/#{file_name}_prompt.rb"
+      end
+    end
+  end
+end

data/lib/generators/active_harness/prompt/templates/prompt.rb.tt

@@ -0,0 +1,5 @@
+class <%= class_name %>Prompt
+  def call
+    "You are a concise and helpful assistant. Answer in 1-2 sentences."
+  end
+end

data/lib/generators/active_harness/tribunal/templates/tribunal.rb.tt

@@ -0,0 +1,7 @@
+class <%= class_name %>Tribunal < ActiveHarness::Tribunal
+  agents <%= class_name %>Agent
+
+  process do |results|
+    results.all? { |r| r.parsed["result"] == true }
+  end
+end