active_harness 0.2.12 → 0.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4f069d1d894324475b0a5395d5728fa77626698255bc3400428ef66a2228ac4
-  data.tar.gz: 1cbfee779b26e122d0e01e3fa1e4a80907398eec5f0895b28bc1dcb19484d8a4
+  metadata.gz: 6a42e03d329e638e3119d0a259a2286f05a0ec1603b8c2c81636e4ac041ebf9d
+  data.tar.gz: 074cf7e1558d9efee20cd17c3405665be00ef29c30574289b54d6e2251ebeacc
 SHA512:
-  metadata.gz: c2293baad25b739468f7fb3306583f2e475409edc7b1533a03e83da9e28776ba17a27ae6a9aebf47fb18d4a1953957240d2615a277443e39781876a685d7da1e
-  data.tar.gz: 6e78901ee75204d853b53df08baef9215f7f225f15065da771262ee084af17243a796ab132526701b016e4d0066762338e01fd01fd36f5e316e1045599cc8c3b
+  metadata.gz: 3397bd1901b3062030b3a41dc327ed1fd7053ef5d90e8e53d0ebb820375054f112cc80e70e528282703277e90137310318d026c456950f86ae755a09aca4e473
+  data.tar.gz: 7137703c7469a5f1cd8adfb121fd5047231387890756325f7e035955c7a23dab3bd0dfaf22c223343a9fb006279b6b352c01f1cf443da6cfcf60c3e26227cac3

data/lib/active_harness/agent/cost.rb

@@ -0,0 +1,28 @@
+module ActiveHarness
+  class Agent
+    private
+
+    # Calculates the monetary cost of a single request based on token usage
+    # and pricing data from ActiveHarness::Costs.
+    #
+    # Returns a hash { input_cost:, output_cost:, total_cost: } in USD,
+    # or nil if usage is absent or the model is not found in the pricing registry.
+    def calculate_cost(model_id, usage)
+      return nil if model_id.nil? || usage.nil?
+
+      pricing = ActiveHarness::Costs.find(model_id.to_s)
+      return nil unless pricing&.input_per_million && pricing&.output_per_million
+
+      input_cost  = (usage[:input_tokens].to_f  / 1_000_000) * pricing.input_per_million
+      output_cost = (usage[:output_tokens].to_f / 1_000_000) * pricing.output_per_million
+
+      {
+        input_cost:  input_cost.round(8),
+        output_cost: output_cost.round(8),
+        total_cost:  (input_cost + output_cost).round(8)
+      }
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/active_harness/agent.rb

@@ -113,6 +113,7 @@ module ActiveHarness
     def build_result(response, entry, attempts, elapsed)
       raw    = response[:content]
       parsed = parse_output(raw)
+      usage  = response[:usage]
 
       Result.new(
         input:          @input,
@@ -125,7 +126,8 @@ module ActiveHarness
         model_list:     model_list,
         attempts:       attempts,
         execution_time: elapsed,
-        usage:          response[:usage]
+        usage:          usage,
+        cost:           calculate_cost(entry[:model], usage)
       )
     end
 
@@ -152,4 +154,5 @@ require_relative "agent/models"
 require_relative "agent/providers"
 require_relative "agent/output_parser"
 require_relative "agent/ruby_llm_backend"
+require_relative "agent/cost"
 

data/lib/active_harness/costs.rb

@@ -183,10 +183,21 @@ module ActiveHarness
       end
 
       def registry
-        @registry ||= begin
-          source = File.exist?(cache_file) ? cache_file : BUNDLED_DATA_FILE
-          JSON.parse(File.read(source), symbolize_names: true)
+        @registry ||= load_registry
+      end
+
+      def load_registry
+        if File.exist?(cache_file)
+          begin
+            data = JSON.parse(File.read(cache_file), symbolize_names: true)
+            return data if data.is_a?(Array)
+          rescue JSON::ParserError
+            # Cache file corrupted — fall through to bundled data
+          end
         end
+        JSON.parse(File.read(BUNDLED_DATA_FILE), symbolize_names: true)
+      rescue JSON::ParserError, Errno::ENOENT
+        []
       end
 
       def fetch_models_dev

data/lib/active_harness/result.rb

@@ -6,5 +6,6 @@ module ActiveHarness
   # output — raw string from the provider
   # parsed — for format :json: a Ruby Hash/Array; for format :text: same as output
   # usage  — token counts: { input_tokens:, output_tokens:, total_tokens: } or nil for streaming
-  Result = Struct.new(:input, :output, :parsed, :system_prompt, :provider, :model, :temperature, :model_list, :attempts, :execution_time, :usage, keyword_init: true)
+  # cost   — { input_cost:, output_cost:, total_cost: } in USD, or nil if pricing unavailable
+  Result = Struct.new(:input, :output, :parsed, :system_prompt, :provider, :model, :temperature, :model_list, :attempts, :execution_time, :usage, :cost, keyword_init: true)
 end

data/lib/active_harness/tribunal/dsl.rb

@@ -0,0 +1,21 @@
+module ActiveHarness
+  class Tribunal
+    class << self
+      # Declare agents at the class level.
+      #
+      #   agents PolitenessAgent, ConstructivenessAgent
+      #   agents [PolitenessAgent, ConstructivenessAgent]
+      def agents(*list)
+        tribunal_config[:agents] = list.flatten
+      end
+
+      # Class-level process block — defines how to compute the verdict from results.
+      #
+      #   process { |results| results.all? { |r| r.parsed["result"] == true } }
+      #   process { |results| results.count { |r| r.parsed["result"] == true } >= 2 }
+      def process(&block)
+        tribunal_config[:process] = block
+      end
+    end
+  end
+end

data/lib/active_harness/tribunal/hooks.rb

@@ -0,0 +1,87 @@
+module ActiveHarness
+  class Tribunal
+    VALID_HOOKS = %i[
+      before_call
+      before_agent
+      after_agent
+      agent_error
+      after_call
+      before_verdict
+      after_verdict
+    ].freeze
+
+    class << self
+      # Class-level hook registration.
+      #
+      #   on :before_call                  do ... end
+      #   on :before_agent                 do |agent| ... end
+      #   on :after_agent                  do |result| ... end
+      #   on :agent_error                  do |name, error| ... end
+      #   on :after_call                   do |results, errors| ... end
+      #   on :before_verdict               do |results| results end  # transform hook
+      #   on :after_verdict                do |verdict| ... end
+      def on(event, &block)
+        unless VALID_HOOKS.include?(event)
+          raise ArgumentError,
+            "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.map { |h| ":#{h}" }.join(", ")}"
+        end
+
+        tribunal_config[:hooks][event] = block
+      end
+
+      # Rails-style aliases for +on+:
+      #
+      #   before :call                     do ... end       # → on :before_call
+      #   before :agent                    do |agent| end   # → on :before_agent
+      #   before :verdict                  do |results| end # → on :before_verdict (transform)
+      #   after  :call                     do |r, e| end    # → on :after_call
+      #   after  :agent                    do |result| end  # → on :after_agent
+      #   after  :verdict                  do |verdict| end # → on :after_verdict
+      #   callback :agent_error            do |name, e| end # → on :agent_error
+      def before(event, &block)
+        on(:"before_#{event}", &block)
+      end
+
+      def after(event, &block)
+        on(:"after_#{event}", &block)
+      end
+
+      def callback(event, &block)
+        on(event, &block)
+      end
+    end
+
+    # Instance-level hook registration — overrides class-level hooks for this instance.
+    # :before_verdict is a transform hook: its return value replaces the results array
+    # passed to the process block.
+    def on(event, &block)
+      unless VALID_HOOKS.include?(event)
+        raise ArgumentError,
+          "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.map { |h| ":#{h}" }.join(", ")}"
+      end
+
+      @hooks[event] = block
+      self
+    end
+
+    private
+
+    def run_hook(event, *args)
+      return unless @hooks[event]
+
+      if args.any?
+        instance_exec(*args, &@hooks[event])
+      else
+        instance_eval(&@hooks[event])
+      end
+    end
+
+    # Like run_hook but uses the return value to replace the passed value.
+    # Used by :before_verdict to allow results transformation before verdict computation.
+    def transform_hook(event, value)
+      return value unless @hooks[event]
+
+      instance_exec(value, &@hooks[event])
+    end
+  end
+end

data/lib/active_harness/tribunal.rb

@@ -29,62 +29,11 @@ module ActiveHarness
   #   ContentQualityTribunal.new(input: "...").call
   #
   class Tribunal
-    VALID_HOOKS = %i[
-      before_call
-      after_agent
-      agent_error
-      after_call
-      before_verdict
-      after_verdict
-    ].freeze
-
     # -------------------------------------------------------------------------
-    # Class-level DSL — used when subclassing ActiveHarness::Tribunal
+    # Class-level DSL — core
     # -------------------------------------------------------------------------
     class << self
-      # Declare agents at the class level.
-      #   agents PolitenessAgent, ConstructivenessAgent
-      def agents(*list)
-        tribunal_config[:agents] = list.flatten
-      end
-
-      # Class-level hook registration.
-      #   on(:after_agent) { |result| puts result.model }
-      def on(event, &block)
-        unless VALID_HOOKS.include?(event)
-          raise ArgumentError, "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.join(", ")}"
-        end
-
-        tribunal_config[:hooks][event] = block
-      end
-
-      # Rails-style aliases for +on+:
-      #
-      #   before :call                     do ... end   # → on :before_call
-      #   before :agent                    do ... end   # → on :before_agent (not used yet)
-      #   before :verdict                  do |r| end   # → on :before_verdict
-      #   after  :call                     do ... end   # → on :after_call
-      #   after  :agent                    do |r| end   # → on :after_agent
-      #   after  :verdict                  do |v| end   # → on :after_verdict
-      #   callback :agent_error            do |n,e| end # → on :agent_error
-      def before(event, &block)
-        on(:"before_#{event}", &block)
-      end
-
-      def after(event, &block)
-        on(:"after_#{event}", &block)
-      end
-
-      def callback(event, &block)
-        on(event, &block)
-      end
-
-      # Class-level process block.
-      #   process { |results| results.all? { |r| r.parsed["result"] == true } }
-      def process(&block)
-        tribunal_config[:process] = block
-      end
-
+      # Each subclass gets its own isolated config hash.
       def tribunal_config
         @tribunal_config ||= { agents: [], hooks: {} }
       end
@@ -94,36 +43,32 @@ module ActiveHarness
       end
     end
 
-    attr_accessor :input, :context
+    # -------------------------------------------------------------------------
+    # Instance API
+    # -------------------------------------------------------------------------
+    attr_accessor :input, :context, :stream, :agent_event_stream, :tribunal_event_stream
     attr_reader :results, :errors, :verdict, :execution_time, :agent_execution_times
 
-    def initialize(input: nil, context: {}, agents: nil, timeout: 7)
+    def initialize(input: nil, context: {}, agents: nil, timeout: 7,
+                   stream: nil, agent_event_stream: nil, tribunal_event_stream: nil)
       config = self.class.tribunal_config
 
-      @input         = input
-      @context       = context
-      @agents        = agents || config[:agents]
-      @timeout       = timeout
-      @process_block = config[:process]
-      @hooks         = config[:hooks].dup
-      @results       = []
-      @errors        = []
-      @verdict       = nil
-      @execution_time       = nil
+      @input                 = input
+      @context               = context
+      @agents                = agents || config[:agents]
+      @timeout               = timeout
+      @process_block         = config[:process]
+      @hooks                 = config[:hooks].dup
+      @stream                = stream
+      @agent_event_stream    = agent_event_stream
+      @tribunal_event_stream = tribunal_event_stream
+      @results               = []
+      @errors                = []
+      @verdict               = nil
+      @execution_time        = nil
       @agent_execution_times = []
     end
 
-    # Instance-level hook registration — overrides class-level hooks.
-    # :before_verdict is a transform hook: its return value replaces the results array.
-    def on(event, &block)
-      unless VALID_HOOKS.include?(event)
-        raise ArgumentError, "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.join(", ")}"
-      end
-
-      @hooks[event] = block
-      self
-    end
-
     # Instance-level process block — overrides class-level block.
     def process(&block)
       @process_block = block
@@ -143,14 +88,15 @@ module ActiveHarness
 
       started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
-      futures = agents.map do |agent|
+      futures = agents.each_with_index.map do |agent, index|
+        run_hook(:before_agent, agent, index)
         t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         future = Concurrent::Future.execute { agent.call }
         [future, t0]
       end
 
-      @results              = []
-      @errors               = []
+      @results               = []
+      @errors                = []
       @agent_execution_times = []
 
       futures.each_with_index do |(future, t0), index|
@@ -159,19 +105,19 @@ module ActiveHarness
         @agent_execution_times << { agent: agents[index].class.name, time: elapsed }
 
         if future.fulfilled?
-          value = future.value
+          value  = future.value
           result = value.is_a?(ActiveHarness::Agent) ? value.result : value
           @results << result
-          run_hook(:after_agent, result)
+          run_hook(:after_agent, result, index)
         elsif future.incomplete?
           error = Errors::TimeoutError.new(
             "Agent #{agents[index].class.name} timed out after #{@timeout}s"
           )
           @errors << { agent: agents[index].class.name, error: error }
-          run_hook(:agent_error, agents[index].class.name, error)
+          run_hook(:agent_error, agents[index].class.name, error, index)
         else
           @errors << { agent: agents[index].class.name, error: future.reason }
-          run_hook(:agent_error, agents[index].class.name, future.reason)
+          run_hook(:agent_error, agents[index].class.name, future.reason, index)
         end
       end
 
@@ -193,26 +139,25 @@ module ActiveHarness
 
     private
 
-    def run_hook(event, *args)
-      @hooks[event]&.call(*args)
-    end
-
-    # Like run_hook but uses the return value to replace the passed value.
-    def transform_hook(event, value)
-      return value unless @hooks[event]
-
-      @hooks[event].call(value)
-    end
-
     def resolve_agents
       @agents.map do |agent|
         if agent.is_a?(Class)
-          agent.new(input: @input, context: @context.dup)
+          agent.new(
+            input:        @input,
+            context:      @context.dup,
+            stream:       @stream,
+            event_stream: @agent_event_stream
+          )
         else
-          agent.input = @input if @input
+          agent.input        = @input if @input
+          agent.stream       = @stream if @stream
+          agent.event_stream = @agent_event_stream if @agent_event_stream
           agent
         end
       end
     end
   end
 end
+
+require_relative "tribunal/hooks"
+require_relative "tribunal/dsl"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_harness
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.14
 platform: ruby
 authors:
 - the-teacher
@@ -33,6 +33,7 @@ extra_rdoc_files: []
 files:
 - lib/active_harness.rb
 - lib/active_harness/agent.rb
+- lib/active_harness/agent/cost.rb
 - lib/active_harness/agent/hooks.rb
 - lib/active_harness/agent/models.rb
 - lib/active_harness/agent/output_parser.rb
@@ -71,6 +72,8 @@ files:
 - lib/active_harness/railtie.rb
 - lib/active_harness/result.rb
 - lib/active_harness/tribunal.rb
+- lib/active_harness/tribunal/dsl.rb
+- lib/active_harness/tribunal/hooks.rb
 - lib/generators/active_harness/agent/agent_generator.rb
 - lib/generators/active_harness/agent/templates/agent.rb.tt
 - lib/generators/active_harness/install/install_generator.rb