active_harness 0.2.35 → 0.2.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f08d4dd9d2254cb4895919e690fd485cab05d2b89c53786383b0da353aeb5f82
4
- data.tar.gz: 1aa9a3e3a7cd8a2e83179b88a9f3623f39b4fa59e7386d43094de2739f44d50a
3
+ metadata.gz: 5fb1f238ea442c93d556e18ef1b899ad8d8e8345c6ed18c8abd0d45370c847e0
4
+ data.tar.gz: 8c8b0dceb9c779f6432dc86f141fdda65144b365511ee3096957a96954447b4e
5
5
  SHA512:
6
- metadata.gz: 1aa38c722c75fbc04d6d389ebe6d260322eddd1a4d6c5bb9a43117b2b341447bf1883f90beacdcf5c2bc99adc6867a2dc7ee820f3dcd85d44538c5d148953a92
7
- data.tar.gz: 146bf3d9516d70dcc45c2b092855b2b6b01543dd3b76e45de250b2b91fecce700cbabf7f492ebbf2e7417dfca22a48c21b2247a361b83eb28a81b001a3ce5999
6
+ metadata.gz: fd1da6cb093ea99b414576fe9e1fc2ec0fee4a894eca5bd5291582ea5daabdb1ee621f22a5b7e4460455c03fd47df5b152b5d9598c64df9a408cbaec93ca3371
7
+ data.tar.gz: 939c5d39b1f172ae0225087f095e8665c84b34f6cc384bf2e8f949e871a0be63c3ecf59253e253a1066c40c6c41528f57c8b09790720dca765c8760175ce36ad
@@ -54,8 +54,8 @@ module ActiveHarness
54
54
  chat = backend.call(params)
55
55
  chat.with_instructions(system_prompt) if system_prompt
56
56
 
57
- if @token_stream
58
- response = chat.ask(@input) { |chunk| @token_stream.call(chunk.content) if chunk.content }
57
+ if @token
58
+ response = chat.ask(@input) { |chunk| @token.call(chunk.content) if chunk.content }
59
59
  else
60
60
  response = chat.ask(@input)
61
61
  end
@@ -67,11 +67,9 @@ module ActiveHarness
67
67
  run_hooks(@config[:hooks] || {}, event, *args)
68
68
  end
69
69
 
70
- # Unified internal method: fires the DSL hook AND the external event_stream lambda.
71
- # Consistent with Tribunal#fire and Pipeline#fire.
72
70
  def fire(event, *args)
73
71
  run_hook(event, *args)
74
- @event_stream&.call(event, *args)
72
+ @stream&.call(:agent, event, *args)
75
73
  rescue IOError, ActionController::Live::ClientDisconnected
76
74
  end
77
75
  end
@@ -53,7 +53,7 @@ module ActiveHarness
53
53
  messages = build_messages(system_prompt, @input)
54
54
  opts = { model: entry[:model], messages: messages }
55
55
  opts[:temperature] = entry[:temperature] if entry[:temperature]
56
- opts[:stream] = @token_stream if @token_stream
56
+ opts[:stream] = @token if @token
57
57
  opts[:name] = entry[:name] if entry[:name]
58
58
  provider.call(**opts)
59
59
  end
@@ -16,7 +16,8 @@ module ActiveHarness
16
16
  params: {},
17
17
  memory: nil,
18
18
  models: nil,
19
- streams: {}
19
+ token: nil,
20
+ stream: nil
20
21
  )
21
22
  new(
22
23
  input: input,
@@ -24,7 +25,8 @@ module ActiveHarness
24
25
  params: params,
25
26
  memory: memory,
26
27
  models: models,
27
- streams: streams
28
+ token: token,
29
+ stream: stream
28
30
  ).call
29
31
  end
30
32
 
@@ -54,8 +56,8 @@ module ActiveHarness
54
56
  :params,
55
57
  :memory
56
58
  attr_reader :result,
57
- :token_stream,
58
- :event_stream
59
+ :token,
60
+ :stream
59
61
 
60
62
  def models=(list)
61
63
  @models_override = Array(list)
@@ -68,7 +70,8 @@ module ActiveHarness
68
70
  params: {},
69
71
  memory: nil,
70
72
  models: nil,
71
- streams: {}
73
+ token: nil,
74
+ stream: nil
72
75
  )
73
76
  @input = input
74
77
  @config = self.class.agent_config
@@ -77,8 +80,8 @@ module ActiveHarness
77
80
  @params = params
78
81
  @memory = memory
79
82
  @models_override = Array(models) if models
80
- @token_stream = streams[:token]
81
- @event_stream = streams[:agent]
83
+ @token = token
84
+ @stream = stream
82
85
  fire(:setup)
83
86
  end
84
87
 
@@ -88,15 +91,13 @@ module ActiveHarness
88
91
  # Optionally accepts input and stream callback inline:
89
92
  # agent.call("What is the capital of Japan?")
90
93
  # agent.call("...", stream: ->(token) { print token })
91
- def call(input = nil, streams: nil)
94
+ def call(input = nil, token: nil, stream: nil)
92
95
  if input
93
96
  @input = input
94
97
  normalize_input!
95
98
  end
96
- if streams
97
- @token_stream = streams[:token] if streams.key?(:token)
98
- @event_stream = streams[:agent] if streams.key?(:agent)
99
- end
99
+ @token = token if token
100
+ @stream = stream if stream
100
101
  fire(:before_call)
101
102
  @system_prompt = resolve_system_prompt
102
103
  attempts = []
@@ -60,15 +60,14 @@ module ActiveHarness
60
60
 
61
61
  private
62
62
 
63
- # Fires global hook AND pipeline_event_stream. Consistent with Agent#fire and Tribunal#fire.
64
63
  def fire(event, step_name, data, config)
65
64
  run_hooks(config[:hooks], event, step_name, data)
66
- @pipeline_event_stream&.call(event, step_name, data)
65
+ @stream&.call(:pipeline, event, step_name, data)
67
66
  rescue IOError, ActionController::Live::ClientDisconnected
68
67
  nil
69
68
  end
70
69
 
71
- # Per-step hook: receives (data) only — not forwarded to pipeline_event_stream
70
+ # Per-step hook: receives (data) only — not forwarded to stream
72
71
  # (global fire already covers the step event with step_name context).
73
72
  def fire_step(event, step_name, data, config)
74
73
  run_hooks(config[:step_hooks][step_name] || {}, event, data)
@@ -66,8 +66,7 @@ module ActiveHarness
66
66
  # any agent or tribunal executed within this pipeline (including agents
67
67
  # running inside tribunals). Multiple blocks can be registered; all fire.
68
68
  #
69
- # The handler receives the same (event, *args) signature that the runtime
70
- # streams: { agent: lambda } would receive.
69
+ # The handler receives (event, *args) already scoped to the source.
71
70
  #
72
71
  # on_agent_event do |event, result|
73
72
  # Rails.logger.info "[Agent #{event}] #{result.model}" if event == :after_call
@@ -116,24 +115,23 @@ module ActiveHarness
116
115
  context: {},
117
116
  params: {},
118
117
  memory: nil,
119
- streams: {}
118
+ token: nil,
119
+ stream: nil
120
120
  )
121
- @original_input = input
122
- @payload = input
123
- @context = context.dup
124
- @params = params
125
- @memory = memory
126
- @token_stream = streams[:token]
127
- class_streams = self.class.pipeline_config[:streams] || {}
128
- @agent_event_stream = merge_stream(streams[:agent], class_streams[:agent])
129
- @tribunal_event_stream = merge_stream(streams[:tribunal], class_streams[:tribunal])
130
- @pipeline_event_stream = merge_stream(streams[:pipeline], class_streams[:pipeline])
131
- @step_results = {}
132
- @stopped = false
133
- @stopped_at = nil
134
- @stop_reason = nil
135
- @execution_time = nil
136
- @output = nil
121
+ @original_input = input
122
+ @payload = input
123
+ @context = context.dup
124
+ @params = params
125
+ @memory = memory
126
+ @token = token
127
+ class_streams = self.class.pipeline_config[:streams] || {}
128
+ @stream = merge_stream(stream, class_streams)
129
+ @step_results = {}
130
+ @stopped = false
131
+ @stopped_at = nil
132
+ @stop_reason = nil
133
+ @execution_time = nil
134
+ @output = nil
137
135
  end
138
136
 
139
137
  def stopped?
@@ -179,7 +177,7 @@ module ActiveHarness
179
177
  @stopped_at = step.name
180
178
  @stop_reason = result
181
179
  run_hooks(config[:hooks], :stopped, step.name, result)
182
- @pipeline_event_stream&.call(:stopped, step.name, result)
180
+ @stream&.call(:pipeline, :stopped, step.name, result)
183
181
  break
184
182
  end
185
183
  end
@@ -196,7 +194,7 @@ module ActiveHarness
196
194
 
197
195
  last_result = @step_results[@step_results.keys.last]
198
196
  run_hooks(config[:hooks], :complete, last_result)
199
- @pipeline_event_stream&.call(:complete, last_result)
197
+ @stream&.call(:pipeline, :complete, last_result)
200
198
  end
201
199
 
202
200
  self
@@ -204,37 +202,41 @@ module ActiveHarness
204
202
 
205
203
  private
206
204
 
207
- # Combines a runtime-passed stream lambda with zero or more class-level handler
208
- # blocks registered via on_agent_event / on_tribunal_event / on_pipeline_event.
209
- # Returns nil when there are no handlers at all, preserving the existing
210
- # "no stream" fast path in agents and tribunals.
205
+ # Combines a runtime-passed stream lambda with class-level handler blocks
206
+ # registered via on_agent_event / on_tribunal_event / on_pipeline_event.
207
+ # Returns nil when there are no handlers at all.
211
208
  #
212
- # Each class-level handler is evaluated via instance_exec so that blocks
213
- # written in the pipeline class body can access pipeline instance variables
214
- # (e.g. @otel_pipeline_span, @params) and call pipeline instance methods.
209
+ # Class-level handlers receive (event, *args) already scoped to source.
210
+ # Runtime lambda receives (source, event, *args).
211
+ # instance_exec lets class-level blocks access pipeline instance variables.
215
212
  def merge_stream(passed_in, class_handlers)
216
- class_handlers = Array(class_handlers).compact
217
- return passed_in if class_handlers.empty?
213
+ agent_handlers = Array(class_handlers[:agent]).compact
214
+ tribunal_handlers = Array(class_handlers[:tribunal]).compact
215
+ pipeline_handlers = Array(class_handlers[:pipeline]).compact
216
+
217
+ has_class_handlers = agent_handlers.any? || tribunal_handlers.any? || pipeline_handlers.any?
218
+ return passed_in unless has_class_handlers
218
219
 
219
220
  pipeline_instance = self
220
- ->(event, *args) {
221
- class_handlers.each { |h| pipeline_instance.instance_exec(event, *args, &h) }
222
- passed_in&.call(event, *args)
221
+ ->(source, event, *args) {
222
+ handlers = case source
223
+ when :agent then agent_handlers
224
+ when :tribunal then tribunal_handlers
225
+ when :pipeline then pipeline_handlers
226
+ else []
227
+ end
228
+ handlers.each { |h| pipeline_instance.instance_exec(event, *args, &h) }
229
+ passed_in&.call(source, event, *args)
223
230
  }
224
231
  end
225
232
 
226
233
  def execute_step(step)
227
- streams = {
228
- token: @token_stream,
229
- agent: @agent_event_stream,
230
- tribunal: @tribunal_event_stream,
231
- pipeline: @pipeline_event_stream
232
- }.compact
233
234
  step.agent_class.new(
234
235
  input: @payload,
235
236
  context: @context.dup,
236
237
  params: @params,
237
- streams: streams
238
+ token: @token,
239
+ stream: @stream
238
240
  ).call.result
239
241
  end
240
242
  end
@@ -72,10 +72,9 @@ module ActiveHarness
72
72
  run_hooks(@hooks, event, *args)
73
73
  end
74
74
 
75
- # Fire the DSL-registered hook AND the external tribunal_event_stream lambda (if set).
76
75
  def fire(event, *args)
77
76
  run_hook(event, *args)
78
- @tribunal_event_stream&.call(event, *args)
77
+ @stream&.call(:tribunal, event, *args)
79
78
  rescue IOError, ActionController::Live::ClientDisconnected
80
79
  nil
81
80
  end
@@ -55,9 +55,8 @@ module ActiveHarness
55
55
  :verdict,
56
56
  :execution_time,
57
57
  :agent_execution_times,
58
- :token_stream,
59
- :agent_event_stream,
60
- :tribunal_event_stream
58
+ :token,
59
+ :stream
61
60
 
62
61
  def initialize(
63
62
  input: nil,
@@ -66,7 +65,8 @@ module ActiveHarness
66
65
  memory: nil,
67
66
  agents: nil,
68
67
  timeout: 7,
69
- streams: {},
68
+ token: nil,
69
+ stream: nil,
70
70
  may_fail: :_unset
71
71
  )
72
72
  config = self.class.tribunal_config
@@ -82,9 +82,8 @@ module ActiveHarness
82
82
  @evaluate_block = config[:evaluate_block]
83
83
  @may_fail = may_fail == :_unset ? config[:may_fail] : may_fail
84
84
  @hooks = config[:hooks].transform_values { |v| Array(v).dup }
85
- @token_stream = streams[:token]
86
- @agent_event_stream = streams[:agent]
87
- @tribunal_event_stream = streams[:tribunal]
85
+ @token = token
86
+ @stream = stream
88
87
  @results = []
89
88
  @errors = []
90
89
  @verdict = nil
@@ -181,14 +180,13 @@ module ActiveHarness
181
180
  end
182
181
 
183
182
  def resolve_agents
184
- agent_streams = { token: @token_stream, agent: @agent_event_stream }.compact
185
183
  @agents.map do |agent|
186
184
  if agent.is_a?(Class)
187
- agent.new(input: @input, context: @context.dup, params: @params, streams: agent_streams)
185
+ agent.new(input: @input, context: @context.dup, params: @params, token: @token, stream: @stream)
188
186
  else
189
187
  agent.input = @input if @input
190
- agent.instance_variable_set(:@token_stream, @token_stream) if @token_stream
191
- agent.instance_variable_set(:@event_stream, @agent_event_stream) if @agent_event_stream
188
+ agent.instance_variable_set(:@token, @token) if @token
189
+ agent.instance_variable_set(:@stream, @stream) if @stream
192
190
  agent
193
191
  end
194
192
  end
@@ -23,9 +23,7 @@ require_relative "active_harness/providers/vertexai"
23
23
  require_relative "active_harness/providers/custom"
24
24
  require_relative "active_harness/providers/images/openai"
25
25
  require_relative "active_harness/providers/images/openrouter"
26
- require_relative "active_harness/pricing"
27
- require_relative "active_harness/pricing/models_dev"
28
- require_relative "active_harness/pricing/openrouter"
26
+ require "active_harness_pricing"
29
27
  require_relative "active_harness/memory"
30
28
  require_relative "active_harness/agent"
31
29
  require_relative "active_harness/tribunal"
@@ -34,7 +32,7 @@ require_relative "active_harness/pipeline"
34
32
  require_relative "active_harness/railtie" if defined?(Rails::Railtie)
35
33
 
36
34
  module ActiveHarness
37
- VERSION = "0.2.35"
35
+ VERSION = "0.2.37"
38
36
 
39
37
  class << self
40
38
  # Configure ActiveHarness.
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: active_harness
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.35
4
+ version: 0.2.37
5
5
  platform: ruby
6
6
  authors:
7
7
  - the-teacher
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-06-13 00:00:00.000000000 Z
11
+ date: 2026-06-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -24,6 +24,20 @@ dependencies:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: active_harness_pricing
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
27
41
  description:
28
42
  email:
29
43
  - the-teacher@github.com
@@ -56,9 +70,6 @@ files:
56
70
  - lib/active_harness/pipeline/README.md
57
71
  - lib/active_harness/pipeline/hooks.rb
58
72
  - lib/active_harness/pipeline/step.rb
59
- - lib/active_harness/pricing.rb
60
- - lib/active_harness/pricing/models_dev.rb
61
- - lib/active_harness/pricing/openrouter.rb
62
73
  - lib/active_harness/providers/PROVIDER_CONTRACT.md
63
74
  - lib/active_harness/providers/anthropic.rb
64
75
  - lib/active_harness/providers/azure.rb
@@ -1,218 +0,0 @@
1
- require "json"
2
- require "net/http"
3
- require "uri"
4
- require "fileutils"
5
- require "set"
6
-
7
- module ActiveHarness
8
- module Pricing
9
- # Fallback pricing source — fetches model data from models.dev.
10
- #
11
- # Data source:
12
- # {project_root}/tmp/active_harness/pricing_models_dev.json — fetched cache (24h TTL)
13
- # Returns nil/empty if cache is missing and network is unavailable.
14
- #
15
- # Usage:
16
- # Pricing::ModelsDev.find("gpt-4o")
17
- # Pricing::ModelsDev.all
18
- # Pricing::ModelsDev.update
19
- module ModelsDev
20
- MODELS_DEV_URL = "https://models.dev/api.json"
21
- MEMORY_TTL = 3 * 86_400 # 3 days
22
-
23
- MODELS_DEV_PROVIDER_MAP = {
24
- "openai" => "openai",
25
- "anthropic" => "anthropic",
26
- "google" => "gemini",
27
- "google-vertex" => "vertexai",
28
- "amazon-bedrock" => "bedrock",
29
- "deepseek" => "deepseek",
30
- "mistral" => "mistral",
31
- "openrouter" => "openrouter",
32
- "perplexity" => "perplexity",
33
- "xai" => "xai",
34
- "groq" => "groq",
35
- "azure" => "azure"
36
- }.freeze
37
-
38
- class << self
39
- def all
40
- ensure_fresh_registry
41
- registry.map { |raw| build_cost(raw) }
42
- end
43
-
44
- def find(model_id)
45
- ensure_fresh_registry
46
- raw = registry.find { |m| m[:id] == model_id.to_s }
47
- raw ? build_cost(raw) : nil
48
- end
49
-
50
- def providers
51
- @providers_proxy ||= Pricing::ProvidersProxy.new(self)
52
- end
53
-
54
- def for_provider(name)
55
- ensure_fresh_registry
56
- registry
57
- .select { |m| m[:provider] == name.to_s }
58
- .map { |m| build_cost(m) }
59
- end
60
-
61
- def provider_names
62
- @provider_names ||= begin
63
- ensure_fresh_registry
64
- registry.map { |m| m[:provider] }.uniq.sort
65
- end
66
- end
67
-
68
- # Fetches fresh data from models.dev, writes to cache file, loads into memory.
69
- # Called automatically when memory is stale. Can also be called explicitly.
70
- def preload!
71
- update
72
- rescue StandardError
73
- nil
74
- ensure
75
- @registry = load_registry
76
- @loaded_at = @registry.empty? ? nil : Time.now
77
- @provider_names = nil
78
- end
79
-
80
- def update
81
- raw_api = fetch_models_dev
82
- models = extract_models(raw_api)
83
-
84
- FileUtils.mkdir_p(File.dirname(cache_file))
85
- File.write(cache_file, JSON.generate(models))
86
- models.size
87
- end
88
-
89
- def reload!
90
- @registry = nil
91
- @loaded_at = nil
92
- @provider_names = nil
93
- nil
94
- end
95
-
96
- def cache_file
97
- File.join(project_root, "tmp", "active_harness", "models_dev_pricing.json")
98
- end
99
-
100
- def available_providers
101
- @available_providers ||= begin
102
- providers_dir = File.expand_path("../providers", __dir__)
103
- Dir.glob("#{providers_dir}/*.rb")
104
- .map { |f| File.basename(f, ".rb") }
105
- .reject { |n| %w[base custom].include?(n) }
106
- end
107
- end
108
-
109
- private
110
-
111
- def ensure_fresh_registry
112
- return if memory_fresh?
113
-
114
- unless file_fresh?
115
- begin
116
- update
117
- rescue StandardError
118
- nil
119
- end
120
- end
121
-
122
- @registry = load_registry
123
- @loaded_at = @registry.empty? ? nil : Time.now
124
- @provider_names = nil
125
- end
126
-
127
- def memory_fresh?
128
- @loaded_at && (Time.now - @loaded_at) < MEMORY_TTL
129
- end
130
-
131
- def file_fresh?
132
- File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < MEMORY_TTL
133
- end
134
-
135
- def registry
136
- @registry ||= []
137
- end
138
-
139
- def load_registry
140
- return [] unless File.exist?(cache_file)
141
- data = JSON.parse(File.read(cache_file), symbolize_names: true)
142
- data.is_a?(Array) ? data : []
143
- rescue JSON::ParserError
144
- []
145
- end
146
-
147
- def fetch_models_dev
148
- uri = URI(MODELS_DEV_URL)
149
- response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
150
- http.get(uri.request_uri)
151
- end
152
- raise "models.dev returned HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
153
-
154
- JSON.parse(response.body, symbolize_names: true)
155
- end
156
-
157
- def extract_models(raw_api)
158
- allowed = available_providers.to_set
159
-
160
- raw_api.flat_map do |provider_key, provider_data|
161
- ah_provider = MODELS_DEV_PROVIDER_MAP[provider_key.to_s]
162
- next [] unless ah_provider && allowed.include?(ah_provider)
163
-
164
- models_hash = provider_data.is_a?(Hash) ? (provider_data[:models] || {}) : {}
165
- models_hash.values.filter_map do |m|
166
- next unless m.is_a?(Hash) && m[:id]
167
-
168
- cost = m[:cost] || {}
169
- standard = {
170
- input_per_million: cost[:input],
171
- output_per_million: cost[:output],
172
- cache_read_input_per_million: cost[:cache_read],
173
- cache_write_input_per_million: cost[:cache_write]
174
- }.compact
175
-
176
- mods = m[:modalities] || {}
177
- {
178
- id: m[:id],
179
- name: m[:name] || m[:id],
180
- provider: ah_provider,
181
- context_window: m[:context_window] || m.dig(:limit, :context),
182
- max_output_tokens: m[:max_output_tokens] || m.dig(:limit, :output),
183
- input_modalities: Array(mods[:input]),
184
- output_modalities: Array(mods[:output]),
185
- pricing: standard.any? ? { text_tokens: { standard: standard } } : {}
186
- }
187
- end
188
- end
189
- end
190
-
191
- def build_cost(raw)
192
- standard = raw.dig(:pricing, :text_tokens, :standard) || {}
193
- Pricing::ModelPrice.new(
194
- id: raw[:id],
195
- name: raw[:name],
196
- provider: raw[:provider],
197
- input_per_million: standard[:input_per_million],
198
- output_per_million: standard[:output_per_million],
199
- cache_read_input_per_million: standard[:cache_read_input_per_million],
200
- cache_write_input_per_million: standard[:cache_write_input_per_million],
201
- context_window: raw[:context_window],
202
- max_output_tokens: raw[:max_output_tokens],
203
- input_modalities: Array(raw[:input_modalities]),
204
- output_modalities: Array(raw[:output_modalities])
205
- )
206
- end
207
-
208
- def project_root
209
- if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
210
- Rails.root.to_s
211
- else
212
- Dir.pwd
213
- end
214
- end
215
- end
216
- end
217
- end
218
- end
@@ -1,323 +0,0 @@
1
- require "json"
2
- require "net/http"
3
- require "uri"
4
- require "fileutils"
5
-
6
- module ActiveHarness
7
- module Pricing
8
- # Fetches complete pricing for all OpenRouter models across all modalities.
9
- #
10
- # OpenRouter exposes models via several endpoints:
11
- # GET /api/v1/models → 337 text models (base)
12
- # GET /api/v1/models?output_modalities=image → 32 image-gen models (25 extra)
13
- # GET /api/v1/models?output_modalities=embeddings → 26 models (all extra)
14
- # GET /api/v1/models?output_modalities=speech → 9 models (all extra)
15
- # GET /api/v1/models?output_modalities=transcription → 10 models (all extra)
16
- # GET /api/v1/models?output_modalities=video → 14 models (all zero pricing)
17
- # GET /api/v1/models?output_modalities=rerank → 4 models (all zero pricing)
18
- #
19
- # For image-output models, /api/v1/models/{id}/endpoints is also fetched
20
- # to get the accurate `image_output` per-token rate.
21
- #
22
- # All models are merged by id; pricing fields are populated per-modality:
23
- # text_input / text_output — text tokens
24
- # image_input — image tokens accepted as input (vision)
25
- # image_output — image generation tokens (from /endpoints)
26
- # audio_input — audio tokens as input
27
- # audio_output — audio tokens as output (TTS)
28
- # cache_read / cache_write — cache tokens
29
- # web_search — per web-search request
30
- #
31
- # Usage:
32
- # Pricing::OpenRouter.find("openai/gpt-5-image-mini") # → ModelPrice or nil
33
- # Pricing::OpenRouter.all # → Array<ModelPrice>
34
- # Pricing::OpenRouter.update # force refresh
35
- module OpenRouter
36
- API_BASE = "https://openrouter.ai/api/v1/models"
37
- MEMORY_TTL = 3 * 86_400 # 3 days
38
-
39
- # Modalities that have models outside the base text-337 set.
40
- EXTRA_MODALITIES = %w[image embeddings speech transcription video rerank].freeze
41
-
42
- class << self
43
- def find(model_id)
44
- ensure_fresh_registry
45
- raw = registry.find { |m| m[:id] == model_id.to_s }
46
- raw ? build_price(raw) : nil
47
- end
48
-
49
- def all
50
- ensure_fresh_registry
51
- registry.filter_map { |raw| build_price(raw) }
52
- end
53
-
54
- def preload!
55
- update
56
- rescue StandardError
57
- nil
58
- ensure
59
- @registry = load_registry
60
- @loaded_at = @registry.empty? ? nil : Time.now
61
- end
62
-
63
- def update
64
- entries = collect_all_models
65
- FileUtils.mkdir_p(File.dirname(cache_file))
66
- File.write(cache_file, JSON.generate(entries))
67
- entries.size
68
- end
69
-
70
- def reload!
71
- @registry = nil
72
- @loaded_at = nil
73
- end
74
-
75
- def cache_file
76
- File.join(project_root, "tmp", "active_harness", "openrouter_pricing.json")
77
- end
78
-
79
- private
80
-
81
- # ── Freshness ────────────────────────────────────────────────────
82
-
83
- def ensure_fresh_registry
84
- return if memory_fresh?
85
- unless file_fresh?
86
- begin
87
- update
88
- rescue StandardError
89
- nil
90
- end
91
- end
92
- @registry = load_registry
93
- @loaded_at = @registry.empty? ? nil : Time.now
94
- end
95
-
96
- def memory_fresh?
97
- @loaded_at && (Time.now - @loaded_at) < MEMORY_TTL
98
- end
99
-
100
- def file_fresh?
101
- File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < MEMORY_TTL
102
- end
103
-
104
- def registry
105
- @registry ||= []
106
- end
107
-
108
- def load_registry
109
- return [] unless File.exist?(cache_file)
110
- data = JSON.parse(File.read(cache_file), symbolize_names: true)
111
- data.is_a?(Array) ? data : []
112
- rescue JSON::ParserError
113
- []
114
- end
115
-
116
- # ── Data collection ──────────────────────────────────────────────
117
-
118
- # Fetches all modality endpoints, merges by id, enriches image models.
119
- def collect_all_models
120
- models = {}
121
-
122
- # Base text models
123
- fetch_models(API_BASE).each do |m|
124
- models[m[:id]] = normalize(m)
125
- end
126
-
127
- # Specialized modalities — add extra models and merge pricing
128
- EXTRA_MODALITIES.each do |mod|
129
- fetch_models("#{API_BASE}?output_modalities=#{mod}").each do |m|
130
- id = m[:id]
131
- if models[id]
132
- merge_pricing!(models[id], m)
133
- else
134
- models[id] = normalize(m)
135
- end
136
- end
137
- end
138
-
139
- # Enrich image-output models with /endpoints for accurate image_output rate
140
- models.values.map do |entry|
141
- if Array(entry[:output_modalities]).include?("image")
142
- enrich_with_endpoint(entry)
143
- else
144
- entry
145
- end
146
- end
147
- end
148
-
149
- # Normalize a raw API model hash into our cache entry format.
150
- def normalize(m)
151
- p = m[:pricing] || {}
152
- {
153
- id: m[:id],
154
- name: m[:name],
155
- input_modalities: m.dig(:architecture, :input_modalities) || [],
156
- output_modalities: m.dig(:architecture, :output_modalities) || [],
157
- text_input: p[:prompt].to_s,
158
- text_output: p[:completion].to_s,
159
- image_input: p[:image].to_s,
160
- audio_input: p[:audio].to_s,
161
- image_output: "",
162
- audio_output: "",
163
- cache_read: p[:input_cache_read].to_s,
164
- cache_write: p[:input_cache_write].to_s,
165
- web_search: p[:web_search].to_s
166
- }
167
- end
168
-
169
- # Merge non-zero pricing fields from a new API response into existing entry.
170
- def merge_pricing!(entry, raw_model)
171
- p = raw_model[:pricing] || {}
172
- [
173
- [:text_input, p[:prompt]],
174
- [:text_output, p[:completion]],
175
- [:image_input, p[:image]],
176
- [:audio_input, p[:audio]],
177
- [:cache_read, p[:input_cache_read]],
178
- [:cache_write, p[:input_cache_write]],
179
- [:web_search, p[:web_search]]
180
- ].each do |key, val|
181
- entry[key] = val.to_s if val.to_f > 0 && entry[key].to_f == 0
182
- end
183
-
184
- # Merge modalities (union)
185
- new_out = raw_model.dig(:architecture, :output_modalities) || []
186
- entry[:output_modalities] = (Array(entry[:output_modalities]) | new_out).uniq
187
- new_in = raw_model.dig(:architecture, :input_modalities) || []
188
- entry[:input_modalities] = (Array(entry[:input_modalities]) | new_in).uniq
189
- end
190
-
191
- # Fetch /endpoints and add image_output rate to the entry.
192
- def enrich_with_endpoint(entry)
193
- pricing = fetch_endpoint_pricing(entry[:id])
194
- entry[:image_output] = pricing&.dig(:image_output).to_s
195
- entry[:audio_output] = pricing&.dig(:audio_output).to_s
196
- entry
197
- rescue StandardError
198
- entry
199
- end
200
-
201
- def fetch_endpoint_pricing(model_id)
202
- uri = URI("#{API_BASE}/#{model_id}/endpoints")
203
- resp = http_get(uri)
204
- data = JSON.parse(resp.body, symbolize_names: true)
205
- endpoints = data.dig(:data, :endpoints) || []
206
- ep = endpoints.find { |e| e[:status] == 0 } || endpoints.first
207
- ep&.dig(:pricing)
208
- rescue StandardError
209
- nil
210
- end
211
-
212
- def fetch_models(url)
213
- resp = http_get(URI(url))
214
- data = JSON.parse(resp.body, symbolize_names: true)
215
- data[:data] || []
216
- end
217
-
218
- # ── Build ModelPrice ─────────────────────────────────────────────
219
-
220
- def build_price(raw)
221
- out_mods = Array(raw[:output_modalities])
222
- inp_mods = Array(raw[:input_modalities])
223
-
224
- is_imggen = out_mods.include?("image")
225
- is_embed = out_mods.include?("embeddings")
226
- is_speech = out_mods.include?("speech")
227
- is_transcription = out_mods.include?("transcription")
228
-
229
- text_in_pm = to_pm(raw[:text_input])
230
- text_out_pm = to_pm(raw[:text_output])
231
- img_in_pm = to_pm(raw[:image_input])
232
- img_out_pm = to_pm(raw[:image_output])
233
- # p[:audio] field — audio input tokens (multimodal/embedding models like Gemini)
234
- audio_in_pm = to_pm(raw[:audio_input])
235
- aud_out_pm = to_pm(raw[:audio_output])
236
- cache_r_pm = to_pm(raw[:cache_read])
237
- cache_w_pm = to_pm(raw[:cache_write])
238
- # web_search is a flat per-request fee in USD, not a per-token rate
239
- ws_raw = raw[:web_search].to_s
240
- web_search_usd = ws_raw.empty? ? nil : (ws_raw.to_f > 0 ? ws_raw.to_f : nil)
241
-
242
- # Transcription pricing is stored in `prompt` but the unit differs by model:
243
- # prompt < 0.0001 → per-audio-token (e.g. gpt-4o-transcribe $2.5/M) → use to_pm
244
- # prompt >= 0.0001 → per-minute of audio (e.g. Whisper $0.006/min) → raw USD
245
- if is_transcription
246
- raw_rate = raw[:text_input].to_s.to_f
247
- audio_in_pm = if raw_rate > 0 && raw_rate < 0.0001
248
- to_pm(raw[:text_input]) # per-token → convert to per-million
249
- elsif raw_rate > 0
250
- raw_rate # per-minute → keep raw USD value
251
- end
252
- text_in_pm = nil
253
- end
254
-
255
- # Primary output for cost calculation and sorting:
256
- # imggen → image_output rate (from /endpoints)
257
- # speech → audio_output rate (completion is audio)
258
- # embed / transcription → no output cost
259
- # text → text_output rate
260
- primary_output = if is_imggen
261
- img_out_pm || text_out_pm
262
- elsif is_speech
263
- aud_out_pm || text_out_pm
264
- elsif is_embed || is_transcription
265
- nil
266
- else
267
- text_out_pm
268
- end
269
-
270
- # Primary input for cost calculation and sorting
271
- primary_input = is_transcription ? audio_in_pm : text_in_pm
272
-
273
- # Skip models with no id/name; keep zero-priced models (rerank, video) —
274
- # they are real models, just have $0 rates in the OpenRouter API.
275
- return nil unless raw[:id] && raw[:name]
276
-
277
- Pricing::ModelPrice.new(
278
- id: raw[:id],
279
- name: raw[:name],
280
- provider: "openrouter",
281
- input_per_million: primary_input,
282
- output_per_million: primary_output,
283
- cache_read_input_per_million: cache_r_pm,
284
- cache_write_input_per_million: cache_w_pm,
285
- context_window: nil,
286
- max_output_tokens: nil,
287
- input_modalities: inp_mods,
288
- output_modalities: out_mods,
289
- image_input_per_million: img_in_pm,
290
- image_output_per_million: img_out_pm,
291
- audio_input_per_million: audio_in_pm,
292
- audio_output_per_million: aud_out_pm,
293
- web_search_per_request: web_search_usd
294
- )
295
- end
296
-
297
- # Per-token string → per-million float. Returns nil for zero/blank.
298
- def to_pm(value)
299
- return nil if value.nil? || value.to_s.strip.empty?
300
- f = value.to_f
301
- return nil if f <= 0
302
- (f * 1_000_000).round(6)
303
- end
304
-
305
- def http_get(uri)
306
- resp = Net::HTTP.start(uri.host, uri.port, use_ssl: true, read_timeout: 15) do |h|
307
- h.get(uri.request_uri)
308
- end
309
- raise "OpenRouter API #{resp.code} for #{uri}" unless resp.is_a?(Net::HTTPSuccess)
310
- resp
311
- end
312
-
313
- def project_root
314
- if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
315
- Rails.root.to_s
316
- else
317
- Dir.pwd
318
- end
319
- end
320
- end
321
- end
322
- end
323
- end
@@ -1,152 +0,0 @@
1
- require "json"
2
-
3
- module ActiveHarness
4
- # Pricing namespace — shared types and a facade over pricing source modules.
5
- #
6
- # Sources (in priority order):
7
- # Pricing::OpenRouter — live data from OpenRouter API (image models, 24h cache)
8
- # Pricing::ModelsDev — live data from models.dev API (all providers, 24h cache)
9
- #
10
- # Public facade delegates to ModelsDev (used as the general fallback):
11
- # Pricing.find("gpt-4o") → ModelPrice or nil
12
- # Pricing.all → Array<ModelPrice>
13
- # Pricing.providers.openai → Array<ModelPrice>
14
- # Pricing.update → refreshes ModelsDev cache
15
- module Pricing
16
- # Pricing rates for a single model.
17
- # All *_per_million fields are in USD per 1M tokens.
18
- # audio_input_per_million / audio_output_per_million may represent
19
- # per-million audio tokens or per-unit (second/char) depending on provider.
20
- ModelPrice = Struct.new(
21
- :id,
22
- :name,
23
- :provider,
24
- # Primary fields (used for cost calculation, backward-compatible)
25
- :input_per_million, # text tokens input
26
- :output_per_million, # primary output (text or image_output for imggen)
27
- :cache_read_input_per_million,
28
- :cache_write_input_per_million,
29
- :context_window,
30
- :max_output_tokens,
31
- :input_modalities,
32
- :output_modalities,
33
- # Extended modality-specific pricing
34
- :image_input_per_million, # image tokens accepted as input (vision models)
35
- :image_output_per_million, # image generation output tokens (imggen models)
36
- :audio_input_per_million, # audio tokens accepted as input
37
- :audio_output_per_million, # audio output tokens (TTS models)
38
- :web_search_per_request, # per web-search call in USD
39
- keyword_init: true
40
- ) do
41
- # Capability tags derived from modality data.
42
- # Possible values: "vision", "pdf", "audio", "video", "imggen", "embed",
43
- # "speech", "transcription", "rerank"
44
- def categories
45
- inp = Array(input_modalities)
46
- out = Array(output_modalities)
47
- cats = []
48
- cats << "vision" if inp.include?("image")
49
- cats << "pdf" if inp.include?("pdf")
50
- cats << "audio" if inp.include?("audio")
51
- cats << "video" if inp.include?("video") || out.include?("video")
52
- cats << "imggen" if out.include?("image")
53
- cats << "speech" if out.include?("speech")
54
- cats << "transcription" if out.include?("transcription")
55
- cats << "rerank" if out.include?("rerank")
56
- cats << "embed" if out.include?("embeddings")
57
- cats
58
- end
59
-
60
- def inspect
61
- parts = ["id=#{id.inspect}", "provider=#{provider.inspect}"]
62
- parts << "input=$#{input_per_million}/M" if input_per_million
63
- parts << "output=$#{output_per_million}/M" if output_per_million
64
- parts << "ctx=#{context_window}" if context_window
65
- parts << "cats=#{categories.join(',')}" if categories.any?
66
- "#<ModelPrice #{parts.join(' ')}>"
67
- end
68
- end
69
-
70
- # Proxy returned by Pricing.providers — exposes providers as methods and [].
71
- class ProvidersProxy
72
- def initialize(source = nil)
73
- @source = source
74
- end
75
-
76
- def [](name)
77
- source.for_provider(name.to_s)
78
- end
79
-
80
- def list
81
- source.provider_names
82
- end
83
-
84
- def method_missing(name, *args, &block)
85
- provider = name.to_s
86
- if source.provider_names.include?(provider)
87
- source.for_provider(provider)
88
- else
89
- super
90
- end
91
- end
92
-
93
- def respond_to_missing?(name, include_private = false)
94
- source.provider_names.include?(name.to_s) || super
95
- end
96
-
97
- private
98
-
99
- def source
100
- @source || ModelsDev
101
- end
102
- end
103
-
104
- # ---------------------------------------------------------------------------
105
- # Facade — delegates to ModelsDev (general fallback source)
106
- # ---------------------------------------------------------------------------
107
- class << self
108
- # Eagerly fetch all pricing sources and load them into memory.
109
- # Called at Rails startup. Network failures are silently ignored.
110
- def preload!
111
- ModelsDev.preload!
112
- OpenRouter.preload!
113
- end
114
-
115
- def find(model_id)
116
- ModelsDev.find(model_id)
117
- end
118
-
119
- def all
120
- ModelsDev.all
121
- end
122
-
123
- def providers
124
- ModelsDev.providers
125
- end
126
-
127
- def for_provider(name)
128
- ModelsDev.for_provider(name)
129
- end
130
-
131
- def provider_names
132
- ModelsDev.provider_names
133
- end
134
-
135
- def update
136
- ModelsDev.update
137
- end
138
-
139
- def reload!
140
- ModelsDev.reload!
141
- end
142
-
143
- def cache_file
144
- ModelsDev.cache_file
145
- end
146
-
147
- def available_providers
148
- ModelsDev.available_providers
149
- end
150
- end
151
- end
152
- end