active_harness 0.2.33 → 0.2.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/agent/image.rb +56 -0
- data/lib/active_harness/agent/models.rb +3 -1
- data/lib/active_harness/agent/providers.rb +17 -0
- data/lib/active_harness/agent.rb +14 -5
- data/lib/active_harness/pricing/models_dev.rb +218 -0
- data/lib/active_harness/pricing/openrouter.rb +323 -0
- data/lib/active_harness/pricing.rb +72 -214
- data/lib/active_harness/providers/base.rb +5 -1
- data/lib/active_harness/providers/images/openai.rb +73 -0
- data/lib/active_harness/providers/images/openrouter.rb +66 -0
- data/lib/active_harness/railtie.rb +4 -0
- data/lib/active_harness.rb +5 -1
- metadata +7 -3
- data/lib/active_harness/data/models.json +0 -61458
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f08d4dd9d2254cb4895919e690fd485cab05d2b89c53786383b0da353aeb5f82
|
|
4
|
+
data.tar.gz: 1aa9a3e3a7cd8a2e83179b88a9f3623f39b4fa59e7386d43094de2739f44d50a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1aa38c722c75fbc04d6d389ebe6d260322eddd1a4d6c5bb9a43117b2b341447bf1883f90beacdcf5c2bc99adc6867a2dc7ee820f3dcd85d44538c5d148953a92
|
|
7
|
+
data.tar.gz: 146bf3d9516d70dcc45c2b092855b2b6b01543dd3b76e45de250b2b91fecce700cbabf7f492ebbf2e7417dfca22a48c21b2247a361b83eb28a81b001a3ce5999
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
module ActiveHarness
|
|
2
|
+
class Agent
|
|
3
|
+
class << self
|
|
4
|
+
# Mark this agent as an image generation agent.
|
|
5
|
+
#
|
|
6
|
+
# class ImageAgent < ActiveHarness::Agent
|
|
7
|
+
# image true
|
|
8
|
+
# size "1024x1024"
|
|
9
|
+
#
|
|
10
|
+
# model do
|
|
11
|
+
# use provider: :openrouter, model: "openai/gpt-5-image-mini"
|
|
12
|
+
# fallback provider: :openai, model: "gpt-image-1"
|
|
13
|
+
# end
|
|
14
|
+
# end
|
|
15
|
+
#
|
|
16
|
+
# result.output — base64 string or data-URI (provider-dependent)
|
|
17
|
+
# result.processed — same as output (format :text default)
|
|
18
|
+
def image(value = true)
|
|
19
|
+
agent_config[:image] = value
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Default image size for all models in this agent's chain.
|
|
23
|
+
# Can be overridden per-model via: use provider: :openai, model: "...", size: "1024x1792"
|
|
24
|
+
def size(default_size)
|
|
25
|
+
agent_config[:image_size] = default_size
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
alias_method :_model_list_base, :model_list
|
|
30
|
+
|
|
31
|
+
def model_list
|
|
32
|
+
list = _model_list_base
|
|
33
|
+
validate_image_models!(list) if @config[:image]
|
|
34
|
+
list
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# Models absent from the Pricing registry are silently skipped — unknown
|
|
40
|
+
# models are assumed valid to avoid false negatives on new/private models.
|
|
41
|
+
def validate_image_models!(list)
|
|
42
|
+
list.each do |entry|
|
|
43
|
+
info = Pricing.find(entry[:model].to_s)
|
|
44
|
+
next unless info
|
|
45
|
+
|
|
46
|
+
unless info.categories.include?("imggen")
|
|
47
|
+
raise ArgumentError,
|
|
48
|
+
"#{self.class.name}: model #{entry[:model].inspect} (provider: #{entry[:provider]}) " \
|
|
49
|
+
"does not support image generation " \
|
|
50
|
+
"(output_modalities: #{info.output_modalities.inspect}). " \
|
|
51
|
+
"Use a model that has 'image' in output_modalities."
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -134,12 +134,14 @@ module ActiveHarness
|
|
|
134
134
|
@models = []
|
|
135
135
|
end
|
|
136
136
|
|
|
137
|
-
def use(provider:, model:, temperature: nil, name: nil, retry_attempts: nil, retry_delay: nil)
|
|
137
|
+
def use(provider:, model:, temperature: nil, name: nil, size: nil, quality: nil, retry_attempts: nil, retry_delay: nil)
|
|
138
138
|
@models << {
|
|
139
139
|
provider: provider,
|
|
140
140
|
model: model,
|
|
141
141
|
temperature: temperature,
|
|
142
142
|
name: name,
|
|
143
|
+
size: size,
|
|
144
|
+
quality: quality,
|
|
143
145
|
retry_attempts: retry_attempts,
|
|
144
146
|
retry_delay: retry_delay
|
|
145
147
|
}.compact
|
|
@@ -38,10 +38,16 @@ module ActiveHarness
|
|
|
38
38
|
custom: -> { Providers::Custom.new }
|
|
39
39
|
}.freeze
|
|
40
40
|
|
|
41
|
+
IMAGE_PROVIDERS = {
|
|
42
|
+
openai: -> { Providers::Images::OpenAI.new },
|
|
43
|
+
openrouter: -> { Providers::Images::OpenRouter.new }
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
41
46
|
private
|
|
42
47
|
|
|
43
48
|
def attempt_model(entry, system_prompt)
|
|
44
49
|
return attempt_via_custom_llm(entry, system_prompt) if @config[:custom_llm_backend]
|
|
50
|
+
return attempt_image_model(entry) if @config[:image]
|
|
45
51
|
|
|
46
52
|
provider = resolve_provider(entry[:provider])
|
|
47
53
|
messages = build_messages(system_prompt, @input)
|
|
@@ -52,6 +58,17 @@ module ActiveHarness
|
|
|
52
58
|
provider.call(**opts)
|
|
53
59
|
end
|
|
54
60
|
|
|
61
|
+
def attempt_image_model(entry)
|
|
62
|
+
factory = IMAGE_PROVIDERS[entry[:provider].to_sym]
|
|
63
|
+
raise ArgumentError, "Provider #{entry[:provider].inspect} does not support image generation. " \
|
|
64
|
+
"Supported image providers: #{IMAGE_PROVIDERS.keys.join(', ')}" unless factory
|
|
65
|
+
|
|
66
|
+
size = entry[:size] || @config[:image_size] || "1024x1024"
|
|
67
|
+
opts = { model: entry[:model], prompt: @input.to_s, size: size }
|
|
68
|
+
opts[:quality] = entry[:quality] if entry[:quality]
|
|
69
|
+
factory.call.call(**opts)
|
|
70
|
+
end
|
|
71
|
+
|
|
55
72
|
def resolve_provider(name)
|
|
56
73
|
factory = PROVIDERS[name.to_sym]
|
|
57
74
|
raise ArgumentError, "Unknown provider: #{name.inspect}. Supported: #{PROVIDERS.keys.join(', ')}" unless factory
|
data/lib/active_harness/agent.rb
CHANGED
|
@@ -188,15 +188,23 @@ module ActiveHarness
|
|
|
188
188
|
total: raw_usage[:total_tokens]
|
|
189
189
|
)
|
|
190
190
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
191
|
+
cost = if raw_usage.key?(:provider_cost)
|
|
192
|
+
CostBreakdown.new(total: raw_usage[:provider_cost].round(8))
|
|
193
|
+
else
|
|
194
|
+
calculate_cost(model_cost, tokens)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
UsageInfo.new(tokens: tokens, cost: cost)
|
|
195
198
|
end
|
|
196
199
|
|
|
197
200
|
def lookup_model_cost(entry)
|
|
198
201
|
return nil unless entry
|
|
199
|
-
|
|
202
|
+
|
|
203
|
+
if entry[:provider].to_sym == :openrouter
|
|
204
|
+
Pricing::OpenRouter.find(entry[:model].to_s) || Pricing.find(entry[:model].to_s)
|
|
205
|
+
else
|
|
206
|
+
Pricing.find(entry[:model].to_s)
|
|
207
|
+
end
|
|
200
208
|
rescue StandardError
|
|
201
209
|
nil
|
|
202
210
|
end
|
|
@@ -215,4 +223,5 @@ require_relative "agent/providers"
|
|
|
215
223
|
require_relative "agent/output_parser"
|
|
216
224
|
require_relative "agent/custom_llm_backend"
|
|
217
225
|
require_relative "agent/cost"
|
|
226
|
+
require_relative "agent/image"
|
|
218
227
|
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "net/http"
|
|
3
|
+
require "uri"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "set"
|
|
6
|
+
|
|
7
|
+
module ActiveHarness
|
|
8
|
+
module Pricing
|
|
9
|
+
# Fallback pricing source — fetches model data from models.dev.
|
|
10
|
+
#
|
|
11
|
+
# Data source:
|
|
12
|
+
# {project_root}/tmp/active_harness/pricing_models_dev.json — fetched cache (24h TTL)
|
|
13
|
+
# Returns nil/empty if cache is missing and network is unavailable.
|
|
14
|
+
#
|
|
15
|
+
# Usage:
|
|
16
|
+
# Pricing::ModelsDev.find("gpt-4o")
|
|
17
|
+
# Pricing::ModelsDev.all
|
|
18
|
+
# Pricing::ModelsDev.update
|
|
19
|
+
module ModelsDev
|
|
20
|
+
MODELS_DEV_URL = "https://models.dev/api.json"
|
|
21
|
+
MEMORY_TTL = 3 * 86_400 # 3 days
|
|
22
|
+
|
|
23
|
+
MODELS_DEV_PROVIDER_MAP = {
|
|
24
|
+
"openai" => "openai",
|
|
25
|
+
"anthropic" => "anthropic",
|
|
26
|
+
"google" => "gemini",
|
|
27
|
+
"google-vertex" => "vertexai",
|
|
28
|
+
"amazon-bedrock" => "bedrock",
|
|
29
|
+
"deepseek" => "deepseek",
|
|
30
|
+
"mistral" => "mistral",
|
|
31
|
+
"openrouter" => "openrouter",
|
|
32
|
+
"perplexity" => "perplexity",
|
|
33
|
+
"xai" => "xai",
|
|
34
|
+
"groq" => "groq",
|
|
35
|
+
"azure" => "azure"
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
class << self
|
|
39
|
+
def all
|
|
40
|
+
ensure_fresh_registry
|
|
41
|
+
registry.map { |raw| build_cost(raw) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def find(model_id)
|
|
45
|
+
ensure_fresh_registry
|
|
46
|
+
raw = registry.find { |m| m[:id] == model_id.to_s }
|
|
47
|
+
raw ? build_cost(raw) : nil
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def providers
|
|
51
|
+
@providers_proxy ||= Pricing::ProvidersProxy.new(self)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def for_provider(name)
|
|
55
|
+
ensure_fresh_registry
|
|
56
|
+
registry
|
|
57
|
+
.select { |m| m[:provider] == name.to_s }
|
|
58
|
+
.map { |m| build_cost(m) }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def provider_names
|
|
62
|
+
@provider_names ||= begin
|
|
63
|
+
ensure_fresh_registry
|
|
64
|
+
registry.map { |m| m[:provider] }.uniq.sort
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Fetches fresh data from models.dev, writes to cache file, loads into memory.
|
|
69
|
+
# Called automatically when memory is stale. Can also be called explicitly.
|
|
70
|
+
def preload!
|
|
71
|
+
update
|
|
72
|
+
rescue StandardError
|
|
73
|
+
nil
|
|
74
|
+
ensure
|
|
75
|
+
@registry = load_registry
|
|
76
|
+
@loaded_at = @registry.empty? ? nil : Time.now
|
|
77
|
+
@provider_names = nil
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def update
|
|
81
|
+
raw_api = fetch_models_dev
|
|
82
|
+
models = extract_models(raw_api)
|
|
83
|
+
|
|
84
|
+
FileUtils.mkdir_p(File.dirname(cache_file))
|
|
85
|
+
File.write(cache_file, JSON.generate(models))
|
|
86
|
+
models.size
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def reload!
|
|
90
|
+
@registry = nil
|
|
91
|
+
@loaded_at = nil
|
|
92
|
+
@provider_names = nil
|
|
93
|
+
nil
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def cache_file
|
|
97
|
+
File.join(project_root, "tmp", "active_harness", "models_dev_pricing.json")
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def available_providers
|
|
101
|
+
@available_providers ||= begin
|
|
102
|
+
providers_dir = File.expand_path("../providers", __dir__)
|
|
103
|
+
Dir.glob("#{providers_dir}/*.rb")
|
|
104
|
+
.map { |f| File.basename(f, ".rb") }
|
|
105
|
+
.reject { |n| %w[base custom].include?(n) }
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
private
|
|
110
|
+
|
|
111
|
+
def ensure_fresh_registry
|
|
112
|
+
return if memory_fresh?
|
|
113
|
+
|
|
114
|
+
unless file_fresh?
|
|
115
|
+
begin
|
|
116
|
+
update
|
|
117
|
+
rescue StandardError
|
|
118
|
+
nil
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
@registry = load_registry
|
|
123
|
+
@loaded_at = @registry.empty? ? nil : Time.now
|
|
124
|
+
@provider_names = nil
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def memory_fresh?
|
|
128
|
+
@loaded_at && (Time.now - @loaded_at) < MEMORY_TTL
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def file_fresh?
|
|
132
|
+
File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < MEMORY_TTL
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def registry
|
|
136
|
+
@registry ||= []
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def load_registry
|
|
140
|
+
return [] unless File.exist?(cache_file)
|
|
141
|
+
data = JSON.parse(File.read(cache_file), symbolize_names: true)
|
|
142
|
+
data.is_a?(Array) ? data : []
|
|
143
|
+
rescue JSON::ParserError
|
|
144
|
+
[]
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def fetch_models_dev
|
|
148
|
+
uri = URI(MODELS_DEV_URL)
|
|
149
|
+
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
|
150
|
+
http.get(uri.request_uri)
|
|
151
|
+
end
|
|
152
|
+
raise "models.dev returned HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
|
|
153
|
+
|
|
154
|
+
JSON.parse(response.body, symbolize_names: true)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def extract_models(raw_api)
|
|
158
|
+
allowed = available_providers.to_set
|
|
159
|
+
|
|
160
|
+
raw_api.flat_map do |provider_key, provider_data|
|
|
161
|
+
ah_provider = MODELS_DEV_PROVIDER_MAP[provider_key.to_s]
|
|
162
|
+
next [] unless ah_provider && allowed.include?(ah_provider)
|
|
163
|
+
|
|
164
|
+
models_hash = provider_data.is_a?(Hash) ? (provider_data[:models] || {}) : {}
|
|
165
|
+
models_hash.values.filter_map do |m|
|
|
166
|
+
next unless m.is_a?(Hash) && m[:id]
|
|
167
|
+
|
|
168
|
+
cost = m[:cost] || {}
|
|
169
|
+
standard = {
|
|
170
|
+
input_per_million: cost[:input],
|
|
171
|
+
output_per_million: cost[:output],
|
|
172
|
+
cache_read_input_per_million: cost[:cache_read],
|
|
173
|
+
cache_write_input_per_million: cost[:cache_write]
|
|
174
|
+
}.compact
|
|
175
|
+
|
|
176
|
+
mods = m[:modalities] || {}
|
|
177
|
+
{
|
|
178
|
+
id: m[:id],
|
|
179
|
+
name: m[:name] || m[:id],
|
|
180
|
+
provider: ah_provider,
|
|
181
|
+
context_window: m[:context_window] || m.dig(:limit, :context),
|
|
182
|
+
max_output_tokens: m[:max_output_tokens] || m.dig(:limit, :output),
|
|
183
|
+
input_modalities: Array(mods[:input]),
|
|
184
|
+
output_modalities: Array(mods[:output]),
|
|
185
|
+
pricing: standard.any? ? { text_tokens: { standard: standard } } : {}
|
|
186
|
+
}
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def build_cost(raw)
|
|
192
|
+
standard = raw.dig(:pricing, :text_tokens, :standard) || {}
|
|
193
|
+
Pricing::ModelPrice.new(
|
|
194
|
+
id: raw[:id],
|
|
195
|
+
name: raw[:name],
|
|
196
|
+
provider: raw[:provider],
|
|
197
|
+
input_per_million: standard[:input_per_million],
|
|
198
|
+
output_per_million: standard[:output_per_million],
|
|
199
|
+
cache_read_input_per_million: standard[:cache_read_input_per_million],
|
|
200
|
+
cache_write_input_per_million: standard[:cache_write_input_per_million],
|
|
201
|
+
context_window: raw[:context_window],
|
|
202
|
+
max_output_tokens: raw[:max_output_tokens],
|
|
203
|
+
input_modalities: Array(raw[:input_modalities]),
|
|
204
|
+
output_modalities: Array(raw[:output_modalities])
|
|
205
|
+
)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def project_root
|
|
209
|
+
if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
|
|
210
|
+
Rails.root.to_s
|
|
211
|
+
else
|
|
212
|
+
Dir.pwd
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "net/http"
|
|
3
|
+
require "uri"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
|
|
6
|
+
module ActiveHarness
|
|
7
|
+
module Pricing
|
|
8
|
+
# Fetches complete pricing for all OpenRouter models across all modalities.
|
|
9
|
+
#
|
|
10
|
+
# OpenRouter exposes models via several endpoints:
|
|
11
|
+
# GET /api/v1/models → 337 text models (base)
|
|
12
|
+
# GET /api/v1/models?output_modalities=image → 32 image-gen models (25 extra)
|
|
13
|
+
# GET /api/v1/models?output_modalities=embeddings → 26 models (all extra)
|
|
14
|
+
# GET /api/v1/models?output_modalities=speech → 9 models (all extra)
|
|
15
|
+
# GET /api/v1/models?output_modalities=transcription → 10 models (all extra)
|
|
16
|
+
# GET /api/v1/models?output_modalities=video → 14 models (all zero pricing)
|
|
17
|
+
# GET /api/v1/models?output_modalities=rerank → 4 models (all zero pricing)
|
|
18
|
+
#
|
|
19
|
+
# For image-output models, /api/v1/models/{id}/endpoints is also fetched
|
|
20
|
+
# to get the accurate `image_output` per-token rate.
|
|
21
|
+
#
|
|
22
|
+
# All models are merged by id; pricing fields are populated per-modality:
|
|
23
|
+
# text_input / text_output — text tokens
|
|
24
|
+
# image_input — image tokens accepted as input (vision)
|
|
25
|
+
# image_output — image generation tokens (from /endpoints)
|
|
26
|
+
# audio_input — audio tokens as input
|
|
27
|
+
# audio_output — audio tokens as output (TTS)
|
|
28
|
+
# cache_read / cache_write — cache tokens
|
|
29
|
+
# web_search — per web-search request
|
|
30
|
+
#
|
|
31
|
+
# Usage:
|
|
32
|
+
# Pricing::OpenRouter.find("openai/gpt-5-image-mini") # → ModelPrice or nil
|
|
33
|
+
# Pricing::OpenRouter.all # → Array<ModelPrice>
|
|
34
|
+
# Pricing::OpenRouter.update # force refresh
|
|
35
|
+
module OpenRouter
|
|
36
|
+
API_BASE = "https://openrouter.ai/api/v1/models"
|
|
37
|
+
MEMORY_TTL = 3 * 86_400 # 3 days
|
|
38
|
+
|
|
39
|
+
# Modalities that have models outside the base text-337 set.
|
|
40
|
+
EXTRA_MODALITIES = %w[image embeddings speech transcription video rerank].freeze
|
|
41
|
+
|
|
42
|
+
class << self
|
|
43
|
+
def find(model_id)
|
|
44
|
+
ensure_fresh_registry
|
|
45
|
+
raw = registry.find { |m| m[:id] == model_id.to_s }
|
|
46
|
+
raw ? build_price(raw) : nil
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def all
|
|
50
|
+
ensure_fresh_registry
|
|
51
|
+
registry.filter_map { |raw| build_price(raw) }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def preload!
|
|
55
|
+
update
|
|
56
|
+
rescue StandardError
|
|
57
|
+
nil
|
|
58
|
+
ensure
|
|
59
|
+
@registry = load_registry
|
|
60
|
+
@loaded_at = @registry.empty? ? nil : Time.now
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def update
|
|
64
|
+
entries = collect_all_models
|
|
65
|
+
FileUtils.mkdir_p(File.dirname(cache_file))
|
|
66
|
+
File.write(cache_file, JSON.generate(entries))
|
|
67
|
+
entries.size
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def reload!
|
|
71
|
+
@registry = nil
|
|
72
|
+
@loaded_at = nil
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def cache_file
|
|
76
|
+
File.join(project_root, "tmp", "active_harness", "openrouter_pricing.json")
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# ── Freshness ────────────────────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
def ensure_fresh_registry
|
|
84
|
+
return if memory_fresh?
|
|
85
|
+
unless file_fresh?
|
|
86
|
+
begin
|
|
87
|
+
update
|
|
88
|
+
rescue StandardError
|
|
89
|
+
nil
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
@registry = load_registry
|
|
93
|
+
@loaded_at = @registry.empty? ? nil : Time.now
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def memory_fresh?
|
|
97
|
+
@loaded_at && (Time.now - @loaded_at) < MEMORY_TTL
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def file_fresh?
|
|
101
|
+
File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < MEMORY_TTL
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def registry
|
|
105
|
+
@registry ||= []
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def load_registry
|
|
109
|
+
return [] unless File.exist?(cache_file)
|
|
110
|
+
data = JSON.parse(File.read(cache_file), symbolize_names: true)
|
|
111
|
+
data.is_a?(Array) ? data : []
|
|
112
|
+
rescue JSON::ParserError
|
|
113
|
+
[]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# ── Data collection ──────────────────────────────────────────────
|
|
117
|
+
|
|
118
|
+
# Fetches all modality endpoints, merges by id, enriches image models.
|
|
119
|
+
def collect_all_models
|
|
120
|
+
models = {}
|
|
121
|
+
|
|
122
|
+
# Base text models
|
|
123
|
+
fetch_models(API_BASE).each do |m|
|
|
124
|
+
models[m[:id]] = normalize(m)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Specialized modalities — add extra models and merge pricing
|
|
128
|
+
EXTRA_MODALITIES.each do |mod|
|
|
129
|
+
fetch_models("#{API_BASE}?output_modalities=#{mod}").each do |m|
|
|
130
|
+
id = m[:id]
|
|
131
|
+
if models[id]
|
|
132
|
+
merge_pricing!(models[id], m)
|
|
133
|
+
else
|
|
134
|
+
models[id] = normalize(m)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Enrich image-output models with /endpoints for accurate image_output rate
|
|
140
|
+
models.values.map do |entry|
|
|
141
|
+
if Array(entry[:output_modalities]).include?("image")
|
|
142
|
+
enrich_with_endpoint(entry)
|
|
143
|
+
else
|
|
144
|
+
entry
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Normalize a raw API model hash into our cache entry format.
|
|
150
|
+
def normalize(m)
|
|
151
|
+
p = m[:pricing] || {}
|
|
152
|
+
{
|
|
153
|
+
id: m[:id],
|
|
154
|
+
name: m[:name],
|
|
155
|
+
input_modalities: m.dig(:architecture, :input_modalities) || [],
|
|
156
|
+
output_modalities: m.dig(:architecture, :output_modalities) || [],
|
|
157
|
+
text_input: p[:prompt].to_s,
|
|
158
|
+
text_output: p[:completion].to_s,
|
|
159
|
+
image_input: p[:image].to_s,
|
|
160
|
+
audio_input: p[:audio].to_s,
|
|
161
|
+
image_output: "",
|
|
162
|
+
audio_output: "",
|
|
163
|
+
cache_read: p[:input_cache_read].to_s,
|
|
164
|
+
cache_write: p[:input_cache_write].to_s,
|
|
165
|
+
web_search: p[:web_search].to_s
|
|
166
|
+
}
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Merge non-zero pricing fields from a new API response into existing entry.
|
|
170
|
+
def merge_pricing!(entry, raw_model)
|
|
171
|
+
p = raw_model[:pricing] || {}
|
|
172
|
+
[
|
|
173
|
+
[:text_input, p[:prompt]],
|
|
174
|
+
[:text_output, p[:completion]],
|
|
175
|
+
[:image_input, p[:image]],
|
|
176
|
+
[:audio_input, p[:audio]],
|
|
177
|
+
[:cache_read, p[:input_cache_read]],
|
|
178
|
+
[:cache_write, p[:input_cache_write]],
|
|
179
|
+
[:web_search, p[:web_search]]
|
|
180
|
+
].each do |key, val|
|
|
181
|
+
entry[key] = val.to_s if val.to_f > 0 && entry[key].to_f == 0
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Merge modalities (union)
|
|
185
|
+
new_out = raw_model.dig(:architecture, :output_modalities) || []
|
|
186
|
+
entry[:output_modalities] = (Array(entry[:output_modalities]) | new_out).uniq
|
|
187
|
+
new_in = raw_model.dig(:architecture, :input_modalities) || []
|
|
188
|
+
entry[:input_modalities] = (Array(entry[:input_modalities]) | new_in).uniq
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Fetch /endpoints and add image_output rate to the entry.
|
|
192
|
+
def enrich_with_endpoint(entry)
|
|
193
|
+
pricing = fetch_endpoint_pricing(entry[:id])
|
|
194
|
+
entry[:image_output] = pricing&.dig(:image_output).to_s
|
|
195
|
+
entry[:audio_output] = pricing&.dig(:audio_output).to_s
|
|
196
|
+
entry
|
|
197
|
+
rescue StandardError
|
|
198
|
+
entry
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def fetch_endpoint_pricing(model_id)
|
|
202
|
+
uri = URI("#{API_BASE}/#{model_id}/endpoints")
|
|
203
|
+
resp = http_get(uri)
|
|
204
|
+
data = JSON.parse(resp.body, symbolize_names: true)
|
|
205
|
+
endpoints = data.dig(:data, :endpoints) || []
|
|
206
|
+
ep = endpoints.find { |e| e[:status] == 0 } || endpoints.first
|
|
207
|
+
ep&.dig(:pricing)
|
|
208
|
+
rescue StandardError
|
|
209
|
+
nil
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def fetch_models(url)
|
|
213
|
+
resp = http_get(URI(url))
|
|
214
|
+
data = JSON.parse(resp.body, symbolize_names: true)
|
|
215
|
+
data[:data] || []
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# ── Build ModelPrice ─────────────────────────────────────────────
|
|
219
|
+
|
|
220
|
+
def build_price(raw)
|
|
221
|
+
out_mods = Array(raw[:output_modalities])
|
|
222
|
+
inp_mods = Array(raw[:input_modalities])
|
|
223
|
+
|
|
224
|
+
is_imggen = out_mods.include?("image")
|
|
225
|
+
is_embed = out_mods.include?("embeddings")
|
|
226
|
+
is_speech = out_mods.include?("speech")
|
|
227
|
+
is_transcription = out_mods.include?("transcription")
|
|
228
|
+
|
|
229
|
+
text_in_pm = to_pm(raw[:text_input])
|
|
230
|
+
text_out_pm = to_pm(raw[:text_output])
|
|
231
|
+
img_in_pm = to_pm(raw[:image_input])
|
|
232
|
+
img_out_pm = to_pm(raw[:image_output])
|
|
233
|
+
# p[:audio] field — audio input tokens (multimodal/embedding models like Gemini)
|
|
234
|
+
audio_in_pm = to_pm(raw[:audio_input])
|
|
235
|
+
aud_out_pm = to_pm(raw[:audio_output])
|
|
236
|
+
cache_r_pm = to_pm(raw[:cache_read])
|
|
237
|
+
cache_w_pm = to_pm(raw[:cache_write])
|
|
238
|
+
# web_search is a flat per-request fee in USD, not a per-token rate
|
|
239
|
+
ws_raw = raw[:web_search].to_s
|
|
240
|
+
web_search_usd = ws_raw.empty? ? nil : (ws_raw.to_f > 0 ? ws_raw.to_f : nil)
|
|
241
|
+
|
|
242
|
+
# Transcription pricing is stored in `prompt` but the unit differs by model:
|
|
243
|
+
# prompt < 0.0001 → per-audio-token (e.g. gpt-4o-transcribe $2.5/M) → use to_pm
|
|
244
|
+
# prompt >= 0.0001 → per-minute of audio (e.g. Whisper $0.006/min) → raw USD
|
|
245
|
+
if is_transcription
|
|
246
|
+
raw_rate = raw[:text_input].to_s.to_f
|
|
247
|
+
audio_in_pm = if raw_rate > 0 && raw_rate < 0.0001
|
|
248
|
+
to_pm(raw[:text_input]) # per-token → convert to per-million
|
|
249
|
+
elsif raw_rate > 0
|
|
250
|
+
raw_rate # per-minute → keep raw USD value
|
|
251
|
+
end
|
|
252
|
+
text_in_pm = nil
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Primary output for cost calculation and sorting:
|
|
256
|
+
# imggen → image_output rate (from /endpoints)
|
|
257
|
+
# speech → audio_output rate (completion is audio)
|
|
258
|
+
# embed / transcription → no output cost
|
|
259
|
+
# text → text_output rate
|
|
260
|
+
primary_output = if is_imggen
|
|
261
|
+
img_out_pm || text_out_pm
|
|
262
|
+
elsif is_speech
|
|
263
|
+
aud_out_pm || text_out_pm
|
|
264
|
+
elsif is_embed || is_transcription
|
|
265
|
+
nil
|
|
266
|
+
else
|
|
267
|
+
text_out_pm
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Primary input for cost calculation and sorting
|
|
271
|
+
primary_input = is_transcription ? audio_in_pm : text_in_pm
|
|
272
|
+
|
|
273
|
+
# Skip models with no id/name; keep zero-priced models (rerank, video) —
|
|
274
|
+
# they are real models, just have $0 rates in the OpenRouter API.
|
|
275
|
+
return nil unless raw[:id] && raw[:name]
|
|
276
|
+
|
|
277
|
+
Pricing::ModelPrice.new(
|
|
278
|
+
id: raw[:id],
|
|
279
|
+
name: raw[:name],
|
|
280
|
+
provider: "openrouter",
|
|
281
|
+
input_per_million: primary_input,
|
|
282
|
+
output_per_million: primary_output,
|
|
283
|
+
cache_read_input_per_million: cache_r_pm,
|
|
284
|
+
cache_write_input_per_million: cache_w_pm,
|
|
285
|
+
context_window: nil,
|
|
286
|
+
max_output_tokens: nil,
|
|
287
|
+
input_modalities: inp_mods,
|
|
288
|
+
output_modalities: out_mods,
|
|
289
|
+
image_input_per_million: img_in_pm,
|
|
290
|
+
image_output_per_million: img_out_pm,
|
|
291
|
+
audio_input_per_million: audio_in_pm,
|
|
292
|
+
audio_output_per_million: aud_out_pm,
|
|
293
|
+
web_search_per_request: web_search_usd
|
|
294
|
+
)
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# Per-token string → per-million float. Returns nil for zero/blank.
|
|
298
|
+
def to_pm(value)
|
|
299
|
+
return nil if value.nil? || value.to_s.strip.empty?
|
|
300
|
+
f = value.to_f
|
|
301
|
+
return nil if f <= 0
|
|
302
|
+
(f * 1_000_000).round(6)
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def http_get(uri)
|
|
306
|
+
resp = Net::HTTP.start(uri.host, uri.port, use_ssl: true, read_timeout: 15) do |h|
|
|
307
|
+
h.get(uri.request_uri)
|
|
308
|
+
end
|
|
309
|
+
raise "OpenRouter API #{resp.code} for #{uri}" unless resp.is_a?(Net::HTTPSuccess)
|
|
310
|
+
resp
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def project_root
|
|
314
|
+
if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
|
|
315
|
+
Rails.root.to_s
|
|
316
|
+
else
|
|
317
|
+
Dir.pwd
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
end
|