active_harness 0.2.34 → 0.2.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 39b817ef529f0158a20634ba42a61ae0078404238038b22d63c2c8cc50a1ff3a
-  data.tar.gz: bcf8d21cb68449e3962f2bb9b515c39237c7774a6ca300a803775ae09c275567
+  metadata.gz: f08d4dd9d2254cb4895919e690fd485cab05d2b89c53786383b0da353aeb5f82
+  data.tar.gz: 1aa9a3e3a7cd8a2e83179b88a9f3623f39b4fa59e7386d43094de2739f44d50a
 SHA512:
-  metadata.gz: e09428cecc83c7fe3caed3c45c4caf000ee07a5b783acbfe3b591de02615af98498e7b490f0b6e5d7a34b5dcbdf12a4afd252a197c4fa71e6a1eabfd466d4b92
-  data.tar.gz: e80805c251e3a48ce0ab032a9c8904dae3401c6d02bedb9d1372df32ada7b9609880096c2fb39c62d575aecdace7266d068f6e0ed165ece989038b29e1b74885
+  metadata.gz: 1aa38c722c75fbc04d6d389ebe6d260322eddd1a4d6c5bb9a43117b2b341447bf1883f90beacdcf5c2bc99adc6867a2dc7ee820f3dcd85d44538c5d148953a92
+  data.tar.gz: 146bf3d9516d70dcc45c2b092855b2b6b01543dd3b76e45de250b2b91fecce700cbabf7f492ebbf2e7417dfca22a48c21b2247a361b83eb28a81b001a3ce5999

data/lib/active_harness/pricing/models_dev.rb

@@ -18,7 +18,7 @@ module ActiveHarness
     #   Pricing::ModelsDev.update
     module ModelsDev
       MODELS_DEV_URL = "https://models.dev/api.json"
-      CACHE_TTL         = 86_400
+      MEMORY_TTL     = 3 * 86_400  # 3 days
 
       MODELS_DEV_PROVIDER_MAP = {
         "openai"         => "openai",
@@ -65,25 +65,36 @@ module ActiveHarness
           end
         end
 
+        # Fetches fresh data from models.dev, writes to cache file, loads into memory.
+        # Called automatically when memory is stale. Can also be called explicitly.
+        def preload!
+          update
+        rescue StandardError
+          nil
+        ensure
+          @registry   = load_registry
+          @loaded_at  = @registry.empty? ? nil : Time.now
+          @provider_names = nil
+        end
+
         def update
           raw_api = fetch_models_dev
           models  = extract_models(raw_api)
 
           FileUtils.mkdir_p(File.dirname(cache_file))
           File.write(cache_file, JSON.generate(models))
-
-          reload!
           models.size
         end
 
         def reload!
           @registry       = nil
+          @loaded_at      = nil
           @provider_names = nil
           nil
         end
 
         def cache_file
-          File.join(project_root, "tmp", "active_harness", "pricing_models_dev.json")
+          File.join(project_root, "tmp", "active_harness", "models_dev_pricing.json")
         end
 
         def available_providers
@@ -98,18 +109,31 @@ module ActiveHarness
         private
 
         def ensure_fresh_registry
-          return if cache_file_fresh?
-          update
-        rescue StandardError
-          # Network unavailable — fall back to bundled/stale cache silently
+          return if memory_fresh?
+
+          unless file_fresh?
+            begin
+              update
+            rescue StandardError
+              nil
+            end
+          end
+
+          @registry       = load_registry
+          @loaded_at      = @registry.empty? ? nil : Time.now
+          @provider_names = nil
+        end
+
+        def memory_fresh?
+          @loaded_at && (Time.now - @loaded_at) < MEMORY_TTL
         end
 
-        def cache_file_fresh?
-          File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < CACHE_TTL
+        def file_fresh?
+          File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < MEMORY_TTL
         end
 
         def registry
-          @registry ||= load_registry
+          @registry ||= []
         end
 
         def load_registry

data/lib/active_harness/pricing/openrouter.rb

@@ -5,162 +5,309 @@ require "fileutils"
 
 module ActiveHarness
   module Pricing
-    # Fetches image-model pricing directly from the OpenRouter API.
+    # Fetches complete pricing for all OpenRouter models across all modalities.
     #
-    # models.dev only has generic prompt/completion rates for OpenRouter models.
-    # OpenRouter's own /endpoints API exposes a separate `image_output` rate
-    # that reflects the real cost of image generation tokens.
+    # OpenRouter exposes models via several endpoints:
+    #   GET /api/v1/models                          → 337 text models (base)
+    #   GET /api/v1/models?output_modalities=image  → 32 image-gen models (25 extra)
+    #   GET /api/v1/models?output_modalities=embeddings    → 26 models (all extra)
+    #   GET /api/v1/models?output_modalities=speech        →  9 models (all extra)
+    #   GET /api/v1/models?output_modalities=transcription → 10 models (all extra)
+    #   GET /api/v1/models?output_modalities=video         → 14 models (all zero pricing)
+    #   GET /api/v1/models?output_modalities=rerank        →  4 models (all zero pricing)
     #
-    # Data flow:
-    #   1. GET /api/v1/models?output_modalities=image  → list of image models
-    #   2. GET /api/v1/models/{id}/endpoints           → per-model, picks first
-    #      active endpoint to get image_output rate
-    #   3. Result cached to tmp/active_harness/pricing_openrouter.json for 24h
+    # For image-output models, /api/v1/models/{id}/endpoints is also fetched
+    # to get the accurate `image_output` per-token rate.
+    #
+    # All models are merged by id; pricing fields are populated per-modality:
+    #   text_input / text_output — text tokens
+    #   image_input              — image tokens accepted as input (vision)
+    #   image_output             — image generation tokens (from /endpoints)
+    #   audio_input              — audio tokens as input
+    #   audio_output             — audio tokens as output (TTS)
+    #   cache_read / cache_write — cache tokens
+    #   web_search               — per web-search request
     #
     # Usage:
     #   Pricing::OpenRouter.find("openai/gpt-5-image-mini")  # → ModelPrice or nil
+    #   Pricing::OpenRouter.all                              # → Array<ModelPrice>
     #   Pricing::OpenRouter.update                           # force refresh
     module OpenRouter
-      API_BASE  = "https://openrouter.ai/api/v1/models"
-      CACHE_TTL = 86_400
+      API_BASE   = "https://openrouter.ai/api/v1/models"
+      MEMORY_TTL = 3 * 86_400  # 3 days
+
+      # Modalities that have models outside the base text-337 set.
+      EXTRA_MODALITIES = %w[image embeddings speech transcription video rerank].freeze
 
       class << self
-        # Returns a ModelPrice for the given OpenRouter model id, or nil.
-        # Automatically refreshes the cache if missing or stale.
         def find(model_id)
           ensure_fresh_registry
           raw = registry.find { |m| m[:id] == model_id.to_s }
           raw ? build_price(raw) : nil
         end
 
-        # Fetches fresh data from OpenRouter and writes the cache.
-        # Returns the number of models saved.
-        def update
-          image_models = fetch_image_models
-          enriched = image_models.map { |m| enrich_with_endpoint(m) }
+        def all
+          ensure_fresh_registry
+          registry.filter_map { |raw| build_price(raw) }
+        end
 
+        def preload!
+          update
+        rescue StandardError
+          nil
+        ensure
+          @registry  = load_registry
+          @loaded_at = @registry.empty? ? nil : Time.now
+        end
+
+        def update
+          entries = collect_all_models
           FileUtils.mkdir_p(File.dirname(cache_file))
-          File.write(cache_file, JSON.generate(enriched))
-          reload!
-          enriched.size
+          File.write(cache_file, JSON.generate(entries))
+          entries.size
         end
 
         def reload!
-          @registry = nil
+          @registry  = nil
+          @loaded_at = nil
         end
 
         def cache_file
-          File.join(project_root, "tmp", "active_harness", "pricing_openrouter.json")
+          File.join(project_root, "tmp", "active_harness", "openrouter_pricing.json")
         end
 
         private
 
+        # ── Freshness ────────────────────────────────────────────────────
+
         def ensure_fresh_registry
-          return if cache_fresh?
-          update
-        rescue StandardError
-          # network unavailable — fall back to stale cache silently
+          return if memory_fresh?
+          unless file_fresh?
+            begin
+              update
+            rescue StandardError
+              nil
+            end
+          end
+          @registry  = load_registry
+          @loaded_at = @registry.empty? ? nil : Time.now
+        end
+
+        def memory_fresh?
+          @loaded_at && (Time.now - @loaded_at) < MEMORY_TTL
         end
 
-        def cache_fresh?
-          File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < CACHE_TTL
+        def file_fresh?
+          File.exist?(cache_file) && (Time.now - File.mtime(cache_file)) < MEMORY_TTL
         end
 
         def registry
-          @registry ||= begin
-            return [] unless File.exist?(cache_file)
-            JSON.parse(File.read(cache_file), symbolize_names: true)
-          rescue JSON::ParserError
-            []
-          end
+          @registry ||= []
         end
 
-        # Fetch all models with image output from OpenRouter.
-        def fetch_image_models
-          uri = URI("#{API_BASE}?output_modalities=image")
-          response = http_get(uri)
-          data = JSON.parse(response.body, symbolize_names: true)
-          data[:data] || []
+        def load_registry
+          return [] unless File.exist?(cache_file)
+          data = JSON.parse(File.read(cache_file), symbolize_names: true)
+          data.is_a?(Array) ? data : []
+        rescue JSON::ParserError
+          []
         end
 
-        # Fetch /endpoints for the model and merge image_output pricing.
-        def enrich_with_endpoint(model)
-          model_id = model[:id]
-          base_pricing = model[:pricing] || {}
+        # ── Data collection ──────────────────────────────────────────────
+
+        # Fetches all modality endpoints, merges by id, enriches image models.
+        def collect_all_models
+          models = {}
+
+          # Base text models
+          fetch_models(API_BASE).each do |m|
+            models[m[:id]] = normalize(m)
+          end
+
+          # Specialized modalities — add extra models and merge pricing
+          EXTRA_MODALITIES.each do |mod|
+            fetch_models("#{API_BASE}?output_modalities=#{mod}").each do |m|
+              id = m[:id]
+              if models[id]
+                merge_pricing!(models[id], m)
+              else
+                models[id] = normalize(m)
+              end
+            end
+          end
 
-          endpoint_pricing = fetch_endpoint_pricing(model_id)
+          # Enrich image-output models with /endpoints for accurate image_output rate
+          models.values.map do |entry|
+            if Array(entry[:output_modalities]).include?("image")
+              enrich_with_endpoint(entry)
+            else
+              entry
+            end
+          end
+        end
 
+        # Normalize a raw API model hash into our cache entry format.
+        def normalize(m)
+          p = m[:pricing] || {}
           {
-            id:               model_id,
-            name:             model[:name],
-            input_modalities:  model.dig(:architecture, :input_modalities)  || [],
-            output_modalities: model.dig(:architecture, :output_modalities) || [],
-            prompt:            base_pricing[:prompt].to_s,
-            completion:        base_pricing[:completion].to_s,
-            image_output:      endpoint_pricing&.dig(:image_output).to_s,
-            image:             endpoint_pricing&.dig(:image).to_s,
-            cache_read:        (endpoint_pricing&.dig(:input_cache_read) || base_pricing[:input_cache_read]).to_s
+            id:                m[:id],
+            name:              m[:name],
+            input_modalities:  m.dig(:architecture, :input_modalities)  || [],
+            output_modalities: m.dig(:architecture, :output_modalities) || [],
+            text_input:        p[:prompt].to_s,
+            text_output:       p[:completion].to_s,
+            image_input:       p[:image].to_s,
+            audio_input:       p[:audio].to_s,
+            image_output:      "",
+            audio_output:      "",
+            cache_read:        p[:input_cache_read].to_s,
+            cache_write:       p[:input_cache_write].to_s,
+            web_search:        p[:web_search].to_s
           }
         end
 
-        # Returns the pricing hash from the first active endpoint, or nil.
+        # Merge non-zero pricing fields from a new API response into existing entry.
+        def merge_pricing!(entry, raw_model)
+          p = raw_model[:pricing] || {}
+          [
+            [:text_input,  p[:prompt]],
+            [:text_output, p[:completion]],
+            [:image_input, p[:image]],
+            [:audio_input, p[:audio]],
+            [:cache_read,  p[:input_cache_read]],
+            [:cache_write, p[:input_cache_write]],
+            [:web_search,  p[:web_search]]
+          ].each do |key, val|
+            entry[key] = val.to_s if val.to_f > 0 && entry[key].to_f == 0
+          end
+
+          # Merge modalities (union)
+          new_out = raw_model.dig(:architecture, :output_modalities) || []
+          entry[:output_modalities] = (Array(entry[:output_modalities]) | new_out).uniq
+          new_in  = raw_model.dig(:architecture, :input_modalities)  || []
+          entry[:input_modalities]  = (Array(entry[:input_modalities])  | new_in).uniq
+        end
+
+        # Fetch /endpoints and add image_output rate to the entry.
+        def enrich_with_endpoint(entry)
+          pricing = fetch_endpoint_pricing(entry[:id])
+          entry[:image_output] = pricing&.dig(:image_output).to_s
+          entry[:audio_output] = pricing&.dig(:audio_output).to_s
+          entry
+        rescue StandardError
+          entry
+        end
+
         def fetch_endpoint_pricing(model_id)
           uri = URI("#{API_BASE}/#{model_id}/endpoints")
-          response = http_get(uri)
-          data = JSON.parse(response.body, symbolize_names: true)
+          resp = http_get(uri)
+          data = JSON.parse(resp.body, symbolize_names: true)
           endpoints = data.dig(:data, :endpoints) || []
-
-          # Prefer the first endpoint with status == 0 (online), else first available.
           ep = endpoints.find { |e| e[:status] == 0 } || endpoints.first
           ep&.dig(:pricing)
         rescue StandardError
           nil
         end
 
-        # Build a ModelPrice compatible with the rest of the Pricing system.
-        # For image-output models, uses image_output rate for output_per_million.
+        def fetch_models(url)
+          resp = http_get(URI(url))
+          data = JSON.parse(resp.body, symbolize_names: true)
+          data[:data] || []
+        end
+
+        # ── Build ModelPrice ─────────────────────────────────────────────
+
         def build_price(raw)
-          is_image_output = Array(raw[:output_modalities]).include?("image")
+          out_mods = Array(raw[:output_modalities])
+          inp_mods = Array(raw[:input_modalities])
+
+          is_imggen        = out_mods.include?("image")
+          is_embed         = out_mods.include?("embeddings")
+          is_speech        = out_mods.include?("speech")
+          is_transcription = out_mods.include?("transcription")
 
-          input_pm      = to_per_million(raw[:prompt])
-          completion_pm = to_per_million(raw[:completion])
-          image_out_pm  = to_per_million(raw[:image_output])
-          cache_pm      = to_per_million(raw[:cache_read])
+          text_in_pm  = to_pm(raw[:text_input])
+          text_out_pm = to_pm(raw[:text_output])
+          img_in_pm   = to_pm(raw[:image_input])
+          img_out_pm  = to_pm(raw[:image_output])
+          # p[:audio] field — audio input tokens (multimodal/embedding models like Gemini)
+          audio_in_pm = to_pm(raw[:audio_input])
+          aud_out_pm  = to_pm(raw[:audio_output])
+          cache_r_pm  = to_pm(raw[:cache_read])
+          cache_w_pm  = to_pm(raw[:cache_write])
+          # web_search is a flat per-request fee in USD, not a per-token rate
+          ws_raw         = raw[:web_search].to_s
+          web_search_usd = ws_raw.empty? ? nil : (ws_raw.to_f > 0 ? ws_raw.to_f : nil)
+
+          # Transcription pricing is stored in `prompt` but the unit differs by model:
+          #   prompt < 0.0001  → per-audio-token  (e.g. gpt-4o-transcribe $2.5/M)  → use to_pm
+          #   prompt >= 0.0001 → per-minute of audio (e.g. Whisper $0.006/min)      → raw USD
+          if is_transcription
+            raw_rate = raw[:text_input].to_s.to_f
+            audio_in_pm = if raw_rate > 0 && raw_rate < 0.0001
+              to_pm(raw[:text_input])    # per-token → convert to per-million
+            elsif raw_rate > 0
+              raw_rate                   # per-minute → keep raw USD value
+            end
+            text_in_pm = nil
+          end
+
+          # Primary output for cost calculation and sorting:
+          # imggen  → image_output rate (from /endpoints)
+          # speech  → audio_output rate (completion is audio)
+          # embed / transcription → no output cost
+          # text    → text_output rate
+          primary_output = if is_imggen
+            img_out_pm || text_out_pm
+          elsif is_speech
+            aud_out_pm || text_out_pm
+          elsif is_embed || is_transcription
+            nil
+          else
+            text_out_pm
+          end
 
-          output_pm = (is_image_output && image_out_pm) ? image_out_pm : completion_pm
+          # Primary input for cost calculation and sorting
+          primary_input = is_transcription ? audio_in_pm : text_in_pm
 
-          return nil unless input_pm || output_pm
+          # Skip models with no id/name; keep zero-priced models (rerank, video) —
+          # they are real models, just have $0 rates in the OpenRouter API.
+          return nil unless raw[:id] && raw[:name]
 
           Pricing::ModelPrice.new(
             id:                           raw[:id],
             name:                         raw[:name],
             provider:                     "openrouter",
-            input_per_million:            input_pm,
-            output_per_million:           output_pm,
-            cache_read_input_per_million: cache_pm,
-            cache_write_input_per_million: nil,
+            input_per_million:            primary_input,
+            output_per_million:           primary_output,
+            cache_read_input_per_million: cache_r_pm,
+            cache_write_input_per_million: cache_w_pm,
             context_window:               nil,
             max_output_tokens:            nil,
-            input_modalities:             Array(raw[:input_modalities]),
-            output_modalities:            Array(raw[:output_modalities])
+            input_modalities:             inp_mods,
+            output_modalities:            out_mods,
+            image_input_per_million:      img_in_pm,
+            image_output_per_million:     img_out_pm,
+            audio_input_per_million:      audio_in_pm,
+            audio_output_per_million:     aud_out_pm,
+            web_search_per_request:       web_search_usd
           )
         end
 
-        # OpenRouter pricing fields are per-token strings (e.g. "0.000008").
-        # Convert to per-million float. Returns nil for zero/blank values.
-        def to_per_million(value)
-          return nil if value.nil? || value.to_s.empty?
+        # Per-token string → per-million float. Returns nil for zero/blank.
+        def to_pm(value)
+          return nil if value.nil? || value.to_s.strip.empty?
           f = value.to_f
           return nil if f <= 0
           (f * 1_000_000).round(6)
         end
 
         def http_get(uri)
-          response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", read_timeout: 10) do |http|
-            http.get(uri.request_uri)
+          resp = Net::HTTP.start(uri.host, uri.port, use_ssl: true, read_timeout: 15) do |h|
+            h.get(uri.request_uri)
           end
-          raise "OpenRouter API returned HTTP #{response.code} for #{uri}" unless response.is_a?(Net::HTTPSuccess)
-          response
+          raise "OpenRouter API #{resp.code} for #{uri}" unless resp.is_a?(Net::HTTPSuccess)
+          resp
         end
 
         def project_root

data/lib/active_harness/pricing.rb

@@ -13,33 +13,47 @@ module ActiveHarness
   #   Pricing.providers.openai     → Array<ModelPrice>
   #   Pricing.update               → refreshes ModelsDev cache
   module Pricing
-    # Pricing rates for a single model (per-million USD).
+    # Pricing rates for a single model.
+    # All *_per_million fields are in USD per 1M tokens.
+    # audio_input_per_million / audio_output_per_million may represent
+    # per-million audio tokens or per-unit (second/char) depending on provider.
     ModelPrice = Struct.new(
       :id,
       :name,
       :provider,
-      :input_per_million,
-      :output_per_million,
+      # Primary fields (used for cost calculation, backward-compatible)
+      :input_per_million,               # text tokens input
+      :output_per_million,              # primary output (text or image_output for imggen)
       :cache_read_input_per_million,
       :cache_write_input_per_million,
       :context_window,
       :max_output_tokens,
       :input_modalities,
       :output_modalities,
+      # Extended modality-specific pricing
+      :image_input_per_million,         # image tokens accepted as input (vision models)
+      :image_output_per_million,        # image generation output tokens (imggen models)
+      :audio_input_per_million,         # audio tokens accepted as input
+      :audio_output_per_million,        # audio output tokens (TTS models)
+      :web_search_per_request,          # per web-search call in USD
       keyword_init: true
     ) do
       # Capability tags derived from modality data.
-      # Possible values: "vision", "pdf", "audio", "video", "imggen", "embed"
+      # Possible values: "vision", "pdf", "audio", "video", "imggen", "embed",
+      #                  "speech", "transcription", "rerank"
       def categories
-        inp = input_modalities  || []
-        out = output_modalities || []
+        inp = Array(input_modalities)
+        out = Array(output_modalities)
         cats = []
-        cats << "vision" if inp.include?("image")
-        cats << "pdf"    if inp.include?("pdf")
-        cats << "audio"  if inp.include?("audio") || out.include?("audio")
-        cats << "video"  if inp.include?("video")  || out.include?("video")
-        cats << "imggen" if out.include?("image")
-        cats << "embed"  if id.to_s.match?(/embed/i) || name.to_s.match?(/embed/i)
+        cats << "vision"        if inp.include?("image")
+        cats << "pdf"           if inp.include?("pdf")
+        cats << "audio"         if inp.include?("audio")
+        cats << "video"         if inp.include?("video") || out.include?("video")
+        cats << "imggen"        if out.include?("image")
+        cats << "speech"        if out.include?("speech")
+        cats << "transcription" if out.include?("transcription")
+        cats << "rerank"        if out.include?("rerank")
+        cats << "embed"         if out.include?("embeddings")
         cats
       end
 
@@ -91,6 +105,13 @@ module ActiveHarness
     # Facade — delegates to ModelsDev (general fallback source)
     # ---------------------------------------------------------------------------
     class << self
+      # Eagerly fetch all pricing sources and load them into memory.
+      # Called at Rails startup. Network failures are silently ignored.
+      def preload!
+        ModelsDev.preload!
+        OpenRouter.preload!
+      end
+
       def find(model_id)
         ModelsDev.find(model_id)
       end

data/lib/active_harness/railtie.rb

@@ -8,5 +8,9 @@ module ActiveHarness
         app.config.autoload_paths << path.to_s if path.exist?
       end
     end
+
+    config.after_initialize do
+      ActiveHarness::Pricing.preload!
+    end
   end
 end

data/lib/active_harness.rb

@@ -34,7 +34,7 @@ require_relative "active_harness/pipeline"
 require_relative "active_harness/railtie" if defined?(Rails::Railtie)
 
 module ActiveHarness
-  VERSION = "0.2.34"
+  VERSION = "0.2.35"
 
   class << self
     # Configure ActiveHarness.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_harness
 version: !ruby/object:Gem::Version
-  version: 0.2.34
+  version: 0.2.35
 platform: ruby
 authors:
 - the-teacher