active_harness 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 779258acc7a5947e923ac8f5ad262f653e96ad0509585f8a309e362bc0ee3703
-  data.tar.gz: 1364d064abfe80ccdd5fdadd7bfb6d55396ec2ceaf7680d0bdd1373011cd9923
+  metadata.gz: 7de169d09a7a2e65620b93b57b9b760e8cf740342d35383deea404d1edf0f91e
+  data.tar.gz: 4a36da37f5953611eb9c9fe5ad6710381ec332e6ea49e1355af50540baab2fd0
 SHA512:
-  metadata.gz: 94bf07f8a86b78da1f5507e02644c1a6bb2310074ea3b02eca476164480a546c286d12c699ee97743a4e8b9e1c52fd74f1fc1f33e90f1302c4235b0009e2bbd6
-  data.tar.gz: 3355b0cb47da66faa4164f4712ba7f5e9e0e23d405a00177599ce26d34255ea3e80b29effa8f9fa7d5bfc34e6dfb45005e0a6281df70dfa847db965e7039e54e
+  metadata.gz: ab66c8538123326d8bbe8a634ee22420646cfd66f316ea503499585d058a521a5903b4e1487d29628117ddb4d7d8872839359f7977e5dda905ac7eb4c05453a4
+  data.tar.gz: 60e8a27f74ad023a94c039d09b374525fa571f4b2e09b15c42678c1acbd1ba16bd0a4012c1399f343f10eb24785af05ac4f102d11cf039580faa7381510a0f56

data/lib/active_harness/configuration.rb

@@ -99,6 +99,12 @@ module ActiveHarness
     attr_accessor :azure_api_base       # e.g. "https://my-resource.openai.azure.com"
     attr_accessor :azure_api_version    # e.g. "2024-05-01-preview"
 
+    # -------------------------------------------------------------------------
+    # Vercel AI Gateway — TypeSafe-compatible evaluation endpoint (Jev)
+    # -------------------------------------------------------------------------
+    attr_accessor :vercel_api_key
+    attr_accessor :vercel_api_url
+
     # -------------------------------------------------------------------------
     # Custom providers
     #
@@ -170,6 +176,9 @@ module ActiveHarness
       @azure_ai_auth_token = ENV["AZURE_AI_AUTH_TOKEN"]
       @azure_api_base      = ENV["AZURE_API_BASE"]
       @azure_api_version   = ENV.fetch("AZURE_API_VERSION", "2024-05-01-preview")
+
+      @vercel_api_key = ENV["VERCEL_API_KEY"]
+      @vercel_api_url = "https://ai-gateway.vercel.sh/typesafe/v1/systemone"
     end
   end
 end

data/lib/active_harness/providers/vercel.rb

@@ -0,0 +1,120 @@
+require "uri"
+
+module ActiveHarness
+  module Providers
+    # Vercel AI Gateway — TypeSafe-compatible "System One" evaluation endpoint (Jev).
+    #
+    # This is fundamentally different from every other provider here: Jev does not
+    # take free-form chat messages and does not return free text. It evaluates a
+    # single `state` string against typed `questions` and returns typed answers
+    # (a score/probability/choice per question), e.g.:
+    #
+    #   POST https://ai-gateway.vercel.sh/typesafe/v1/systemone
+    #   { "model": "typesafe-ai/jev", "state": "...", "questions": { "answer": { "type": "noul", "instructions": "..." } } }
+    #   => { "answers": { "answer": { "type": "noul", "noul": 0.8 } }, "usage": {...} }
+    #
+    # Question types are "noul" (0..1 probability, optionally scoped by a
+    # true/false `criteria` hash), "choice" (requires `criteria` — a name=>
+    # description hash), and "score" (requires `criteria` — an ordered array of
+    # at least 2 labels).
+    #
+    # To fit the standard Request(messages:) call shape without building a whole
+    # typed-questions DSL into Request itself, this bridges it minimally: the
+    # last user message becomes `state`, and — unless the model-chain entry
+    # supplies its own `questions:` hash (already in the API's own shape) — the
+    # system message (if any) becomes a single default "noul" question's
+    # `instructions`. Either way the raw `answers` payload is returned as-is
+    # (pretty JSON) as `content` — no interpretation of the typed answer is
+    # attempted.
+    #
+    #   # simplest form — one implicit yes/no-ish question from the system prompt
+    #   model do
+    #     use provider: :vercel, model: "typesafe-ai/jev"
+    #   end
+    #
+    #   # explicit multi-question form — full control over the question set
+    #   model do
+    #     use provider: :vercel, model: "typesafe-ai/jev", questions: {
+    #       sentiment: { type: "score",  instructions: "...", criteria: ["negative", "neutral", "positive"] },
+    #       topic:     { type: "choice", instructions: "...", criteria: { support: "...", spam: "..." } },
+    #       urgent:    { type: "noul",   instructions: "..." }
+    #     }
+    #   end
+    class Vercel < Base
+      DEFAULT_INSTRUCTIONS = "Evaluate the given state.".freeze
+
+      def call(model:, messages:, temperature: nil, stream: nil, questions: nil)
+        raise Errors::InvalidRequestError, "provider: :vercel (Jev) does not support token streaming" if stream
+
+        state = messages.reverse.find { |m| m[:role] == "user" }&.fetch(:content, nil).to_s
+
+        headers = {
+          "Content-Type"  => "application/json",
+          "Authorization" => "Bearer #{api_key}"
+        }
+        body = {
+          model:     model,
+          state:     state,
+          questions: questions || default_questions(messages)
+        }
+
+        raw  = post_json(URI(config.vercel_api_url), headers: headers, body: body)
+        data = parse!(raw)
+        handle_error!(data)
+
+        {
+          content:  JSON.pretty_generate(data["answers"]),
+          provider: :vercel,
+          model:    data["model"] || model,
+          usage:    extract_usage(data)
+        }
+      end
+
+      private
+
+      def default_questions(messages)
+        instructions = messages.find { |m| m[:role] == "system" }&.fetch(:content, nil).to_s
+        instructions = DEFAULT_INSTRUCTIONS if instructions.empty?
+        { answer: { type: "noul", instructions: instructions } }
+      end
+
+      def api_key
+        key = config.vercel_api_key.to_s
+        raise Errors::InvalidApiKeyError, "vercel_api_key is not configured" if key.empty?
+        key
+      end
+
+      # Jev's usage object only has input/output tokens, OpenAI-shaped but
+      # under different keys than the chat-completions providers use.
+      def extract_usage(data)
+        u = data["usage"]
+        return nil unless u
+
+        input  = u["input_tokens"].to_i
+        output = u["output_tokens"].to_i
+        { input_tokens: input, output_tokens: output, total_tokens: input + output }
+      end
+
+      # Two error shapes seen in practice:
+      # - TypeSafe request-validation errors: { "message": "...", "error_type": "..." }
+      # - AI Gateway account/billing errors (OpenAI-style, nested): { "error": { "message": "...", "type": "..." } }
+      def handle_error!(data)
+        msg, type =
+          if data["message"] && data["error_type"]
+            [data["message"].to_s, data["error_type"].to_s]
+          elsif data["error"].is_a?(Hash)
+            [data["error"]["message"].to_s, data["error"]["type"].to_s]
+          end
+        return unless msg
+
+        case type
+        when "invalid_request"                then raise Errors::InvalidRequestError.new(msg, error_code: type)
+        when "unauthorized", "authentication_error" then raise Errors::InvalidApiKeyError.new(msg, error_code: type)
+        when "rate_limit", "rate_limit_error" then raise Errors::RateLimitError.new(msg, error_code: type)
+        when "customer_verification_required" then raise Errors::InvalidApiKeyError.new(msg, error_code: type)
+        else                                        raise Errors::ProviderError.new(msg, error_code: type)
+        end
+      end
+    end
+  end
+end

data/lib/active_harness/request/models.rb

@@ -134,7 +134,7 @@ module ActiveHarness
       @models = []
     end
 
-    def use(provider:, model:, temperature: nil, name: nil, size: nil, quality: nil, retry_attempts: nil, retry_delay: nil)
+    def use(provider:, model:, temperature: nil, name: nil, size: nil, quality: nil, retry_attempts: nil, retry_delay: nil, questions: nil)
       @models << {
         provider:       provider,
         model:          model,
@@ -143,7 +143,8 @@ module ActiveHarness
         size:           size,
         quality:        quality,
         retry_attempts: retry_attempts,
-        retry_delay:    retry_delay
+        retry_delay:    retry_delay,
+        questions:      questions
       }.compact
     end
 

data/lib/active_harness/request/providers.rb

@@ -35,7 +35,8 @@ module ActiveHarness
       azure:       -> { Providers::Azure.new },
       bedrock:     -> { Providers::Bedrock.new },
       vertexai:    -> { Providers::VertexAI.new },
-      custom:      -> { Providers::Custom.new }
+      custom:      -> { Providers::Custom.new },
+      vercel:      -> { Providers::Vercel.new }
     }.freeze
 
     IMAGE_PROVIDERS = {
@@ -61,6 +62,7 @@ module ActiveHarness
       opts[:temperature] = entry[:temperature] if entry[:temperature]
       opts[:stream]      = @token               if @token
       opts[:name]        = entry[:name]        if entry[:name]
+      opts[:questions]   = entry[:questions]   if entry[:questions]
       provider.call(**opts)
     end
 

data/lib/active_harness.rb

@@ -21,6 +21,7 @@ require_relative "active_harness/providers/azure"
 require_relative "active_harness/providers/bedrock"
 require_relative "active_harness/providers/vertexai"
 require_relative "active_harness/providers/custom"
+require_relative "active_harness/providers/vercel"
 require_relative "active_harness/providers/images/openai"
 require_relative "active_harness/providers/images/openrouter"
 require_relative "active_harness/providers/audio/openai"
@@ -34,7 +35,7 @@ require_relative "active_harness/pipeline"
 require_relative "active_harness/railtie" if defined?(Rails::Railtie)
 
 module ActiveHarness
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 
   class << self
     # Configure ActiveHarness.

data/lib/generators/active_harness/install/install_generator.rb

@@ -43,11 +43,11 @@ module ActiveHarness
       def inject_routes
         route <<~ROUTES.strip
           # ActiveHarness — AI support endpoints
-          post "ai/agent",        to: "ai_support#agent"
-          post "ai/agent_memory", to: "ai_support#agent_memory"
-          post "ai/tribunal",     to: "ai_support#tribunal"
-          post "ai/pipeline",     to: "ai_support#pipeline"
-          get  "ai/agent_stream", to: "ai_support#agent_stream"
+          post "ai/agent",          to: "ai_support#agent" # not "request" — would override ActionController::Base#request
+          post "ai/request_memory", to: "ai_support#request_memory"
+          post "ai/tribunal",       to: "ai_support#tribunal"
+          post "ai/pipeline",       to: "ai_support#pipeline"
+          get  "ai/request_stream", to: "ai_support#request_stream"
         ROUTES
       end
 

data/lib/generators/active_harness/install/templates/controllers/ai_controller.rb

@@ -6,6 +6,10 @@ class AiSupportController < ApplicationController
   # ---------------------------------------------------------------------------
   # POST /ai/agent
   # body: { input: "What is your return policy?" }
+  #
+  # Kept named "agent" (not "request") — defining a #request method on a Rails
+  # controller would override ActionController::Base#request and break
+  # request.env / CSRF / anything else that relies on the real HTTP request.
   # ---------------------------------------------------------------------------
   def agent
     result = SupportRequest.call(input: params.require(:input))
@@ -18,13 +22,13 @@ class AiSupportController < ApplicationController
   end
 
   # ---------------------------------------------------------------------------
-  # POST /ai/agent_memory
+  # POST /ai/request_memory
   # body: { input: "Does that apply to accessories?", session_id: "user_42" }
   #
   # Uses AppMemory so the same session keeps conversational context
   # across multiple requests.
   # ---------------------------------------------------------------------------
-  def agent_memory
+  def request_memory
     memory = AppMemory.new(session_id: params.require(:session_id))
     result = SupportRequest.call(input: params.require(:input), memory: memory)
 
@@ -71,21 +75,21 @@ class AiSupportController < ApplicationController
   end
 
   # ---------------------------------------------------------------------------
-  # GET /ai/agent_stream?input=What+is+your+return+policy%3F
+  # GET /ai/request_stream?input=What+is+your+return+policy%3F
   #
   # Streams the response token by token using Server-Sent Events.
   # Each token arrives as:   data: {"token":"..."}
   # End of stream is marked: data: {"done":true}
   #
   # JavaScript client example:
-  #   const es = new EventSource('/ai/agent_stream?input=Hello');
+  #   const es = new EventSource('/ai/request_stream?input=Hello');
   #   es.onmessage = ({ data }) => {
   #     const { token, done } = JSON.parse(data);
   #     if (done) { es.close(); return; }
   #     document.querySelector('#output').insertAdjacentText('beforeend', token);
   #   };
   # ---------------------------------------------------------------------------
-  def agent_stream
+  def request_stream
     input = params.require(:input)
 
     response.headers["Content-Type"]  = "text/event-stream"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_harness
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - the-teacher
@@ -80,6 +80,7 @@ files:
 - lib/active_harness/providers/openai.rb
 - lib/active_harness/providers/openrouter.rb
 - lib/active_harness/providers/perplexity.rb
+- lib/active_harness/providers/vercel.rb
 - lib/active_harness/providers/vertexai.rb
 - lib/active_harness/providers/xai.rb
 - lib/active_harness/railtie.rb