aispec 0.1.0

data/lib/aispec/core/providers/openai.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module AISpec
+  module Core
+    module Providers
+      class OpenAI < Base
+        BASE_URI = "https://api.openai.com/v1/chat/completions"
+
+        def initialize(model: "gpt-4o", api_key: nil, **options)
+          super(model: model, **options)
+          @api_key = api_key || ENV["OPENAI_API_KEY"]
+        end
+
+        def generate(prompt, system_prompt: nil, temperature: 0.7, **_params)
+          return fallback_mock_if_no_key(prompt) unless @api_key && !@api_key.empty?
+
+          messages = []
+          messages << { role: "system", content: system_prompt } if system_prompt
+          messages << { role: "user", content: prompt }
+
+          payload = {
+            model: @model || "gpt-4o",
+            messages: messages,
+            temperature: temperature
+          }
+
+          uri = URI.parse(BASE_URI)
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          req = Net::HTTP::Post.new(uri.path, {
+            "Content-Type" => "application/json",
+            "Authorization" => "Bearer #{@api_key}"
+          })
+          req.body = JSON.generate(payload)
+
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.read_timeout = AISpec.configuration.timeout
+
+          res = http.request(req)
+          elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+
+          if res.is_a?(Net::HTTPSuccess)
+            data = JSON.parse(res.body)
+            output = data.dig("choices", 0, "message", "content") || ""
+            prompt_tokens = data.dig("usage", "prompt_tokens") || 0
+            comp_tokens = data.dig("usage", "completion_tokens") || 0
+            total_tokens = data.dig("usage", "total_tokens") || (prompt_tokens + comp_tokens)
+
+            cost = estimate_cost(@model, prompt_tokens, comp_tokens)
+
+            build_response(
+              output: output,
+              latency_ms: elapsed_ms,
+              tokens: total_tokens,
+              cost: cost,
+              raw: data
+            )
+          else
+            raise "OpenAI API Error (#{res.code}): #{res.body}"
+          end
+        end
+
+        private
+
+        def fallback_mock_if_no_key(prompt)
+          Mock.new(model: @model).generate(prompt)
+        end
+
+        def estimate_cost(model_name, input_tokens, output_tokens)
+          # Estimated standard pricing per 1k tokens
+          case model_name.to_s.downcase
+          when /gpt-4o/
+            (input_tokens * 0.000005) + (output_tokens * 0.000015)
+          when /gpt-4/
+            (input_tokens * 0.00003) + (output_tokens * 0.00006)
+          else
+            (input_tokens * 0.0000015) + (output_tokens * 0.000002)
+          end
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Providers::Base.register("openai", AISpec::Core::Providers::OpenAI)

data/lib/aispec/core/providers/vllm.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module AISpec
+  module Core
+    module Providers
+      class VLLM < Base
+        def initialize(model: "llama-4", endpoint: nil, api_key: nil, **options)
+          super(model: model, **options)
+          @endpoint = endpoint || ENV["VLLM_API_BASE"] || ENV["VLLM_HOST"] || "http://localhost:8000/v1/chat/completions"
+          @api_key = api_key || ENV["VLLM_API_KEY"]
+        end
+
+        def generate(prompt, system_prompt: nil, temperature: 0.7, **_params)
+          uri = URI.parse(@endpoint)
+
+          messages = []
+          messages << { role: "system", content: system_prompt } if system_prompt
+          messages << { role: "user", content: prompt }
+
+          payload = {
+            model: @model || "llama-4",
+            messages: messages,
+            temperature: temperature
+          }
+
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          headers = { "Content-Type" => "application/json" }
+          headers["Authorization"] = "Bearer #{@api_key}" if @api_key && !@api_key.empty?
+
+          req = Net::HTTP::Post.new(uri.path.empty? ? "/v1/chat/completions" : uri.path, headers)
+          req.body = JSON.generate(payload)
+
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.scheme == "https"
+          http.read_timeout = AISpec.configuration.timeout
+
+          begin
+            res = http.request(req)
+            elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+
+            if res.is_a?(Net::HTTPSuccess)
+              data = JSON.parse(res.body)
+              output = data.dig("choices", 0, "message", "content") || ""
+              prompt_tokens = data.dig("usage", "prompt_tokens") || 0
+              comp_tokens = data.dig("usage", "completion_tokens") || 0
+              total_tokens = data.dig("usage", "total_tokens") || (prompt_tokens + comp_tokens)
+
+              build_response(
+                output: output,
+                latency_ms: elapsed_ms,
+                tokens: total_tokens,
+                cost: 0.0, # Local/Self-hosted vLLM infrastructure
+                raw: data
+              )
+            else
+              raise "vLLM API Error (#{res.code}): #{res.body}"
+            end
+          rescue Exception => e
+            # If network connection fails or VCR/WebMock blocks connection in tests without active server, fallback to mock
+            if is_network_or_mock_error?(e)
+              Mock.new(model: @model).generate(prompt)
+            else
+              raise e
+            end
+          end
+        end
+
+        private
+
+        def is_network_or_mock_error?(err)
+          err.is_a?(Errno::ECONNREFUSED) ||
+            err.is_a?(SocketError) ||
+            err.class.name.include?("WebMock") ||
+            err.class.name.include?("VCR")
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Providers::Base.register("vllm", AISpec::Core::Providers::VLLM)

data/lib/aispec/core/reporters/base.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Reporters
+      class Base
+        @registry = {}
+
+        class << self
+          attr_reader :registry
+
+          def register(name, reporter_class)
+            @registry[name.to_s.downcase] = reporter_class
+          end
+
+          def get(name)
+            @registry[name.to_s.downcase] || Console
+          end
+        end
+
+        attr_reader :results, :output_io
+
+        def initialize(results, output_io: $stdout)
+          @results = results
+          @output_io = output_io
+        end
+
+        def render
+          raise NotImplementedError, "#{self.class.name} must implement #render"
+        end
+      end
+    end
+  end
+end

data/lib/aispec/core/reporters/console.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Reporters
+      class Console < Base
+        def render
+          contract = results[:contract]
+          stats = results[:statistics]
+          summary = stats.summary
+
+          output_io.puts "Running #{summary[:total_assertions]} contracts for #{contract.name} (#{results[:provider_name]} / #{results[:model]})...\n\n"
+
+          results[:runs].each_with_index do |run, idx|
+            output_io.puts "Case ##{idx + 1} (Input: \"#{truncate(run[:input])}\")" if results[:runs].size > 1
+            run[:assertion_results].each do |ar|
+              status = ar.passed? ? "\e[32mPASS\e[0m" : "\e[31mFAIL\e[0m"
+              output_io.puts "  #{status}  #{ar.name}: #{ar.message}"
+            end
+            output_io.puts ""
+          end
+
+          output_io.puts "Overall"
+          output_io.puts "  Passed: #{summary[:passed_assertions]}/#{summary[:total_assertions]} (#{summary[:success_rate]}%)"
+          output_io.puts "  Confidence Interval: ±#{summary[:confidence_interval]}%"
+          output_io.puts "  Mean Latency: #{summary[:mean_latency]}ms"
+          output_io.puts "  Total Cost: $#{sprintf('%.4f', summary[:total_cost])}"
+          output_io.puts "  Average Judge Score: #{summary[:average_judge_score] || 'N/A'}"
+          output_io.puts "  Variance: #{summary[:variance]}"
+          output_io.puts "  Recommendation: \e[1m#{summary[:recommendation]}\e[0m\n"
+        end
+
+        private
+
+        def truncate(str, max = 50)
+          s = str.to_s
+          s.length > max ? "#{s[0...max]}..." : s
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Reporters::Base.register("console", AISpec::Core::Reporters::Console)

data/lib/aispec/core/reporters/html.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Reporters
+      class HTML < Base
+        def render
+          contract = results[:contract]
+          stats = results[:statistics]
+          summary = stats.summary
+
+          html = <<~HTML
+            <!DOCTYPE html>
+            <html lang="en">
+            <head>
+              <meta charset="UTF-8">
+              <meta name="viewport" content="width=device-width, initial-scale=1.0">
+              <title>AISpec Contract Verification Report - #{contract.name}</title>
+              <style>
+                :root {
+                  --bg: #0f172a;
+                  --card-bg: #1e293b;
+                  --text: #f8fafc;
+                  --text-muted: #94a3b8;
+                  --accent: #6366f1;
+                  --success: #22c55e;
+                  --danger: #ef4444;
+                  --border: #334155;
+                }
+                body {
+                  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+                  background-color: var(--bg);
+                  color: var(--text);
+                  margin: 0;
+                  padding: 2rem;
+                }
+                .container { max-width: 1100px; margin: 0 auto; }
+                header { margin-bottom: 2rem; border-bottom: 1px solid var(--border); padding-bottom: 1rem; }
+                h1 { margin: 0 0 0.5rem 0; font-size: 2rem; }
+                .subtitle { color: var(--text-muted); font-size: 1.1rem; }
+                .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 1rem; margin-bottom: 2rem; }
+                .card { background: var(--card-bg); border: 1px solid var(--border); border-radius: 12px; padding: 1.25rem; }
+                .card-title { color: var(--text-muted); font-size: 0.85rem; text-transform: uppercase; letter-spacing: 0.05em; }
+                .card-value { font-size: 1.8rem; font-weight: 700; margin-top: 0.5rem; }
+                .badge-pass { color: var(--success); }
+                .badge-fail { color: var(--danger); }
+                section { background: var(--card-bg); border: 1px solid var(--border); border-radius: 12px; padding: 1.5rem; margin-bottom: 1.5rem; }
+                h2 { margin-top: 0; font-size: 1.3rem; border-bottom: 1px solid var(--border); padding-bottom: 0.5rem; }
+                table { width: 100%; border-collapse: collapse; margin-top: 1rem; }
+                th, td { text-align: left; padding: 0.75rem; border-bottom: 1px solid var(--border); }
+                th { color: var(--text-muted); font-weight: 600; font-size: 0.9rem; }
+                pre { background: #090d16; padding: 1rem; border-radius: 8px; overflow-x: auto; font-size: 0.9rem; color: #e2e8f0; }
+              </style>
+            </head>
+            <body>
+              <div class="container">
+                <header>
+                  <h1>AISpec Behavioral Contract Report</h1>
+                  <div class="subtitle">Contract: <strong>#{contract.name}</strong> | Provider: <strong>#{results[:provider_name]}</strong> | Model: <strong>#{results[:model]}</strong></div>
+                </header>
+
+                <div class="grid">
+                  <div class="card">
+                    <div class="card-title">Behavior Score</div>
+                    <div class="card-value #{summary[:success_rate] >= 90 ? 'badge-pass' : 'badge-fail'}">#{summary[:success_rate]}%</div>
+                    <div style="font-size:0.8rem; color:var(--text-muted);">CI: &plusmn;#{summary[:confidence_interval]}%</div>
+                  </div>
+                  <div class="card">
+                    <div class="card-title">Assertions</div>
+                    <div class="card-value">#{summary[:passed_assertions]} / #{summary[:total_assertions]}</div>
+                  </div>
+                  <div class="card">
+                    <div class="card-title">Mean Latency</div>
+                    <div class="card-value">#{summary[:mean_latency]} ms</div>
+                  </div>
+                  <div class="card">
+                    <div class="card-title">Total Cost</div>
+                    <div class="card-value">$#{sprintf('%.4f', summary[:total_cost])}</div>
+                  </div>
+                  <div class="card">
+                    <div class="card-title">Recommendation</div>
+                    <div class="card-value" style="font-size:1.2rem; margin-top:0.8rem;">#{summary[:recommendation]}</div>
+                  </div>
+                </div>
+
+                <section>
+                  <h2>Summary & Analysis</h2>
+                  <p><strong>Average Judge Score:</strong> #{summary[:average_judge_score] || 'N/A'}</p>
+                  <p><strong>Variance:</strong> #{summary[:variance]}</p>
+                </section>
+
+                <section>
+                  <h2>Test Runs & Prompt Details</h2>
+                  #{render_runs_table}
+                </section>
+              </div>
+            </body>
+            </html>
+          HTML
+
+          output_io.puts html
+          html
+        end
+
+        private
+
+        def render_runs_table
+          rows = results[:runs].map.with_index do |run, idx|
+            assertions_html = run[:assertion_results].map do |ar|
+              cls = ar.passed? ? "badge-pass" : "badge-fail"
+              st = ar.passed? ? "PASS" : "FAIL"
+              "<div><strong class='#{cls}'>[#{st}] #{ar.name}</strong>: #{ar.message}</div>"
+            end.join
+
+            <<~TR
+              <tr>
+                <td>#{idx + 1}</td>
+                <td><pre>#{run[:input]}</pre></td>
+                <td><pre>#{run[:output]}</pre></td>
+                <td>#{run[:latency_ms].round(1)} ms</td>
+                <td>#{assertions_html}</td>
+              </tr>
+            TR
+          end.join
+
+          <<~TABLE
+            <table>
+              <thead>
+                <tr>
+                  <th>#</th>
+                  <th>Input / Prompt</th>
+                  <th>AI Output</th>
+                  <th>Latency</th>
+                  <th>Assertion Results</th>
+                </tr>
+              </thead>
+              <tbody>
+                #{rows}
+              </tbody>
+            </table>
+          TABLE
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Reporters::Base.register("html", AISpec::Core::Reporters::HTML)

data/lib/aispec/core/reporters/json.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "json"
+
+module AISpec
+  module Core
+    module Reporters
+      class JSONReporter < Base
+        def render
+          contract = results[:contract]
+          stats = results[:statistics]
+
+          data = {
+            contract: contract.name,
+            provider: results[:provider_name],
+            model: results[:model],
+            summary: stats.summary,
+            runs: results[:runs].map do |run|
+              {
+                input: run[:input],
+                output: run[:output],
+                latency_ms: run[:latency_ms],
+                cost: run[:cost],
+                tokens: run[:tokens],
+                passed: run[:passed],
+                assertions: run[:assertion_results].map do |ar|
+                  {
+                    name: ar.name,
+                    passed: ar.passed?,
+                    message: ar.message,
+                    metadata: ar.metadata
+                  }
+                end
+              }
+            end
+          }
+
+          formatted = JSON.pretty_generate(data)
+          output_io.puts formatted
+          formatted
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Reporters::Base.register("json", AISpec::Core::Reporters::JSONReporter)

data/lib/aispec/core/reporters/junit.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+begin
+  require "rexml/document"
+rescue LoadError
+end
+
+module AISpec
+  module Core
+    module Reporters
+      class JUnit < Base
+        def render
+          contract = results[:contract]
+          stats = results[:statistics]
+          summary = stats.summary
+
+          doc = REXML::Document.new
+          doc << REXML::XMLDecl.new("1.0", "UTF-8")
+
+          testsuite = doc.add_element("testsuite", {
+            "name" => contract.name,
+            "tests" => summary[:total_assertions].to_s,
+            "failures" => summary[:failed_assertions].to_s,
+            "time" => (summary[:mean_latency] / 1000.0).to_s
+          })
+
+          results[:runs].each_with_index do |run, i|
+            run[:assertion_results].each do |ar|
+              tc = testsuite.add_element("testcase", {
+                "classname" => "#{contract.name}.Case#{i + 1}",
+                "name" => ar.name.to_s,
+                "time" => (run[:latency_ms] / 1000.0).to_s
+              })
+
+              unless ar.passed?
+                failure = tc.add_element("failure", { "message" => ar.message })
+                failure.text = ar.message
+              end
+            end
+          end
+
+          output = ""
+          doc.write(output, 2)
+          output_io.puts output
+          output
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Reporters::Base.register("junit", AISpec::Core::Reporters::JUnit)

data/lib/aispec/core/reporters/markdown.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Reporters
+      class Markdown < Base
+        def render
+          contract = results[:contract]
+          stats = results[:statistics]
+          summary = stats.summary
+
+          md = []
+          md << "## AISpec Contract Verification Report: `#{contract.name}`"
+          md << ""
+          md << "| Metric | Value |"
+          md << "| ------ | ----- |"
+          md << "| **Provider / Model** | `#{results[:provider_name]}` / `#{results[:model]}` |"
+          md << "| **Behavior Score** | **#{summary[:success_rate]}%** (Confidence ±#{summary[:confidence_interval]}%) |"
+          md << "| **Assertions Passed** | #{summary[:passed_assertions]} / #{summary[:total_assertions]} |"
+          md << "| **Mean Latency** | #{summary[:mean_latency]}ms |"
+          md << "| **Total Cost** | $#{sprintf('%.4f', summary[:total_cost])} |"
+          md << "| **Judge Score** | #{summary[:average_judge_score] || 'N/A'} |"
+          md << "| **Variance** | #{summary[:variance]} |"
+          md << "| **Recommendation** | **#{summary[:recommendation]}** |"
+          md << ""
+          md << "### Assertion Breakdown"
+          md << ""
+
+          results[:runs].each_with_index do |run, i|
+            md << "#### Case ##{i + 1}: `#{run[:input]}`"
+            run[:assertion_results].each do |ar|
+              badge = ar.passed? ? "✅ PASS" : "❌ FAIL"
+              md << "- #{badge} **#{ar.name}**: #{ar.message}"
+            end
+            md << ""
+          end
+
+          text = md.join("\n")
+          output_io.puts text
+          text
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Reporters::Base.register("markdown", AISpec::Core::Reporters::Markdown)

data/lib/aispec/core/runner.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    class Runner
+      attr_reader :contract, :provider_override
+
+      def initialize(contract, provider_override: nil)
+        @contract = contract.is_a?(Contract) ? contract : Contract.load_file(contract.to_s)
+        @provider_override = provider_override
+      end
+
+      def run(dataset_item_filter: nil)
+        provider_name = provider_override || contract.provider_name
+        provider_class = Providers::Base.get(provider_name)
+        provider_instance = provider_class.new(model: contract.model)
+
+        items = contract.dataset_items
+        items = [ { input: "Default test input" } ] if items.empty?
+
+        run_results = items.map do |item|
+          execute_item(item, provider_instance)
+        end
+
+        stats = Statistics::Calculator.new(run_results)
+
+        {
+          contract: contract,
+          provider_name: provider_name,
+          model: contract.model,
+          runs: run_results,
+          statistics: stats
+        }
+      end
+
+      private
+
+      def execute_item(item, provider)
+        input_text = item[:input] || item[:prompt] || item.to_s
+        rendered_prompt = render_prompt(contract.prompt_template, item)
+
+        context = {
+          input: input_text,
+          system_prompt: contract.raw_hash[:system_prompt],
+          item: item
+        }
+
+        response = provider.generate(rendered_prompt, system_prompt: context[:system_prompt])
+
+        assertion_results = contract.assertions.map do |assertion_def|
+          type = assertion_def[:type]
+          options = assertion_def[:options] || {}
+          Assertion.evaluate(type, response, context, options)
+        end
+
+        {
+          item: item,
+          input: input_text,
+          prompt: rendered_prompt,
+          response: response,
+          output: response[:output],
+          latency_ms: response[:latency_ms],
+          cost: response[:cost],
+          tokens: response[:tokens],
+          assertion_results: assertion_results,
+          passed: assertion_results.all?(&:passed?)
+        }
+      end
+
+      def render_prompt(template, item)
+        result = template.dup
+        if item.is_a?(Hash)
+          item.each do |k, v|
+            result.gsub!("{{#{k}}}", v.to_s)
+            result.gsub!("{{ #{k} }}", v.to_s)
+          end
+        end
+        result.gsub!("{{input}}", (item[:input] || item[:prompt] || "").to_s)
+        result
+      end
+    end
+  end
+end