aispec 0.1.0

data/lib/aispec/core/assertions/deterministic.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "json"
+begin
+  require "rexml/document"
+rescue LoadError
+  # REXML fallback handled in valid_xml assertion
+end
+
+module AISpec
+  module Core
+    module Assertions
+      module Deterministic
+        def self.register_all!
+          # Contains
+          Assertion.register(:contains) do |response, _context, options|
+            expected = options[:value] || options[:target] || options[:raw]
+            text = response[:output].to_s
+            match = text.include?(expected.to_s)
+            [match, match ? "Output contains '#{expected}'" : "Output does not contain '#{expected}'"]
+          end
+
+          # Not Contains
+          Assertion.register(:not_contains) do |response, _context, options|
+            expected = options[:value] || options[:target]
+            text = response[:output].to_s
+            match = !text.include?(expected.to_s)
+            [match, match ? "Output does not contain '#{expected}'" : "Output contains forbidden text '#{expected}'"]
+          end
+
+          # Starts With
+          Assertion.register(:starts_with) do |response, _context, options|
+            prefix = options[:value] || options[:target]
+            text = response[:output].to_s.strip
+            match = text.start_with?(prefix.to_s)
+            [match, match ? "Output starts with '#{prefix}'" : "Output does not start with '#{prefix}'"]
+          end
+
+          # Ends With
+          Assertion.register(:ends_with) do |response, _context, options|
+            suffix = options[:value] || options[:target]
+            text = response[:output].to_s.strip
+            match = text.end_with?(suffix.to_s)
+            [match, match ? "Output ends with '#{suffix}'" : "Output does not end with '#{suffix}'"]
+          end
+
+          # Regex
+          Assertion.register(:regex) do |response, _context, options|
+            pattern = options[:pattern] || options[:value] || options[:regex]
+            regex = pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern.to_s)
+            text = response[:output].to_s
+            match = !!(text =~ regex)
+            [match, match ? "Output matches regex /#{pattern}/" : "Output failed to match regex /#{pattern}/"]
+          end
+
+          # JSON
+          Assertion.register(:json) do |response, _context, options|
+            text = response[:output].to_s.strip
+            # Clean markdown codeblocks if present
+            cleaned = text.sub(/^```json\s*/i, "").sub(/^```\s*/, "").sub(/\s*```$/, "").strip
+            cleaned = $1 if cleaned =~ /(\{[\s\S]*\}|\[[\s\S]*\])/
+
+            begin
+              parsed = JSON.parse(cleaned)
+              if options[:schema]
+                # If schema is given, verify top level key requirements if simple hash
+                if options[:schema].is_a?(Array)
+                  missing_keys = options[:schema].map(&:to_s) - parsed.keys.map(&:to_s)
+                  [missing_keys.empty?, missing_keys.empty? ? "JSON satisfies schema keys" : "JSON missing schema keys: #{missing_keys.join(', ')}"]
+                else
+                  [true, "Valid JSON"]
+                end
+              else
+                [true, "Output is valid JSON"]
+              end
+            rescue JSON::ParserError => e
+              [false, "Output is not valid JSON: #{e.message}"]
+            end
+          end
+
+          # Valid Markdown
+          Assertion.register(:valid_markdown) do |response, _context, _options|
+            text = response[:output].to_s
+            # Basic markdown check (unclosed code fences detection)
+            fence_count = text.scan(/^```/).length
+            valid = fence_count.even?
+            [valid, valid ? "Valid Markdown structural formatting" : "Invalid Markdown (unmatched code block fences)"]
+          end
+
+          # Valid XML
+          Assertion.register(:valid_xml) do |response, _context, _options|
+            text = response[:output].to_s.strip
+            if defined?(REXML::Document)
+              begin
+                REXML::Document.new(text)
+                [true, "Output is valid XML"]
+              rescue REXML::ParseException => e
+                [false, "Invalid XML: #{e.message}"]
+              end
+            else
+              valid = text =~ /\A\s*<[a-zA-Z_][\s\S]*>\s*\z/
+              [!!valid, valid ? "Output is valid XML structure" : "Invalid XML structure"]
+            end
+          end
+
+          # Max Latency
+          Assertion.register(:max_latency) do |response, _context, options|
+            limit = (options[:value] || options[:max] || options[:limit] || 5000).to_f
+            actual = (response[:latency_ms] || 0.0).to_f
+            passed = actual <= limit
+            [passed, passed ? "Latency #{actual.round(1)}ms <= #{limit.round(1)}ms" : "Latency #{actual.round(1)}ms exceeded max #{limit.round(1)}ms"]
+          end
+
+          # Cost
+          Assertion.register(:cost) do |response, _context, options|
+            max_cost = (options[:max] || options[:value] || 0.05).to_f
+            actual_cost = (response[:cost] || 0.0).to_f
+            passed = actual_cost <= max_cost
+            [passed, passed ? "Cost $#{sprintf('%.4f', actual_cost)} <= $#{sprintf('%.4f', max_cost)}" : "Cost $#{sprintf('%.4f', actual_cost)} exceeded max $#{sprintf('%.4f', max_cost)}"]
+          end
+
+          # Must Not Reveal System Prompt
+          Assertion.register(:must_not_reveal_system_prompt) do |response, context, _options|
+            sys_prompt = context[:system_prompt].to_s.strip
+            text = response[:output].to_s
+            if sys_prompt.empty?
+              [true, "No system prompt leak detected"]
+            else
+              match = text.include?(sys_prompt)
+              [!match, match ? "System prompt leaked in output" : "No system prompt leak detected"]
+            end
+          end
+
+          # Tool Called
+          Assertion.register(:tool_called) do |response, _context, options|
+            expected = (options[:value] || options[:name] || options[:tool]).to_s
+            tools = Array(response[:tools_called]).map(&:to_s)
+            called = tools.include?(expected)
+            [called, called ? "Tool '#{expected}' was called" : "Tool '#{expected}' was not called"]
+          end
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Assertions::Deterministic.register_all!

data/lib/aispec/core/assertions/judge.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Assertions
+      module Judge
+        def self.register!
+          Assertion.register(:judge) do |response, context, options|
+            judge_prompt = options[:prompt] || options[:criteria] || "Evaluate if the response fulfills the contract requirements."
+            threshold = (options[:threshold] || options[:min_score] || 0.80).to_f
+
+            target_output = response[:output].to_s
+            input_context = context[:input] ? "Input: #{context[:input]}\n" : ""
+
+            evaluation_instructions = <<~SYSTEM
+              You are an impartial AI behavioral evaluator judge.
+              Evaluate the following AI response against the criteria below.
+
+              Criteria:
+              #{judge_prompt}
+
+              #{input_context}
+              AI Response to evaluate:
+              \"\"\"
+              #{target_output}
+              \"\"\"
+
+              Respond ONLY in valid JSON format with:
+              {
+                "score": <float between 0.0 and 1.0>,
+                "reason": "<brief justification>"
+              }
+            SYSTEM
+
+            judge_provider_name = options[:provider] || AISpec.configuration.judge_provider
+            judge_model = options[:model] || AISpec.configuration.judge_model
+
+            provider = Providers::Base.get(judge_provider_name).new(model: judge_model)
+            eval_result = provider.generate(evaluation_instructions)
+
+            begin
+              cleaned = eval_result[:output].to_s.sub(/^```json\s*/i, "").sub(/^```\s*/, "").sub(/\s*```$/, "").strip
+              parsed = JSON.parse(cleaned)
+              score = (parsed["score"] || parsed[:score] || 0.0).to_f
+              reason = parsed["reason"] || parsed[:reason] || "No explanation provided"
+              passed = score >= threshold
+
+              [
+                passed,
+                "Judge score #{sprintf('%.2f', score)} (threshold #{threshold}): #{reason}",
+                { score: score, threshold: threshold, reason: reason }
+              ]
+            rescue JSON::ParserError => _e
+              # Heuristic fallback if judge didn't output strict JSON
+              match = eval_result[:output].to_s.match(/(?:score|rating)?\s*[:=]?\s*(0\.\d+|1\.0|\d+)/i)
+              score = match ? match[1].to_f : 0.5
+              score = score / 10.0 if score > 1.0
+              passed = score >= threshold
+              [
+                passed,
+                "Judge score #{sprintf('%.2f', score)} (threshold #{threshold})",
+                { score: score, threshold: threshold }
+              ]
+            end
+          end
+
+          # Alias for confidence / semantic quality
+          Assertion.register(:confidence) do |response, context, options|
+            threshold = (options[:min] || options[:threshold] || 0.8).to_f
+            options_with_judge = options.merge(
+              prompt: "Evaluate confidence and factual accuracy of the response.",
+              threshold: threshold
+            )
+            Assertion.evaluate(:judge, response, context, options_with_judge)
+          end
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Assertions::Judge.register!

data/lib/aispec/core/contract.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module AISpec
+  module Core
+    class Contract
+      attr_reader :name, :model, :provider_name, :prompt_template, :dataset_path, :dataset_items, :assertions, :raw_hash, :file_path
+
+      def self.load_file(path)
+        raise ArgumentError, "Contract file not found: #{path}" unless File.exist?(path)
+
+        hash = YAML.safe_load(File.read(path), symbolize_names: true, aliases: true) || {}
+        new(hash, file_path: path)
+      end
+
+      def self.from_hash(hash)
+        new(hash)
+      end
+
+      def initialize(hash, file_path: nil)
+        @raw_hash = hash
+        @file_path = file_path
+        @name = hash[:name] || (file_path ? File.basename(file_path, ".*") : "unnamed-contract")
+        
+        model_config = hash[:model] || {}
+        if model_config.is_a?(Hash)
+          @provider_name = (model_config[:provider] || AISpec.configuration.default_provider).to_s
+          @model = (model_config[:model] || AISpec.configuration.default_model).to_s
+        else
+          @provider_name = AISpec.configuration.default_provider.to_s
+          @model = model_config.to_s
+        end
+
+        @prompt_template = hash[:prompt] || "{{input}}"
+        
+        dataset_config = hash[:dataset]
+        if dataset_config.is_a?(Hash)
+          @dataset_path = dataset_config[:path]
+          @dataset_items = dataset_config[:items] || []
+        elsif dataset_config.is_a?(Array)
+          @dataset_items = dataset_config
+        else
+          @dataset_items = []
+        end
+
+        load_external_dataset! if @dataset_path && @file_path
+
+        raw_contracts = hash[:contracts] || hash[:assertions] || []
+        @assertions = parse_assertions(raw_contracts)
+      end
+
+      private
+
+      def load_external_dataset!
+        full_path = File.expand_path(@dataset_path, File.dirname(@file_path))
+        return unless File.exist?(full_path)
+
+        data = YAML.safe_load(File.read(full_path), symbolize_names: true, aliases: true)
+        if data.is_a?(Array)
+          @dataset_items += data
+        elsif data.is_a?(Hash) && data[:items].is_a?(Array)
+          @dataset_items += data[:items]
+        end
+      end
+
+      def parse_assertions(raw_contracts)
+        raw_contracts.map do |item|
+          normalize_assertion_definition(item)
+        end.compact
+      end
+
+      def normalize_assertion_definition(item)
+        if item.is_a?(Hash)
+          type = (item[:type] || item[:name] || item[:assertion]).to_sym
+          { type: type, options: item }
+        elsif item.is_a?(String)
+          parse_string_assertion(item)
+        elsif item.is_a?(Symbol)
+          { type: item, options: {} }
+        end
+      end
+
+      def parse_string_assertion(str)
+        s = str.strip
+        if s =~ /^(latency|max_latency)\s*(<|<=)\s*(\d+(?:\.\d+)?)\s*(ms|s)?$/i
+          val = $3.to_f
+          val *= 1000 if $4.to_s.downcase == "s"
+          { type: :max_latency, options: { value: val } }
+        elsif s =~ /^cost\s*(<|<=)\s*\$?(\d+(?:\.\d+)?)$/i
+          { type: :cost, options: { max: $2.to_f } }
+        elsif s =~ /^confidence\s*(>=|>)\s*(\d+(?:\.\d+)?)$/i
+          { type: :confidence, options: { min: $2.to_f } }
+        elsif s =~ /^must_return_valid_json$/i || s == "json"
+          { type: :json, options: {} }
+        elsif s =~ /^must_cite_sources$/i
+          { type: :citation_format, options: {} }
+        else
+          # Fallback to symbol type
+          { type: s.to_sym, options: { raw: str } }
+        end
+      end
+    end
+  end
+end

data/lib/aispec/core/providers/anthropic.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module AISpec
+  module Core
+    module Providers
+      class Anthropic < Base
+        BASE_URI = "https://api.anthropic.com/v1/messages"
+
+        def initialize(model: "claude-3-5-sonnet-20241022", api_key: nil, **options)
+          super(model: model, **options)
+          @api_key = api_key || ENV["ANTHROPIC_API_KEY"]
+        end
+
+        def generate(prompt, system_prompt: nil, temperature: 0.7, max_tokens: 1024, **_params)
+          return fallback_mock_if_no_key(prompt) unless @api_key && !@api_key.empty?
+
+          payload = {
+            model: @model || "claude-3-5-sonnet-20241022",
+            max_tokens: max_tokens,
+            messages: [{ role: "user", content: prompt }],
+            temperature: temperature
+          }
+          payload[:system] = system_prompt if system_prompt
+
+          uri = URI.parse(BASE_URI)
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          req = Net::HTTP::Post.new(uri.path, {
+            "Content-Type" => "application/json",
+            "x-api-key" => @api_key,
+            "anthropic-version" => "2023-06-01"
+          })
+          req.body = JSON.generate(payload)
+
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.read_timeout = AISpec.configuration.timeout
+
+          res = http.request(req)
+          elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+
+          if res.is_a?(Net::HTTPSuccess)
+            data = JSON.parse(res.body)
+            output = data.dig("content", 0, "text") || ""
+            input_tokens = data.dig("usage", "input_tokens") || 0
+            output_tokens = data.dig("usage", "output_tokens") || 0
+            total_tokens = input_tokens + output_tokens
+
+            cost = (input_tokens * 0.000003) + (output_tokens * 0.000015)
+
+            build_response(
+              output: output,
+              latency_ms: elapsed_ms,
+              tokens: total_tokens,
+              cost: cost,
+              raw: data
+            )
+          else
+            raise "Anthropic API Error (#{res.code}): #{res.body}"
+          end
+        end
+
+        private
+
+        def fallback_mock_if_no_key(prompt)
+          Mock.new(model: @model).generate(prompt)
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Providers::Base.register("anthropic", AISpec::Core::Providers::Anthropic)

data/lib/aispec/core/providers/base.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Providers
+      class Base
+        @registry = {}
+
+        class << self
+          attr_reader :registry
+
+          def register(name, provider_class)
+            @registry[name.to_s.downcase] = provider_class
+          end
+
+          def get(name)
+            key = name.to_s.downcase
+            @registry[key] || Mock
+          end
+        end
+
+        attr_reader :model, :options
+
+        def initialize(model: nil, **options)
+          @model = model
+          @options = options
+        end
+
+        def generate(prompt, **params)
+          raise NotImplementedError, "#{self.class.name} must implement #generate"
+        end
+
+        protected
+
+        def build_response(output:, latency_ms:, tokens: 0, cost: 0.0, tools_called: [], raw: {})
+          {
+            output: output,
+            latency_ms: latency_ms,
+            tokens: tokens,
+            cost: cost,
+            tools_called: tools_called,
+            raw: raw
+          }
+        end
+      end
+    end
+  end
+end

data/lib/aispec/core/providers/mock.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Providers
+      class Mock < Base
+        attr_accessor :canned_response, :canned_latency_ms, :canned_cost, :canned_tools
+
+        def initialize(model: "mock-model", canned_response: nil, canned_latency_ms: 120.0, canned_cost: 0.001, canned_tools: [], **options)
+          super(model: model, **options)
+          @canned_response = canned_response
+          @canned_latency_ms = canned_latency_ms
+          @canned_cost = canned_cost
+          @canned_tools = canned_tools
+        end
+
+        def generate(prompt, **params)
+          output = @canned_response || params[:canned_response] || default_mock_output(prompt)
+          latency = params[:latency_ms] || @canned_latency_ms
+          cost = params[:cost] || @canned_cost
+          tools = params[:tools_called] || @canned_tools
+          tokens = (output.to_s.split.length * 1.3).to_i
+
+          build_response(
+            output: output,
+            latency_ms: latency,
+            tokens: tokens,
+            cost: cost,
+            tools_called: tools,
+            raw: { mock: true }
+          )
+        end
+
+        private
+
+        def default_mock_output(prompt)
+          if prompt =~ /judge|evaluat/i
+            JSON.generate({
+              score: 0.92,
+              reason: "Response is accurate, empathetic, and concise."
+            })
+          elsif prompt =~ /JSON/i
+            JSON.generate({
+              status: "success",
+              message: "Refund processed successfully.",
+              amount: 50.0,
+              sources: ["https://policy.example.com/refunds"]
+            })
+          else
+            "Thank you for contacting support! Your refund of $50.00 has been processed according to our policy [1]. Sources: https://support.example.com/faq"
+          end
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Providers::Base.register("mock", AISpec::Core::Providers::Mock)

data/lib/aispec/core/providers/ollama.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module AISpec
+  module Core
+    module Providers
+      class Ollama < Base
+        def initialize(model: "llama3", host: nil, **options)
+          super(model: model, **options)
+          @host = host || ENV["OLLAMA_HOST"] || "http://localhost:11434"
+        end
+
+        def generate(prompt, system_prompt: nil, **_params)
+          uri = URI.parse("#{@host.sub(/\/$/, '')}/api/generate")
+          
+          payload = {
+            model: @model || "llama3",
+            prompt: prompt,
+            stream: false
+          }
+          payload[:system] = system_prompt if system_prompt
+
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          req = Net::HTTP::Post.new(uri.path, { "Content-Type" => "application/json" })
+          req.body = JSON.generate(payload)
+
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.scheme == "https"
+          http.read_timeout = AISpec.configuration.timeout
+
+          begin
+            res = http.request(req)
+            elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+
+            if res.is_a?(Net::HTTPSuccess)
+              data = JSON.parse(res.body)
+              output = data["response"] || ""
+              eval_count = data["eval_count"] || (output.split.length * 1.3).to_i
+
+              build_response(
+                output: output,
+                latency_ms: elapsed_ms,
+                tokens: eval_count,
+                cost: 0.0, # Local execution cost $0.00
+                raw: data
+              )
+            else
+              raise "Ollama Error (#{res.code}): #{res.body}"
+            end
+          rescue Errno::ECONNREFUSED
+            # Fallback to mock if local Ollama daemon is offline during dev/test
+            Mock.new(model: @model).generate(prompt)
+          end
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Providers::Base.register("ollama", AISpec::Core::Providers::Ollama)

data/lib/aispec/core/providers/open_router.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module AISpec
+  module Core
+    module Providers
+      class OpenRouter < Base
+        BASE_URI = "https://openrouter.ai/api/v1/chat/completions"
+
+        def initialize(model: "openai/gpt-4o-mini", api_key: nil, **options)
+          super(model: model, **options)
+          @api_key = api_key || ENV["OPENROUTER_API_KEY"]
+        end
+
+        def generate(prompt, system_prompt: nil, temperature: 0.7, **_params)
+          return fallback_mock_if_no_key(prompt) unless @api_key && !@api_key.empty?
+
+          messages = []
+          messages << { role: "system", content: system_prompt } if system_prompt
+          messages << { role: "user", content: prompt }
+
+          payload = {
+            model: @model || "openai/gpt-4o-mini",
+            messages: messages,
+            temperature: temperature
+          }
+
+          uri = URI.parse(BASE_URI)
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          req = Net::HTTP::Post.new(uri.path, {
+            "Content-Type" => "application/json",
+            "Authorization" => "Bearer #{@api_key}",
+            "HTTP-Referer" => "https://github.com/wilburhimself/aispec",
+            "X-Title" => "AISpec Framework"
+          })
+          req.body = JSON.generate(payload)
+
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.read_timeout = AISpec.configuration.timeout
+
+          res = http.request(req)
+          elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+
+          if res.is_a?(Net::HTTPSuccess)
+            data = JSON.parse(res.body)
+            output = data.dig("choices", 0, "message", "content") || ""
+            tokens = data.dig("usage", "total_tokens") || 0
+
+            build_response(
+              output: output,
+              latency_ms: elapsed_ms,
+              tokens: tokens,
+              cost: 0.0005,
+              raw: data
+            )
+          else
+            raise "OpenRouter API Error (#{res.code}): #{res.body}"
+          end
+        end
+
+        private
+
+        def fallback_mock_if_no_key(prompt)
+          Mock.new(model: @model).generate(prompt)
+        end
+      end
+    end
+  end
+end
+
+AISpec::Core::Providers::Base.register("openrouter", AISpec::Core::Providers::OpenRouter)