aispec 0.1.0

data/lib/aispec/core/statistics/calculator.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module AISpec
+  module Core
+    module Statistics
+      class Calculator
+        attr_reader :results
+
+        def initialize(results)
+          @results = results || []
+        end
+
+        def total_runs
+          @results.size
+        end
+
+        def total_assertions
+          @results.sum { |r| r[:assertion_results].size }
+        end
+
+        def passed_assertions
+          @results.sum { |r| r[:assertion_results].count(&:passed?) }
+        end
+
+        def failed_assertions
+          total_assertions - passed_assertions
+        end
+
+        def success_rate
+          return 0.0 if total_assertions.zero?
+          (passed_assertions.to_f / total_assertions * 100.0).round(1)
+        end
+
+        def confidence_interval
+          return 0.0 if total_assertions.zero?
+
+          # Wilson score interval z = 1.96 for 95% confidence
+          p = passed_assertions.to_f / total_assertions
+          n = total_assertions.to_f
+          z = 1.96
+
+          margin = z * Math.sqrt((p * (1 - p) + (z**2 / (4 * n))) / n) / (1 + (z**2 / n))
+          (margin * 100.0).round(1)
+        end
+
+        def mean_latency
+          return 0.0 if @results.empty?
+          (@results.sum { |r| r[:latency_ms].to_f } / @results.size).round(1)
+        end
+
+        def mean_cost
+          return 0.0 if @results.empty?
+          @results.sum { |r| r[:cost].to_f } / @results.size
+        end
+
+        def total_cost
+          @results.sum { |r| r[:cost].to_f }
+        end
+
+        def judge_scores
+          @results.flat_map do |r|
+            r[:assertion_results].select { |ar| ar.name == :judge && ar.metadata[:score] }.map { |ar| ar.metadata[:score].to_f }
+          end
+        end
+
+        def average_judge_score
+          scores = judge_scores
+          return nil if scores.empty?
+          (scores.sum / scores.size).round(2)
+        end
+
+        def variance_rating
+          scores = judge_scores
+          return "N/A" if scores.size < 2
+
+          mean = scores.sum / scores.size
+          var = scores.sum { |s| (s - mean)**2 } / scores.size
+
+          if var < 0.01
+            "Low"
+          elsif var < 0.05
+            "Moderate"
+          else
+            "High"
+          end
+        end
+
+        def recommendation
+          rate = success_rate
+          var = variance_rating
+
+          if rate >= 95.0 && var != "High"
+            "Stable"
+          elsif rate >= 80.0 || var == "Moderate"
+            "Needs Attention (Flaky)"
+          else
+            "Critical Failure"
+          end
+        end
+
+        def summary
+          {
+            total_runs: total_runs,
+            total_assertions: total_assertions,
+            passed_assertions: passed_assertions,
+            failed_assertions: failed_assertions,
+            success_rate: success_rate,
+            confidence_interval: confidence_interval,
+            mean_latency: mean_latency,
+            mean_cost: mean_cost,
+            total_cost: total_cost,
+            average_judge_score: average_judge_score,
+            variance: variance_rating,
+            recommendation: recommendation
+          }
+        end
+      end
+    end
+  end
+end

data/lib/aispec/plugin.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module AISpec
+  class PluginDSL
+    def assertion(name, &block)
+      AISpec::Core::Assertion.register(name, &block)
+    end
+
+    def provider(name, provider_class)
+      AISpec::Core::Providers::Base.register(name, provider_class)
+    end
+
+    def reporter(name, reporter_class)
+      AISpec::Core::Reporters::Base.register(name, reporter_class)
+    end
+  end
+
+  class << self
+    def plugin(&block)
+      dsl = PluginDSL.new
+      dsl.instance_eval(&block) if block_given?
+    end
+  end
+end

data/lib/aispec/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module AISpec
+  VERSION = "0.1.0"
+end

data/lib/aispec.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "aispec/version"
+require_relative "aispec/configuration"
+require_relative "aispec/plugin"
+
+require_relative "aispec/core/contract"
+require_relative "aispec/core/assertion"
+require_relative "aispec/core/assertions/deterministic"
+require_relative "aispec/core/assertions/judge"
+
+require_relative "aispec/core/providers/base"
+require_relative "aispec/core/providers/mock"
+require_relative "aispec/core/providers/openai"
+require_relative "aispec/core/providers/anthropic"
+require_relative "aispec/core/providers/ollama"
+require_relative "aispec/core/providers/open_router"
+require_relative "aispec/core/providers/vllm"
+
+require_relative "aispec/core/statistics/calculator"
+require_relative "aispec/core/runner"
+
+require_relative "aispec/core/reporters/base"
+require_relative "aispec/core/reporters/console"
+require_relative "aispec/core/reporters/json"
+require_relative "aispec/core/reporters/markdown"
+require_relative "aispec/core/reporters/junit"
+require_relative "aispec/core/reporters/html"
+
+require_relative "aispec/cli"
+
+module AISpec
+  class Error < StandardError; end
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: aispec
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Wilbur Suero
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+description: AISpec provides RSpec-like behavioral contract definition, execution,
+  and verification across AI providers for testing, CI, and production.
+email:
+- wilbur.himself@gmail.com
+executables:
+- aispec
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- bin/aispec
+- lib/aispec.rb
+- lib/aispec/cli.rb
+- lib/aispec/configuration.rb
+- lib/aispec/core/assertion.rb
+- lib/aispec/core/assertions/deterministic.rb
+- lib/aispec/core/assertions/judge.rb
+- lib/aispec/core/contract.rb
+- lib/aispec/core/providers/anthropic.rb
+- lib/aispec/core/providers/base.rb
+- lib/aispec/core/providers/mock.rb
+- lib/aispec/core/providers/ollama.rb
+- lib/aispec/core/providers/open_router.rb
+- lib/aispec/core/providers/openai.rb
+- lib/aispec/core/providers/vllm.rb
+- lib/aispec/core/reporters/base.rb
+- lib/aispec/core/reporters/console.rb
+- lib/aispec/core/reporters/html.rb
+- lib/aispec/core/reporters/json.rb
+- lib/aispec/core/reporters/junit.rb
+- lib/aispec/core/reporters/markdown.rb
+- lib/aispec/core/runner.rb
+- lib/aispec/core/statistics/calculator.rb
+- lib/aispec/plugin.rb
+- lib/aispec/version.rb
+homepage: https://github.com/wilburhimself/aispec
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/wilburhimself/aispec
+  source_code_uri: https://github.com/wilburhimself/aispec
+  changelog_uri: https://github.com/wilburhimself/aispec/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.7
+specification_version: 4
+summary: The engineering framework for behavioral contracts for LLM applications.
+test_files: []