@ozperium/agentspec 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ozperium
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,170 @@
+# AgentSpec
+
+> Testing framework for AI agents — Jest for non-deterministic AI behavior
+
+AI agents are non-deterministic. When you change a prompt, swap a model, or update a tool, behavior shifts in ways you can't predict. AgentSpec lets you write tests that catch those shifts before they reach production.
+
+## Quick start
+
+```bash
+npm install -g agentspec
+agentspec init
+```
+
+This creates `agentspec.yaml`:
+
+```yaml
+name: "my-agent-tests"
+tests:
+  - name: "handles greeting"
+    input: "hello"
+    expect:
+      contains: "help"
+```
+
+Run tests:
+
+```bash
+agentspec run
+```
+
+## Assertions for non-deterministic output
+
+AgentSpec provides assertions designed for AI output, not just string equality:
+
+```yaml
+tests:
+  - name: "handles expired token"
+    input: "my token expired"
+    expect:
+      contains: "refresh token"
+      not_contains: "I don't know"
+      max_latency_ms: 5000
+
+  - name: "routes billing question"
+    input: "I want a refund"
+    expect:
+      contains_any: ["billing", "refund", "support"]
+      tool_called: "transfer_to_billing"
+
+  - name: "response is semantically on-topic"
+    input: "how do I reset my password?"
+    expect:
+      semantically_similar: "reset password credentials"
+      min_confidence: 0.5
+
+  - name: "JSON output has correct structure"
+    input: "get user profile"
+    expect:
+      json_path: "data.email"
+      json_value: "user@example.com"
+
+  - name: "matches error code pattern"
+    input: "simulate error"
+    expect:
+      regex: "ERR-\\d{5}"
+```
+
+## LLM-as-judge
+
+Use a local model to evaluate response quality — no API costs, full privacy:
+
+```bash
+# Use local Ollama as judge
+agentspec run --judge-endpoint http://127.0.0.1:11434/v1/chat/completions --judge-model qwen2.5:7b
+```
+
+```yaml
+tests:
+  - name: "response is helpful"
+    input: "How do I reset my password?"
+    expect:
+      llm_judge: "The response should explain how to reset a password"
+```
+
+## Behavior diff reports
+
+AgentSpec stores the last passing output per test. When behavior changes, you see exactly what shifted:
+
+```
+⚠️ REGRESSION support agent > handles expired token
+  ⚠  Behavior REGRESSED — test was passing, now failing
+  + added: your, token, has, expired, please, contact, support
+  - removed: you, need, refresh, the, token, settings, security
+```
+
+## Testing real agents
+
+Use `--endpoint` to test any HTTP-accessible agent:
+
+```bash
+agentspec run --endpoint https://my-agent.example.com/chat
+```
+
+AgentSpec POSTs `{input: "..."}` to your endpoint and expects `{output: "..."}` in the response.
+
+## CI/CD integration
+
+```bash
+# Exit code 1 on failure
+agentspec run --ci
+
+# JUnit XML for CI systems
+agentspec run --junit > results.xml
+
+# JSON output
+agentspec run --json
+```
+
+### GitHub Action
+
+```yaml
+- uses: agentspec/action@v1
+  with:
+    test-dir: tests
+    format: junit
+```
+
+## CLI commands
+
+```
+agentspec run [--dir tests] [--ci] [--json] [--junit] [--watch] [--test "pattern"]
+agentspec init
+agentspec list [--dir tests]
+agentspec version
+```
+
+## Why AgentSpec?
+
+| Problem | AgentSpec |
+|---------|-----------|
+| Agent behavior changes with model updates | Regression tests catch the shift |
+| "It worked yesterday" debugging | Diff reports show what changed |
+| Manual testing before each deploy | CI/CD gates with `--ci` exit codes |
+| Jest/Vitest don't handle non-deterministic output | `contains_any`, `regex`, `llm_judge`, `semantically_similar` |
+| No CI integration for AI features | GitHub Action + JUnit XML |
+
+## Roadmap
+
+- [x] YAML test definitions
+- [x] Core assertions (contains, not_contains, contains_any, contains_all, regex)
+- [x] Metadata assertions (max_latency_ms, max_tokens, tool_called)
+- [x] Semantic similarity (word overlap)
+- [x] JSON path assertions
+- [x] CLI with run/init/list commands
+- [x] CI mode with exit codes
+- [x] JUnit XML and JSON output
+- [x] Test filtering
+- [x] GitHub Action
+- [x] **Behavior diff reports** — see what changed between runs
+- [x] **LLM-as-judge** — use a local model (Ollama) to evaluate output quality
+- [x] **HTTP agent adapter** — test real agents via `--endpoint`
+- [ ] Auto-test generation from conversation history
+- [ ] Python agent support
+- [ ] Watch mode with file watcher
+- [ ] Web dashboard
+- [ ] GitHub PR integration (comment on PRs with behavior diffs)
+
+## License
+
+MIT

package/action.yml

@@ -0,0 +1,32 @@
+name: 'AgentSpec — AI Agent Tests'
+description: 'Run AgentSpec tests for your AI agents in CI'
+branding:
+  icon: 'check-circle'
+  color: 'blue'
+inputs:
+  test-dir:
+    description: 'Directory containing test suites'
+    required: false
+    default: 'tests'
+  format:
+    description: 'Output format (text, json, junit)'
+    required: false
+    default: 'junit'
+  fail-on-failure:
+    description: 'Exit with error code on test failures'
+    required: false
+    default: 'true'
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+    - name: Install AgentSpec
+      shell: bash
+      run: npm install -g agentspec
+    - name: Run AgentSpec tests
+      shell: bash
+      run: agentspec run --dir ${{ inputs.test-dir }} ${{ inputs.format == 'junit' && '--junit' || '' }} ${{ inputs.format == 'json' && '--json' || '' }} --ci
+      continue-on-error: ${{ inputs.fail-on-failure != 'true' }}

package/dist/assertions/index.d.ts

@@ -0,0 +1,14 @@
+import { TestExpect, AssertionResult } from '../types';
+import { LLMJudgeConfig } from '../llm-judge';
+export declare function setLLMJudgeConfig(config: LLMJudgeConfig | undefined): void;
+export declare function getLLMJudgeConfig(): LLMJudgeConfig | undefined;
+export declare function evaluateAssertions(output: string, expect: TestExpect, metadata?: {
+    tokens?: number;
+    latencyMs?: number;
+    toolsCalled?: string[];
+}): AssertionResult[];
+export declare function evaluateAssertionsAsync(output: string, expect: TestExpect, input: string, metadata?: {
+    tokens?: number;
+    latencyMs?: number;
+    toolsCalled?: string[];
+}): Promise<AssertionResult[]>;

package/dist/assertions/index.js

@@ -0,0 +1,215 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setLLMJudgeConfig = setLLMJudgeConfig;
+exports.getLLMJudgeConfig = getLLMJudgeConfig;
+exports.evaluateAssertions = evaluateAssertions;
+exports.evaluateAssertionsAsync = evaluateAssertionsAsync;
+const llm_judge_1 = require("../llm-judge");
+// Global LLM judge config (can be set via CLI or env)
+let globalLLMJudgeConfig;
+function setLLMJudgeConfig(config) {
+    globalLLMJudgeConfig = config;
+}
+function getLLMJudgeConfig() {
+    return globalLLMJudgeConfig;
+}
+function evaluateAssertions(output, expect, metadata) {
+    const results = [];
+    // Synchronous assertions only — llm_judge handled separately in evaluateAssertionsAsync
+    if (expect.contains !== undefined) {
+        const passed = output.includes(expect.contains);
+        results.push({
+            assertion: 'contains',
+            passed,
+            message: passed ? `Output contains "${expect.contains}"` : `Output does not contain "${expect.contains}"`,
+            expected: expect.contains,
+            actual: passed ? 'found' : 'not found',
+        });
+    }
+    if (expect.not_contains !== undefined) {
+        const passed = !output.includes(expect.not_contains);
+        results.push({
+            assertion: 'not_contains',
+            passed,
+            message: passed ? `Output does not contain "${expect.not_contains}"` : `Output contains "${expect.not_contains}"`,
+            expected: `not "${expect.not_contains}"`,
+            actual: passed ? 'absent' : 'present',
+        });
+    }
+    if (expect.contains_any !== undefined) {
+        const found = expect.contains_any.find(s => output.includes(s));
+        const passed = !!found;
+        results.push({
+            assertion: 'contains_any',
+            passed,
+            message: passed ? `Output contains one of: "${found}"` : `Output contains none of: ${JSON.stringify(expect.contains_any)}`,
+            expected: JSON.stringify(expect.contains_any),
+            actual: found || 'none matched',
+        });
+    }
+    if (expect.contains_all !== undefined) {
+        const missing = expect.contains_all.filter(s => !output.includes(s));
+        const passed = missing.length === 0;
+        results.push({
+            assertion: 'contains_all',
+            passed,
+            message: passed ? `Output contains all required strings` : `Missing: ${missing.join(', ')}`,
+            expected: JSON.stringify(expect.contains_all),
+            actual: passed ? 'all found' : `missing: ${missing.join(', ')}`,
+        });
+    }
+    if (expect.regex !== undefined) {
+        try {
+            const re = new RegExp(expect.regex);
+            const passed = re.test(output);
+            results.push({
+                assertion: 'regex',
+                passed,
+                message: passed ? `Output matches /${expect.regex}/` : `Output does not match /${expect.regex}/`,
+                expected: `/${expect.regex}/`,
+                actual: passed ? 'matched' : 'no match',
+            });
+        }
+        catch (e) {
+            results.push({
+                assertion: 'regex',
+                passed: false,
+                message: `Invalid regex: ${expect.regex}`,
+                expected: expect.regex,
+                actual: 'invalid pattern',
+            });
+        }
+    }
+    if (expect.max_latency_ms !== undefined && metadata?.latencyMs !== undefined) {
+        const passed = metadata.latencyMs <= expect.max_latency_ms;
+        results.push({
+            assertion: 'max_latency_ms',
+            passed,
+            message: passed ? `Latency ${metadata.latencyMs}ms ≤ ${expect.max_latency_ms}ms` : `Latency ${metadata.latencyMs}ms > ${expect.max_latency_ms}ms`,
+            expected: `≤ ${expect.max_latency_ms}ms`,
+            actual: `${metadata.latencyMs}ms`,
+        });
+    }
+    if (expect.max_tokens !== undefined && metadata?.tokens !== undefined) {
+        const passed = metadata.tokens <= expect.max_tokens;
+        results.push({
+            assertion: 'max_tokens',
+            passed,
+            message: passed ? `Tokens ${metadata.tokens} ≤ ${expect.max_tokens}` : `Tokens ${metadata.tokens} > ${expect.max_tokens}`,
+            expected: `≤ ${expect.max_tokens}`,
+            actual: `${metadata.tokens}`,
+        });
+    }
+    if (expect.tool_called !== undefined && metadata?.toolsCalled !== undefined) {
+        const passed = metadata.toolsCalled.includes(expect.tool_called);
+        results.push({
+            assertion: 'tool_called',
+            passed,
+            message: passed ? `Tool "${expect.tool_called}" was called` : `Tool "${expect.tool_called}" was not called. Called: ${metadata.toolsCalled.join(', ') || 'none'}`,
+            expected: expect.tool_called,
+            actual: passed ? 'called' : `not called (called: ${metadata.toolsCalled.join(', ') || 'none'})`,
+        });
+    }
+    // json_path assertion: extract value from JSON output and compare
+    if (expect.json_path !== undefined) {
+        try {
+            const json = JSON.parse(output);
+            const value = extractJsonPath(json, expect.json_path);
+            if (expect.json_value !== undefined) {
+                const passed = JSON.stringify(value) === JSON.stringify(expect.json_value);
+                results.push({
+                    assertion: 'json_path',
+                    passed,
+                    message: passed ? `json_path "${expect.json_path}" equals expected value` : `json_path "${expect.json_path}" mismatch`,
+                    expected: JSON.stringify(expect.json_value),
+                    actual: JSON.stringify(value),
+                });
+            }
+            else {
+                const passed = value !== undefined;
+                results.push({
+                    assertion: 'json_path',
+                    passed,
+                    message: passed ? `json_path "${expect.json_path}" found` : `json_path "${expect.json_path}" not found`,
+                    expected: expect.json_path,
+                    actual: JSON.stringify(value),
+                });
+            }
+        }
+        catch (e) {
+            results.push({
+                assertion: 'json_path',
+                passed: false,
+                message: `Output is not valid JSON or path not found: ${e instanceof Error ? e.message : String(e)}`,
+                expected: expect.json_path,
+                actual: 'parse error',
+            });
+        }
+    }
+    // semantically_similar: check if output is semantically similar to expected (using simple word overlap as MVP)
+    if (expect.semantically_similar !== undefined) {
+        const similarity = wordSimilarity(output, expect.semantically_similar);
+        const threshold = expect.min_confidence ?? 0.6;
+        const passed = similarity >= threshold;
+        results.push({
+            assertion: 'semantically_similar',
+            passed,
+            message: passed ? `Similarity ${similarity.toFixed(2)} ≥ ${threshold}` : `Similarity ${similarity.toFixed(2)} < ${threshold}`,
+            expected: `≥ ${threshold} similarity to "${expect.semantically_similar}"`,
+            actual: similarity.toFixed(2),
+        });
+    }
+    return results;
+}
+// Async version that supports llm_judge assertion
+async function evaluateAssertionsAsync(output, expect, input, metadata) {
+    const results = evaluateAssertions(output, expect, metadata);
+    if (expect.llm_judge !== undefined) {
+        try {
+            const judgeResult = await (0, llm_judge_1.llmJudge)(input, output, expect.llm_judge, globalLLMJudgeConfig);
+            results.push({
+                assertion: 'llm_judge',
+                passed: judgeResult.passed,
+                message: judgeResult.passed
+                    ? `LLM judge passed (score: ${judgeResult.score.toFixed(2)}): ${judgeResult.reasoning}`
+                    : `LLM judge failed (score: ${judgeResult.score.toFixed(2)}): ${judgeResult.reasoning}`,
+                expected: expect.llm_judge,
+                actual: `score: ${judgeResult.score.toFixed(2)}`,
+            });
+        }
+        catch (e) {
+            results.push({
+                assertion: 'llm_judge',
+                passed: false,
+                message: `LLM judge error: ${e instanceof Error ? e.message : String(e)}`,
+                expected: expect.llm_judge,
+                actual: 'error',
+            });
+        }
+    }
+    return results;
+}
+function extractJsonPath(obj, path) {
+    const parts = path.split('.');
+    let current = obj;
+    for (const part of parts) {
+        if (current === null || current === undefined)
+            return undefined;
+        if (typeof current !== 'object')
+            return undefined;
+        current = current[part];
+    }
+    return current;
+}
+function wordSimilarity(a, b) {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(w => w.length > 2));
+    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(w => w.length > 2));
+    if (wordsA.size === 0 || wordsB.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const w of Array.from(wordsA)) {
+        if (wordsB.has(w))
+            intersection++;
+    }
+    return (2 * intersection) / (wordsA.size + wordsB.size);
+}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const runner_1 = require("./runner");
+const loader_1 = require("./loader");
+const reporter_1 = require("./reporter");
+const diff_1 = require("./diff");
+const path = __importStar(require("path"));
+async function main() {
+    const args = process.argv.slice(2);
+    const command = args[0];
+    if (command === 'run') {
+        let dir = 'tests';
+        let ci = false;
+        let json = false;
+        let junit = false;
+        let watch = false;
+        let testFilter = null;
+        let judgeEndpoint = null;
+        let judgeModel = null;
+        let agentEndpoint = null;
+        for (let i = 1; i < args.length; i++) {
+            const arg = args[i];
+            if (arg === '--dir')
+                dir = args[++i];
+            else if (arg === '--ci')
+                ci = true;
+            else if (arg === '--json')
+                json = true;
+            else if (arg === '--junit')
+                junit = true;
+            else if (arg === '--watch')
+                watch = true;
+            else if (arg === '--test')
+                testFilter = args[++i];
+            else if (arg === '--judge-endpoint')
+                judgeEndpoint = args[++i];
+            else if (arg === '--judge-model')
+                judgeModel = args[++i];
+            else if (arg === '--endpoint')
+                agentEndpoint = args[++i];
+            else if (arg === '--help' || arg === '-h') {
+                printHelp();
+                return;
+            }
+        }
+        // Use HTTP agent if endpoint specified, otherwise mock
+        const isSelfTest = dir === 'tests';
+        const agent = agentEndpoint
+            ? new (require('./http-agent').HTTPAgent)({ endpoint: agentEndpoint })
+            : isSelfTest
+                ? new runner_1.MockAgent({}, true)
+                : new runner_1.MockAgent({
+                    default: "I'll help you with that. Let me check the system for you.",
+                    "my token expired, what do I do?": "You need to refresh the token. Go to Settings > Security to get a new refresh token.",
+                    "I want a refund": "I'll transfer you to the billing team. Let me connect you with our support team.",
+                    "what can you do?": "I can help with auth, billing, and general support questions.",
+                    "xyzzy random input 12345": "I'm not sure about that, but I can help you find what you need.",
+                });
+        // Configure LLM judge if specified (must be before running tests)
+        if (judgeEndpoint || judgeModel) {
+            const { setLLMJudgeConfig } = await Promise.resolve().then(() => __importStar(require('./assertions')));
+            setLLMJudgeConfig({
+                endpoint: judgeEndpoint || undefined,
+                model: judgeModel || undefined,
+            });
+        }
+        const result = await (0, runner_1.runAll)(dir, agent, testFilter);
+        // Compute behavior diffs
+        const diffs = (0, diff_1.computeDiffs)(result.results);
+        // Save current run for future diffs
+        (0, diff_1.saveRun)(result.results);
+        if (json) {
+            console.log((0, reporter_1.formatResultsJSON)(result));
+        }
+        else if (junit) {
+            console.log((0, reporter_1.formatResultsJUnit)(result));
+        }
+        else {
+            console.log((0, reporter_1.formatResults)(result));
+            // Show diff report if there are changes
+            if (diffs.length > 0) {
+                console.log((0, diff_1.formatDiffs)(diffs));
+            }
+        }
+        if (ci) {
+            process.exit(result.failed > 0 ? 1 : 0);
+        }
+    }
+    else if (command === 'init') {
+        console.log('Creating agentspec.yaml...');
+        const { writeFileSync } = await Promise.resolve().then(() => __importStar(require('fs')));
+        writeFileSync(path.join(process.cwd(), 'agentspec.yaml'), `name: "my-agent-tests"
+description: "Tests for my AI agent"
+tests:
+  - name: "handles greeting"
+    input: "hello"
+    expect:
+      contains: "help"
+`);
+        console.log('Created agentspec.yaml');
+    }
+    else if (command === 'list') {
+        let dir = 'tests';
+        if (args[1] === '--dir')
+            dir = args[2];
+        const suites = (0, loader_1.findTestSuites)(dir);
+        if (suites.length === 0) {
+            console.log('No test suites found in', dir);
+            return;
+        }
+        console.log(`Found ${suites.length} test suite(s):`);
+        for (const s of suites) {
+            console.log(`  ${s}`);
+        }
+    }
+    else if (command === 'version' || command === '--version' || command === '-v') {
+        console.log('agentspec v0.1.0');
+    }
+    else {
+        printHelp();
+    }
+}
+function printHelp() {
+    console.log(`
+AgentSpec — Testing framework for AI agents
+
+Usage:
+  agentspec run [options]     Run tests
+  agentspec init              Create agentspec.yaml
+  agentspec list              List test suites
+  agentspec version           Show version
+
+Options for run:
+  --dir <path>          Test directory (default: tests)
+  --ci                  Exit with code 1 on failure
+  --json                Output results as JSON
+  --junit               Output JUnit XML
+  --watch               Watch for changes and re-run
+  --test <pattern>      Run only matching tests
+  --judge-endpoint <url>  LLM judge endpoint (OpenAI-compatible, default: Ollama)
+  --judge-model <name>    LLM judge model name (default: local small model)
+`);
+}
+main().catch(err => {
+    console.error('Error:', err.message);
+    process.exit(1);
+});

package/dist/diff.d.ts

@@ -0,0 +1,7 @@
+import { TestResult } from './types';
+import { StoredRun, DiffEntry } from './types/diff';
+export declare function ensureStoreDir(): string;
+export declare function saveRun(results: TestResult[]): void;
+export declare function loadPreviousRun(suite: string, test: string): StoredRun | null;
+export declare function computeDiffs(currentResults: TestResult[]): DiffEntry[];
+export declare function formatDiffs(diffs: DiffEntry[]): string;