@ozperium/agentspec 0.1.0

package/dist/runner.js

@@ -0,0 +1,95 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MockAgent = void 0;
+exports.runSuite = runSuite;
+exports.runAll = runAll;
+const assertions_1 = require("./assertions");
+const loader_1 = require("./loader");
+async function runSuite(suitePath, agent) {
+    const suite = (0, loader_1.loadTestSuite)(suitePath);
+    const results = [];
+    for (const test of suite.tests) {
+        const startTime = Date.now();
+        try {
+            const output = await agent.run(test.input, suite);
+            const durationMs = Date.now() - startTime;
+            const hasLLMJudge = test.expect.llm_judge !== undefined;
+            const assertions = hasLLMJudge
+                ? await (0, assertions_1.evaluateAssertionsAsync)(output.text, test.expect, test.input, {
+                    tokens: output.tokens,
+                    latencyMs: durationMs,
+                    toolsCalled: output.toolsCalled,
+                })
+                : (0, assertions_1.evaluateAssertions)(output.text, test.expect, {
+                    tokens: output.tokens,
+                    latencyMs: durationMs,
+                    toolsCalled: output.toolsCalled,
+                });
+            const passed = assertions.every(a => a.passed);
+            results.push({
+                test: test.name,
+                suite: suite.name,
+                passed,
+                duration_ms: durationMs,
+                assertions,
+                output: output.text,
+            });
+        }
+        catch (e) {
+            const durationMs = Date.now() - startTime;
+            results.push({
+                test: test.name,
+                suite: suite.name,
+                passed: false,
+                duration_ms: durationMs,
+                assertions: [],
+                error: e instanceof Error ? e.message : String(e),
+            });
+        }
+    }
+    return results;
+}
+async function runAll(dir, agent, filter) {
+    const suites = (0, loader_1.findTestSuites)(dir);
+    const allResults = [];
+    const startTime = Date.now();
+    for (const suitePath of suites) {
+        let suiteResults = await runSuite(suitePath, agent);
+        if (filter) {
+            const re = new RegExp(filter);
+            suiteResults = suiteResults.filter(r => re.test(r.test) || re.test(r.suite));
+        }
+        allResults.push(...suiteResults);
+    }
+    const passed = allResults.filter(r => r.passed).length;
+    const failed = allResults.length - passed;
+    return {
+        total: allResults.length,
+        passed,
+        failed,
+        results: allResults,
+        duration_ms: Date.now() - startTime,
+    };
+}
+// Built-in mock agent for testing AgentSpec itself
+class MockAgent {
+    responses = {};
+    echo = false;
+    constructor(responses, echo) {
+        this.responses = responses || {
+            default: "Hello! I'm a mock agent. I can help with that.",
+        };
+        this.echo = echo || false;
+    }
+    async run(input, suite) {
+        await new Promise(resolve => setTimeout(resolve, 10));
+        const text = this.echo ? input : (this.responses[input] || this.responses.default);
+        return {
+            text,
+            latencyMs: 10,
+            tokens: Math.ceil(text.length / 4),
+            toolsCalled: [],
+        };
+    }
+}
+exports.MockAgent = MockAgent;

package/dist/types/diff.d.ts

@@ -0,0 +1,18 @@
+export interface StoredRun {
+    test: string;
+    suite: string;
+    passed: boolean;
+    output: string;
+    timestamp: string;
+    wake?: string;
+}
+export interface DiffEntry {
+    test: string;
+    suite: string;
+    status: 'regression' | 'improvement' | 'changed' | 'new' | 'removed';
+    old_output: string;
+    new_output: string;
+    old_passed: boolean;
+    new_passed: boolean;
+    summary: string;
+}

package/dist/types/diff.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/types/index.d.ts

@@ -0,0 +1,58 @@
+export interface TestExpect {
+    contains?: string;
+    not_contains?: string;
+    contains_any?: string[];
+    contains_all?: string[];
+    regex?: string;
+    tool_called?: string;
+    max_latency_ms?: number;
+    max_tokens?: number;
+    json_path?: string;
+    json_value?: unknown;
+    llm_judge?: string;
+    semantically_similar?: string;
+    min_confidence?: number;
+    custom?: string;
+}
+export interface TestCase {
+    name: string;
+    input: string;
+    expect: TestExpect;
+    setup?: Record<string, unknown>;
+}
+export interface TestSuite {
+    name: string;
+    description?: string;
+    agent?: AgentConfig;
+    tests: TestCase[];
+}
+export interface AgentConfig {
+    type: string;
+    endpoint?: string;
+    model?: string;
+    system_prompt?: string;
+    tools?: string[];
+}
+export interface AssertionResult {
+    assertion: string;
+    passed: boolean;
+    message: string;
+    expected?: string;
+    actual?: string;
+}
+export interface TestResult {
+    test: string;
+    suite: string;
+    passed: boolean;
+    duration_ms: number;
+    assertions: AssertionResult[];
+    output?: string;
+    error?: string;
+}
+export interface RunResult {
+    total: number;
+    passed: number;
+    failed: number;
+    results: TestResult[];
+    duration_ms: number;
+}

package/dist/types/index.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@ozperium/agentspec",
+  "version": "0.1.0",
+  "description": "Testing framework for AI agents — Jest for non-deterministic AI behavior",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "agentspec": "dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "test": "node dist/cli.js run --dir tests --ci && node dist/cli.js run --dir examples --ci",
+    "prepublishOnly": "tsc"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "ai",
+    "agent",
+    "testing",
+    "regression",
+    "llm",
+    "jest",
+    "framework",
+    "automation",
+    "ci",
+    "non-deterministic"
+  ],
+  "author": "Ozperium",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Ozperium/agentspec.git"
+  },
+  "homepage": "https://agentspec.dev",
+  "bugs": {
+    "url": "https://github.com/Ozperium/agentspec/issues"
+  },
+  "files": [
+    "dist/",
+    "action.yml",
+    "README.md",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "yaml": "^2.4.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.4.0",
+    "@types/node": "^20.0.0"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}