@gnsx/genesys.agent.eval 1.0.0

package/dist/src/cli-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-runner.js","sourceRoot":"","sources":["../../src/cli-runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAe3C;;GAEG;AACH,MAAM,OAAO,QAAS,SAAQ,KAAK;IAGf;IACA;IACA;IAJlB,YACE,OAAe,EACC,OAAe,EACf,QAAgB,EAChB,MAAc;QAE9B,KAAK,CAAC,OAAO,CAAC,CAAC;QAJC,YAAO,GAAP,OAAO,CAAQ;QACf,aAAQ,GAAR,QAAQ,CAAQ;QAChB,WAAM,GAAN,MAAM,CAAQ;QAG9B,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;IACzB,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,SAAS,iBAAiB,CAAC,YAAoB;IAC7C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC;IAEpC,sCAAsC;IACtC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACvB,CAAC;IAED,4FAA4F;IAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACrB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACrB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAc,EACd,OAAmB;IAEnB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,iHAAiH;IAEjH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAEhD,2EAA2E;QAC3E,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,iBAAiB,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC;QAEnE,oEAAoE;QACpE,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC;QAErC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE;YAClC,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE;YACvB,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QAEH,qCAAqC;QACrC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACnC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACxC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACxC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,MAAM,CAAC,IAAI,QAAQ,CACjB,2BAA2B,OAAO,CAAC,OAAO,IAAI,EAC9C,GAAG,KAAK,KAAK,EACb,CAAC,CAAC,EACF,MAAM,CACP,CAAC,CAAC;QACL,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,CAAC,IAAI,QAAQ,CACjB,mBAAmB,KAAK,KAAK,KAAK,CAAC,OAAO,mDAAmD,EAC7F,GAAG,KAAK,KAAK,EACb,CAAC,CAAC,EACF,MAAM,CACP,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAE1C,OAAO,CAAC;gBACN,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;gBACrB,QAAQ,EAAE,IAAI,IAAI,CAAC;gBACnB,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;gBACrB,UAAU;aACX,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/embedding-judge.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Embedding-based cosine similarity judge using an actual embedding model.
+ *
+ * Uses @huggingface/transformers with a lightweight local model for semantic similarity.
+ *
+ * @module embedding-judge
+ */
+import type { TestCase } from './types.js';
+/**
+ * Result of judging a test case.
+ */
+export interface JudgeResult {
+    /** Score from 0 to 1 */
+    score: number;
+    /** Reasoning for the score */
+    reasoning: string;
+    /** Whether the test passed (score >= threshold) */
+    passed: boolean;
+}
+/**
+ * Configuration for the embedding judge.
+ */
+export interface EmbeddingJudgeConfig {
+    /** Score threshold for passing (0-1, default: 0.7) */
+    passThreshold?: number;
+    /** Embedding model to use (default: Xenova/all-MiniLM-L6-v2 via Hugging Face) */
+    model?: string;
+}
+/**
+ * Judge that evaluates agent outputs using embedding cosine similarity.
+ *
+ * Uses a local transformer model for semantic similarity comparison.
+ * The model is downloaded on first use and cached locally.
+ */
+export declare class EmbeddingJudge {
+    private _config;
+    private _pipeline;
+    private _modelLoading;
+    constructor(config?: EmbeddingJudgeConfig);
+    /**
+     * Get or create the embedding pipeline.
+     * Lazy loads the model on first use.
+     */
+    private getPipeline;
+    /**
+     * Generate embeddings for text.
+     *
+     * @param text - Text to embed
+     * @returns Embedding vector
+     */
+    private generateEmbedding;
+    /**
+     * Calculate cosine similarity between two vectors.
+     */
+    private cosineSimilarity;
+    /**
+     * Evaluate a test case against the actual output.
+     *
+     * @param test - The test case
+     * @param actualOutput - The actual output from the agent
+     * @returns The judge result with score and reasoning
+     */
+    evaluate(test: TestCase, actualOutput: string): Promise<JudgeResult>;
+    /**
+     * Create a judge function compatible with the TestRunner.
+     *
+     * @returns A function that can be passed to the runner
+     */
+    createEvaluator(): (test: TestCase, actualOutput: string) => Promise<{
+        score: number;
+        reasoning: string;
+        passed: boolean;
+    }>;
+    /**
+     * Get the judge configuration.
+     */
+    get config(): EmbeddingJudgeConfig;
+}
+/**
+ * Create an embedding judge with the given configuration.
+ *
+ * @param config - Judge configuration
+ * @returns A new EmbeddingJudge instance
+ *
+ * @example
+ * ```typescript
+ * const judge = createEmbeddingJudge({ passThreshold: 0.75 });
+ * const result = await judge.evaluate(test, output);
+ * console.log(`Score: ${result.score}, Passed: ${result.passed}`);
+ * ```
+ */
+export declare function createEmbeddingJudge(config?: EmbeddingJudgeConfig): EmbeddingJudge;
+//# sourceMappingURL=embedding-judge.d.ts.map

package/dist/src/embedding-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-judge.d.ts","sourceRoot":"","sources":["../../src/embedding-judge.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAC;IAEd,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAElB,mDAAmD;IACnD,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,sDAAsD;IACtD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,SAAS,CAA0C;IAC3D,OAAO,CAAC,aAAa,CAAmD;gBAE5D,MAAM,GAAE,oBAAyB;IAQ7C;;;OAGG;YACW,WAAW;IAoBzB;;;;;OAKG;YACW,iBAAiB;IAY/B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAqBxB;;;;;;OAMG;IACG,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAwC1E;;;;OAIG;IACH,eAAe,IAAI,CACjB,IAAI,EAAE,QAAQ,EACd,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,OAAO,CAAC;KACjB,CAAC;IAWF;;OAEG;IACH,IAAI,MAAM,IAAI,oBAAoB,CAEjC;CACF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,CAAC,EAAE,oBAAoB,GAAG,cAAc,CAElF"}

package/dist/src/embedding-judge.js

@@ -0,0 +1,160 @@
+/**
+ * Embedding-based cosine similarity judge using an actual embedding model.
+ *
+ * Uses @huggingface/transformers with a lightweight local model for semantic similarity.
+ *
+ * @module embedding-judge
+ */
+import { pipeline } from '@huggingface/transformers';
+/**
+ * Judge that evaluates agent outputs using embedding cosine similarity.
+ *
+ * Uses a local transformer model for semantic similarity comparison.
+ * The model is downloaded on first use and cached locally.
+ */
+export class EmbeddingJudge {
+    _config;
+    _pipeline = null;
+    _modelLoading = null;
+    constructor(config = {}) {
+        this._config = {
+            passThreshold: 0.7,
+            model: 'Xenova/all-MiniLM-L6-v2',
+            ...config,
+        };
+    }
+    /**
+     * Get or create the embedding pipeline.
+     * Lazy loads the model on first use.
+     */
+    async getPipeline() {
+        if (this._pipeline) {
+            return this._pipeline;
+        }
+        if (this._modelLoading) {
+            return this._modelLoading;
+        }
+        this._modelLoading = pipeline('feature-extraction', this._config.model);
+        this._pipeline = await this._modelLoading;
+        return this._pipeline;
+    }
+    /**
+     * Generate embeddings for text.
+     *
+     * @param text - Text to embed
+     * @returns Embedding vector
+     */
+    async generateEmbedding(text) {
+        const pipe = await this.getPipeline();
+        const output = await pipe(text, {
+            pooling: 'mean',
+            normalize: true,
+        });
+        // Convert to array
+        return Array.from(output.data);
+    }
+    /**
+     * Calculate cosine similarity between two vectors.
+     */
+    cosineSimilarity(a, b) {
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        // Vectors are already normalized, but calculate just in case
+        const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+        if (magnitude === 0) {
+            return 0;
+        }
+        return Math.max(0, Math.min(1, dotProduct / magnitude));
+    }
+    /**
+     * Evaluate a test case against the actual output.
+     *
+     * @param test - The test case
+     * @param actualOutput - The actual output from the agent
+     * @returns The judge result with score and reasoning
+     */
+    async evaluate(test, actualOutput) {
+        try {
+            // Generate embeddings for both expected and actual output
+            const expectedEmbedding = await this.generateEmbedding(test.expectedOutput);
+            const actualEmbedding = await this.generateEmbedding(actualOutput);
+            // Calculate similarity
+            const score = this.cosineSimilarity(expectedEmbedding, actualEmbedding);
+            // Generate reasoning
+            let reasoning;
+            if (score >= 0.9) {
+                reasoning = 'Very high semantic similarity - output closely matches expected content.';
+            }
+            else if (score >= 0.75) {
+                reasoning = 'Good semantic similarity with minor differences in meaning or detail.';
+            }
+            else if (score >= this._config.passThreshold) {
+                reasoning = 'Moderate similarity - core concepts match but notable differences exist.';
+            }
+            else if (score >= 0.4) {
+                reasoning = 'Low semantic similarity - significant differences in meaning.';
+            }
+            else {
+                reasoning = 'Very low similarity - output does not match expected content.';
+            }
+            const passed = score >= this._config.passThreshold;
+            return {
+                score,
+                reasoning,
+                passed,
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                score: 0,
+                reasoning: `Embedding evaluation failed: ${errorMessage}`,
+                passed: false,
+            };
+        }
+    }
+    /**
+     * Create a judge function compatible with the TestRunner.
+     *
+     * @returns A function that can be passed to the runner
+     */
+    createEvaluator() {
+        return async (test, actualOutput) => {
+            const result = await this.evaluate(test, actualOutput);
+            return {
+                score: result.score,
+                reasoning: result.reasoning,
+                passed: result.passed,
+            };
+        };
+    }
+    /**
+     * Get the judge configuration.
+     */
+    get config() {
+        return this._config;
+    }
+}
+/**
+ * Create an embedding judge with the given configuration.
+ *
+ * @param config - Judge configuration
+ * @returns A new EmbeddingJudge instance
+ *
+ * @example
+ * ```typescript
+ * const judge = createEmbeddingJudge({ passThreshold: 0.75 });
+ * const result = await judge.evaluate(test, output);
+ * console.log(`Score: ${result.score}, Passed: ${result.passed}`);
+ * ```
+ */
+export function createEmbeddingJudge(config) {
+    return new EmbeddingJudge(config);
+}
+//# sourceMappingURL=embedding-judge.js.map

package/dist/src/embedding-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-judge.js","sourceRoot":"","sources":["../../src/embedding-judge.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAkC,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AA6BrF;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CAAiC;IACxC,SAAS,GAAqC,IAAI,CAAC;IACnD,aAAa,GAA8C,IAAI,CAAC;IAExE,YAAY,SAA+B,EAAE;QAC3C,IAAI,CAAC,OAAO,GAAG;YACb,aAAa,EAAE,GAAG;YAClB,KAAK,EAAE,yBAAyB;YAChC,GAAG,MAAM;SACV,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,SAAS,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC;QAC5B,CAAC;QAGD,IAAI,CAAC,aAAa,GAAG,QAAQ,CAC3B,oBAAoB,EACpB,IAAI,CAAC,OAAO,CAAC,KAAK,CAE8B,CAAC;QAEnD,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC;QAC1C,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAC1C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAEtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE;YAC9B,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,mBAAmB;QACnB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,CAAW,EAAE,CAAW;QAC/C,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QAED,6DAA6D;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEtD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAc,EAAE,YAAoB;QACjD,IAAI,CAAC;YACH,0DAA0D;YAC1D,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC5E,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAEnE,uBAAuB;YACvB,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;YAExE,qBAAqB;YACrB,IAAI,SAAiB,CAAC;YACtB,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,SAAS,GAAG,0EAA0E,CAAC;YACzF,CAAC;iBAAM,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBACzB,SAAS,GAAG,uEAAuE,CAAC;YACtF,CAAC;iBAAM,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC/C,SAAS,GAAG,0EAA0E,CAAC;YACzF,CAAC;iBAAM,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACxB,SAAS,GAAG,+DAA+D,CAAC;YAC9E,CAAC;iBAAM,CAAC;gBACN,SAAS,GAAG,+DAA+D,CAAC;YAC9E,CAAC;YAED,MAAM,MAAM,GAAG,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;YAEnD,OAAO;gBACL,KAAK;gBACL,SAAS;gBACT,MAAM;aACP,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,OAAO;gBACL,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE,gCAAgC,YAAY,EAAE;gBACzD,MAAM,EAAE,KAAK;aACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,eAAe;QAQb,OAAO,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;YAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACvD,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;aACtB,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAA6B;IAChE,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC"}

package/dist/src/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Genesys Agent Eval Harness
+ *
+ * A framework for evaluating AI agents (pi and genesys) against test suites defined in YAML files.
+ *
+ * @packageDocumentation
+ */
+export type { AgentResponse, Args, EvalResults, EvalSummary, JudgeConfig, RunnerConfig, TestCase, TestResult, TestSuite, } from './types.js';
+export { CLIError, runAgent, type RunOptions, } from './cli-runner.js';
+export { createEmbeddingJudge, EmbeddingJudge, type EmbeddingJudgeConfig, type JudgeResult as EmbeddingJudgeResult, } from './embedding-judge.js';
+export { createJudge, evaluate, Judge, type JudgeResult as LLMJudgeResult, } from './judge.js';
+export { formatResults, type OutputFormat, Reporter, type ReporterConfig, reportResults, } from './reporter.js';
+export { runEvaluation, type ProgressCallback, TestRunner, } from './runner.js';
+export { loadTestSuite, type LoadResult, parseTestSuite, TestLoadError, TestValidationError, validateTestSuite, } from './test-loader.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,YAAY,EACV,aAAa,EACb,IAAI,EACJ,WAAW,EACX,WAAW,EACX,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,SAAS,GACV,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,KAAK,UAAU,GAChB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,KAAK,oBAAoB,EACzB,KAAK,WAAW,IAAI,oBAAoB,GACzC,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,WAAW,EACX,QAAQ,EACR,KAAK,EACL,KAAK,WAAW,IAAI,cAAc,GACnC,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,aAAa,EACb,KAAK,YAAY,EACjB,QAAQ,EACR,KAAK,cAAc,EACnB,aAAa,GACd,MAAM,eAAe,CAAC;AAGvB,OAAO,EACL,aAAa,EACb,KAAK,gBAAgB,EACrB,UAAU,GACX,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,aAAa,EACb,KAAK,UAAU,EACf,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,kBAAkB,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,20 @@
+/**
+ * Genesys Agent Eval Harness
+ *
+ * A framework for evaluating AI agents (pi and genesys) against test suites defined in YAML files.
+ *
+ * @packageDocumentation
+ */
+// Export CLI runner functionality
+export { CLIError, runAgent, } from './cli-runner.js';
+// Export embedding judge functionality
+export { createEmbeddingJudge, EmbeddingJudge, } from './embedding-judge.js';
+// Export LLM judge functionality
+export { createJudge, evaluate, Judge, } from './judge.js';
+// Export reporter functionality
+export { formatResults, Reporter, reportResults, } from './reporter.js';
+// Export runner functionality
+export { runEvaluation, TestRunner, } from './runner.js';
+// Export test loader functionality
+export { loadTestSuite, parseTestSuite, TestLoadError, TestValidationError, validateTestSuite, } from './test-loader.js';
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAeH,kCAAkC;AAClC,OAAO,EACL,QAAQ,EACR,QAAQ,GAET,MAAM,iBAAiB,CAAC;AAEzB,uCAAuC;AACvC,OAAO,EACL,oBAAoB,EACpB,cAAc,GAGf,MAAM,sBAAsB,CAAC;AAE9B,iCAAiC;AACjC,OAAO,EACL,WAAW,EACX,QAAQ,EACR,KAAK,GAEN,MAAM,YAAY,CAAC;AAEpB,gCAAgC;AAChC,OAAO,EACL,aAAa,EAEb,QAAQ,EAER,aAAa,GACd,MAAM,eAAe,CAAC;AAEvB,8BAA8B;AAC9B,OAAO,EACL,aAAa,EAEb,UAAU,GACX,MAAM,aAAa,CAAC;AAErB,mCAAmC;AACnC,OAAO,EACL,aAAa,EAEb,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,kBAAkB,CAAC"}

package/dist/src/judge.d.ts

@@ -0,0 +1,95 @@
+/**
+ * LLM-based judge for evaluating agent outputs against expected results.
+ *
+ * The judge uses a separate LLM to score how well the agent's output matches
+ * the expected output description on a scale of 0-1.
+ *
+ * @module judge
+ */
+import type { JudgeConfig, TestCase } from './types.js';
+/**
+ * Result of judging a test case.
+ */
+export interface JudgeResult {
+    /** Score from 0 to 1 */
+    score: number;
+    /** Reasoning for the score */
+    reasoning: string;
+    /** Whether the test passed (score >= threshold) */
+    passed: boolean;
+}
+/**
+ * Judge that evaluates agent outputs using an LLM.
+ *
+ * The judge prompts a separate LLM to compare the actual output against the
+ * expected output description and score it on a scale of 0-1.
+ */
+export declare class Judge {
+    private _config;
+    constructor(config: JudgeConfig);
+    /**
+     * Build the judge prompt.
+     *
+     * @param test - The test case
+     * @param actualOutput - The actual output from the agent
+     * @returns The prompt to send to the judge LLM
+     */
+    private buildPrompt;
+    /**
+     * Get the model instance based on provider.
+     *
+     * @returns Model instance for the Vercel AI SDK
+     */
+    private getModel;
+    /**
+     * Evaluate a test case against the actual output.
+     *
+     * @param test - The test case
+     * @param actualOutput - The actual output from the agent
+     * @returns The judge result with score and reasoning
+     */
+    evaluate(test: TestCase, actualOutput: string): Promise<JudgeResult>;
+    /**
+     * Create a judge function compatible with the TestRunner.
+     *
+     * @returns A function that can be passed to the runner
+     */
+    createEvaluator(): (test: TestCase, actualOutput: string) => Promise<{
+        score: number;
+        reasoning: string;
+        passed: boolean;
+    }>;
+    /**
+     * Get the judge configuration.
+     */
+    get config(): JudgeConfig;
+}
+/**
+ * Create a judge with the given configuration.
+ *
+ * @param config - Judge configuration
+ * @returns A new Judge instance
+ *
+ * @example
+ * ```typescript
+ * const judge = createJudge({
+ *   provider: 'anthropic',
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   passThreshold: 0.8
+ * });
+ *
+ * const result = await judge.evaluate(test, output);
+ * console.log(`Score: ${result.score}, Passed: ${result.passed}`);
+ * ```
+ */
+export declare function createJudge(config: JudgeConfig): Judge;
+/**
+ * Quick evaluate function for one-off judgments.
+ *
+ * @param config - Judge configuration
+ * @param test - Test case
+ * @param actualOutput - Actual output to evaluate
+ * @returns Judge result
+ */
+export declare function evaluate(config: JudgeConfig, test: TestCase, actualOutput: string): Promise<JudgeResult>;
+//# sourceMappingURL=judge.d.ts.map

package/dist/src/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/judge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAQH,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAC;IAEd,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAElB,mDAAmD;IACnD,MAAM,EAAE,OAAO,CAAC;CACjB;AAeD;;;;;GAKG;AACH,qBAAa,KAAK;IAChB,OAAO,CAAC,OAAO,CAAc;gBAEjB,MAAM,EAAE,WAAW;IAQ/B;;;;;;OAMG;IACH,OAAO,CAAC,WAAW;IA+BnB;;;;OAIG;IACH,OAAO,CAAC,QAAQ;IAoBhB;;;;;;OAMG;IACG,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAkC1E;;;;OAIG;IACH,eAAe,IAAI,CACjB,IAAI,EAAE,QAAQ,EACd,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,OAAO,CAAC;KACjB,CAAC;IAWF;;OAEG;IACH,IAAI,MAAM,IAAI,WAAW,CAExB;CACF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,KAAK,CAEtD;AAED;;;;;;;GAOG;AACH,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,WAAW,EACnB,IAAI,EAAE,QAAQ,EACd,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,WAAW,CAAC,CAGtB"}

package/dist/src/judge.js

@@ -0,0 +1,189 @@
+/**
+ * LLM-based judge for evaluating agent outputs against expected results.
+ *
+ * The judge uses a separate LLM to score how well the agent's output matches
+ * the expected output description on a scale of 0-1.
+ *
+ * @module judge
+ */
+import { anthropic } from '@ai-sdk/anthropic';
+import { google } from '@ai-sdk/google';
+import { openai } from '@ai-sdk/openai';
+import { generateObject } from 'ai';
+import { z } from 'zod';
+/**
+ * Zod schema for the judge's structured output.
+ */
+const judgeOutputSchema = z.object({
+    score: z.number().min(0).max(1).describe('Score from 0 to 1 where 1 is perfect'),
+    reasoning: z.string().describe('Explanation for the score'),
+});
+/**
+ * Judge that evaluates agent outputs using an LLM.
+ *
+ * The judge prompts a separate LLM to compare the actual output against the
+ * expected output description and score it on a scale of 0-1.
+ */
+export class Judge {
+    _config;
+    constructor(config) {
+        this._config = {
+            passThreshold: 0.7,
+            temperature: 0,
+            ...config,
+        };
+    }
+    /**
+     * Build the judge prompt.
+     *
+     * @param test - The test case
+     * @param actualOutput - The actual output from the agent
+     * @returns The prompt to send to the judge LLM
+     */
+    buildPrompt(test, actualOutput) {
+        return `You are an expert evaluator assessing the quality of AI responses.
+
+Your task is to evaluate how well the ACTUAL OUTPUT matches the EXPECTED OUTPUT description.
+
+## Test Input
+${test.input}
+
+## Expected Output Description
+${test.expectedOutput}
+
+${test.context ? `## Additional Context\n${test.context}\n\n` : ''}## Actual Output
+${actualOutput}
+
+## Evaluation Instructions
+
+1. Carefully read the expected output description and the actual output
+2. Score the actual output on a scale of 0.0 to 1.0 where:
+   - 1.0 = Perfect match, fully satisfies the expected output
+   - 0.8-0.9 = Good match, minor issues or omissions
+   - 0.6-0.7 = Partial match, significant issues but some correct elements
+   - 0.4-0.5 = Poor match, mostly incorrect or incomplete
+   - 0.0-0.3 = Very poor match, completely wrong or irrelevant
+
+3. Provide clear reasoning for your score
+
+Respond with a structured object containing:
+- score: number from 0 to 1
+- reasoning: string explaining your evaluation`;
+    }
+    /**
+     * Get the model instance based on provider.
+     *
+     * @returns Model instance for the Vercel AI SDK
+     */
+    getModel() {
+        const { provider, model } = this._config;
+        switch (provider) {
+            case 'anthropic': {
+                return anthropic(model);
+            }
+            case 'openai': {
+                return openai(model);
+            }
+            case 'google':
+            case 'gemini': {
+                return google(model);
+            }
+            default: {
+                throw new Error(`Unsupported judge provider: ${provider}`);
+            }
+        }
+    }
+    /**
+     * Evaluate a test case against the actual output.
+     *
+     * @param test - The test case
+     * @param actualOutput - The actual output from the agent
+     * @returns The judge result with score and reasoning
+     */
+    async evaluate(test, actualOutput) {
+        const prompt = this.buildPrompt(test, actualOutput);
+        try {
+            const { object } = await generateObject({
+                model: this.getModel(),
+                schema: judgeOutputSchema,
+                messages: [
+                    {
+                        role: 'user',
+                        content: prompt,
+                    },
+                ],
+                temperature: this._config.temperature,
+            });
+            const passed = object.score >= (this._config.passThreshold ?? 0.7);
+            return {
+                score: object.score,
+                reasoning: object.reasoning,
+                passed,
+            };
+        }
+        catch (error) {
+            // If structured generation fails, return a failed result
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                score: 0,
+                reasoning: `Judge evaluation failed: ${errorMessage}`,
+                passed: false,
+            };
+        }
+    }
+    /**
+     * Create a judge function compatible with the TestRunner.
+     *
+     * @returns A function that can be passed to the runner
+     */
+    createEvaluator() {
+        return async (test, actualOutput) => {
+            const result = await this.evaluate(test, actualOutput);
+            return {
+                score: result.score,
+                reasoning: result.reasoning,
+                passed: result.passed,
+            };
+        };
+    }
+    /**
+     * Get the judge configuration.
+     */
+    get config() {
+        return this._config;
+    }
+}
+/**
+ * Create a judge with the given configuration.
+ *
+ * @param config - Judge configuration
+ * @returns A new Judge instance
+ *
+ * @example
+ * ```typescript
+ * const judge = createJudge({
+ *   provider: 'anthropic',
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   passThreshold: 0.8
+ * });
+ *
+ * const result = await judge.evaluate(test, output);
+ * console.log(`Score: ${result.score}, Passed: ${result.passed}`);
+ * ```
+ */
+export function createJudge(config) {
+    return new Judge(config);
+}
+/**
+ * Quick evaluate function for one-off judgments.
+ *
+ * @param config - Judge configuration
+ * @param test - Test case
+ * @param actualOutput - Actual output to evaluate
+ * @returns Judge result
+ */
+export async function evaluate(config, test, actualOutput) {
+    const judge = createJudge(config);
+    return judge.evaluate(test, actualOutput);
+}
+//# sourceMappingURL=judge.js.map

package/dist/src/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/judge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAkBxB;;GAEG;AACH,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,sCAAsC,CAAC;IAChF,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;CAC5D,CAAC,CAAC;AAOH;;;;;GAKG;AACH,MAAM,OAAO,KAAK;IACR,OAAO,CAAc;IAE7B,YAAY,MAAmB;QAC7B,IAAI,CAAC,OAAO,GAAG;YACb,aAAa,EAAE,GAAG;YAClB,WAAW,EAAE,CAAC;YACd,GAAG,MAAM;SACV,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAc,EAAE,YAAoB;QACtD,OAAO;;;;;EAKT,IAAI,CAAC,KAAK;;;EAGV,IAAI,CAAC,cAAc;;EAEnB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE;EAChE,YAAY;;;;;;;;;;;;;;;;+CAgBiC,CAAC;IAC9C,CAAC;IAED;;;;OAIG;IACK,QAAQ;QACd,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAEzC,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YACD,KAAK,QAAQ,CAAC;YACd,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;YACD,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAc,EAAE,YAAoB;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEpD,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;gBACtC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAA8D;gBAClF,MAAM,EAAE,iBAAiB;gBACzB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,MAAM;qBAChB;iBACF;gBACD,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;aACtC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;YAEnE,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,MAAM;aACP,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yDAAyD;YACzD,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,OAAO;gBACL,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE,4BAA4B,YAAY,EAAE;gBACrD,MAAM,EAAE,KAAK;aACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,eAAe;QAQb,OAAO,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;YAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACvD,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;aACtB,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,WAAW,CAAC,MAAmB;IAC7C,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAmB,EACnB,IAAc,EACd,YAAoB;IAEpB,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;AAC5C,CAAC"}

package/dist/src/launcher.d.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+/**
+ * Genesys Agent Eval CLI launcher.
+ *
+ * Entry point for the evaluation harness. Spawns pi or genesys CLI
+ * processes to run tests and evaluates the results.
+ */
+export {};
+//# sourceMappingURL=launcher.d.ts.map

package/dist/src/launcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"launcher.d.ts","sourceRoot":"","sources":["../../src/launcher.ts"],"names":[],"mappings":";AACA;;;;;GAKG"}