evalforge 0.6.0

package/dist/binary.d.ts

@@ -0,0 +1,2 @@
+export declare function findBinary(): string;
+//# sourceMappingURL=binary.d.ts.map

package/dist/binary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary.d.ts","sourceRoot":"","sources":["../src/binary.ts"],"names":[],"mappings":"AAIA,wBAAgB,UAAU,IAAI,MAAM,CA2BnC"}

package/dist/binary.js

@@ -0,0 +1,67 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findBinary = findBinary;
+const child_process_1 = require("child_process");
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+function findBinary() {
+    // 1. EVALFORGE_BIN env var
+    if (process.env.EVALFORGE_BIN) {
+        return process.env.EVALFORGE_BIN;
+    }
+    // 2. Walk up from this file looking for target/debug/evalforge
+    let current = path.dirname(__filename);
+    for (let i = 0; i < 6; i++) {
+        const candidate = path.join(current, 'target', 'debug', 'evalforge');
+        if (fs.existsSync(candidate))
+            return candidate;
+        const release = path.join(current, 'target', 'release', 'evalforge');
+        if (fs.existsSync(release))
+            return release;
+        current = path.dirname(current);
+    }
+    // 3. On PATH
+    try {
+        const which = (0, child_process_1.execSync)('which evalforge', { encoding: 'utf8' }).trim();
+        if (which)
+            return which;
+    }
+    catch { }
+    throw new Error('EvalForge binary not found.\n' +
+        'Option 1: git clone https://github.com/heManKuMAR6/evalforge && cargo build --release\n' +
+        'Option 2: export EVALFORGE_BIN=/path/to/evalforge/binary');
+}
+//# sourceMappingURL=binary.js.map

package/dist/binary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary.js","sourceRoot":"","sources":["../src/binary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,gCA2BC;AA/BD,iDAAyC;AACzC,2CAA6B;AAC7B,uCAAyB;AAEzB,SAAgB,UAAU;IACxB,2BAA2B;IAC3B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACnC,CAAC;IAED,+DAA+D;IAC/D,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;QACrE,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QACrE,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,OAAO,OAAO,CAAC;QAC3C,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,aAAa;IACb,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAA,wBAAQ,EAAC,iBAAiB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACvE,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IAEV,MAAM,IAAI,KAAK,CACb,+BAA+B;QAC/B,yFAAyF;QACzF,0DAA0D,CAC3D,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+import { EvalResult, RunOptions } from './types';
+export { EvalResult, MetricResult, RunOptions, Trace } from './types';
+export declare function run(tracePath: string, options: RunOptions): EvalResult;
+export declare function demo(): EvalResult;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAS,MAAM,SAAS,CAAC;AAExD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAkCtE,wBAAgB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,GAAG,UAAU,CAmBtE;AAED,wBAAgB,IAAI,IAAI,UAAU,CAuCjC"}

package/dist/index.js

@@ -0,0 +1,132 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = run;
+exports.demo = demo;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const binary_1 = require("./binary");
+function parseOutput(output) {
+    const metrics = [];
+    const lines = output.split('\n');
+    let traceId = '';
+    let framework = '';
+    let overallPassed = true;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (line.startsWith('Trace ID:'))
+            traceId = line.split(':')[1].trim();
+        if (line.startsWith('Framework:'))
+            framework = line.split(':')[1].trim();
+        if (line === 'Overall: FAIL')
+            overallPassed = false;
+        const metricMatch = line.match(/^(\w+)\s+([\d.]+)\s+(PASS|FAIL)$/);
+        if (metricMatch) {
+            const reason = lines[i + 1]?.trim().startsWith('Reason:')
+                ? lines[i + 1].replace('Reason:', '').trim()
+                : '';
+            metrics.push({
+                metric: metricMatch[1],
+                score: parseFloat(metricMatch[2]),
+                passed: metricMatch[3] === 'PASS',
+                reason,
+            });
+        }
+    }
+    return { traceId, framework, metrics, passed: overallPassed };
+}
+function run(tracePath, options) {
+    const binary = (0, binary_1.findBinary)();
+    const args = [
+        'run',
+        '--trace', tracePath,
+        '--metrics', options.metrics.join(','),
+        '--threshold', String(options.threshold ?? 0.7),
+    ];
+    if (options.mock)
+        args.push('--mock');
+    if (options.rubric) {
+        args.push('--rubric');
+        args.push(options.rubric);
+    }
+    const env = { ...process.env };
+    if (options.apiKey)
+        env.ANTHROPIC_API_KEY = options.apiKey;
+    const result = (0, child_process_1.spawnSync)(binary, args, { encoding: 'utf8', env });
+    if (result.error)
+        throw result.error;
+    return parseOutput(result.stdout);
+}
+function demo() {
+    const trace = {
+        evalforge_version: '0.1',
+        trace_id: 'demo-js-001',
+        timestamp: new Date().toISOString(),
+        metadata: {
+            framework: 'openai-agents',
+            model: 'gpt-4o',
+            agent_name: 'demo-agent',
+            duration_ms: 1200,
+            total_tokens: 450,
+        },
+        input: {
+            user: 'What is the capital of Australia?',
+            system: 'You are a helpful assistant.',
+        },
+        steps: [
+            { step_id: 1, type: 'thought', content: 'I know this — Canberra.' },
+            {
+                step_id: 2,
+                type: 'tool_call',
+                tool: 'web_search',
+                input: { query: 'capital of Australia' },
+                output: { result: 'Canberra is the capital of Australia.' },
+                duration_ms: 400,
+            },
+        ],
+        output: { answer: 'The capital of Australia is Canberra.' },
+        eval_hints: {
+            expected_tools: ['web_search'],
+            expected_answer: 'Canberra',
+            context_documents: [],
+        },
+    };
+    const tmp = path.join(os.tmpdir(), `evalforge_demo_${Date.now()}.json`);
+    fs.writeFileSync(tmp, JSON.stringify(trace));
+    return run(tmp, { metrics: ['faithfulness'], mock: true });
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,kBAmBC;AAED,oBAuCC;AArGD,iDAA0C;AAC1C,uCAAyB;AACzB,uCAAyB;AACzB,2CAA6B;AAC7B,qCAAsC;AAKtC,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,aAAa,GAAG,IAAI,CAAC;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7B,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;YAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzE,IAAI,IAAI,KAAK,eAAe;YAAE,aAAa,GAAG,KAAK,CAAC;QAEpD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnE,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;gBACvD,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;gBAC5C,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;gBACtB,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,MAAM;gBACjC,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;AAChE,CAAC;AAED,SAAgB,GAAG,CAAC,SAAiB,EAAE,OAAmB;IACxD,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAC5B,MAAM,IAAI,GAAG;QACX,KAAK;QACL,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;QACtC,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;KAChD,CAAC;IAEF,IAAI,OAAO,CAAC,IAAI;QAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAAC,CAAC;IAEzE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC/B,IAAI,OAAO,CAAC,MAAM;QAAE,GAAG,CAAC,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAE3D,MAAM,MAAM,GAAG,IAAA,yBAAS,EAAC,MAAM,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,KAAK;QAAE,MAAM,MAAM,CAAC,KAAK,CAAC;IAErC,OAAO,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAgB,IAAI;IAClB,MAAM,KAAK,GAAU;QACnB,iBAAiB,EAAE,KAAK;QACxB,QAAQ,EAAE,aAAa;QACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE;YACR,SAAS,EAAE,eAAe;YAC1B,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,YAAY;YACxB,WAAW,EAAE,IAAI;YACjB,YAAY,EAAE,GAAG;SAClB;QACD,KAAK,EAAE;YACL,IAAI,EAAE,mCAAmC;YACzC,MAAM,EAAE,8BAA8B;SACvC;QACD,KAAK,EAAE;YACL,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,yBAAyB,EAAE;YACnE;gBACE,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE;gBACxC,MAAM,EAAE,EAAE,MAAM,EAAE,uCAAuC,EAAE;gBAC3D,WAAW,EAAE,GAAG;aACjB;SACF;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,uCAAuC,EAAE;QAC3D,UAAU,EAAE;YACV,cAAc,EAAE,CAAC,YAAY,CAAC;YAC9B,eAAe,EAAE,UAAU;YAC3B,iBAAiB,EAAE,EAAE;SACtB;KACF,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAE7C,OAAO,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7D,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,54 @@
+export interface MetricResult {
+    metric: string;
+    score: number;
+    passed: boolean;
+    reason: string;
+}
+export interface EvalResult {
+    traceId: string;
+    framework: string;
+    metrics: MetricResult[];
+    passed: boolean;
+}
+export interface RunOptions {
+    metrics: string[];
+    threshold?: number;
+    mock?: boolean;
+    apiKey?: string;
+    rubric?: string;
+}
+export interface Trace {
+    evalforge_version: string;
+    trace_id: string;
+    timestamp: string;
+    metadata: {
+        framework: string;
+        model: string;
+        agent_name: string;
+        duration_ms: number;
+        total_tokens: number;
+    };
+    input: {
+        user: string;
+        system: string;
+    };
+    steps: Array<{
+        step_id: number;
+        type: string;
+        content?: string;
+        tool?: string;
+        input?: Record<string, unknown>;
+        output?: Record<string, unknown>;
+        duration_ms?: number;
+    }>;
+    output: {
+        answer: string;
+        finish_reason?: string;
+    };
+    eval_hints: {
+        expected_tools: string[];
+        expected_answer: string | null;
+        context_documents: string[];
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,KAAK;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,KAAK,EAAE,KAAK,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;IACH,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,UAAU,EAAE;QACV,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,iBAAiB,EAAE,MAAM,EAAE,CAAC;KAC7B,CAAC;CACH"}

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "evalforge",
+  "version": "0.6.0",
+  "description": "Framework-agnostic LLM agent evaluation harness",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "node --test tests/*.test.js",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "llm",
+    "evaluation",
+    "agents",
+    "langchain",
+    "openai"
+  ],
+  "author": "heManKuMAR6",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/heManKuMAR6/evalforge"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.2",
+    "typescript": "^5.0.0"
+  }
+}

package/src/binary.ts

@@ -0,0 +1,32 @@
+import { execSync } from 'child_process';
+import * as path from 'path';
+import * as fs from 'fs';
+
+export function findBinary(): string {
+  // 1. EVALFORGE_BIN env var
+  if (process.env.EVALFORGE_BIN) {
+    return process.env.EVALFORGE_BIN;
+  }
+
+  // 2. Walk up from this file looking for target/debug/evalforge
+  let current = path.dirname(__filename);
+  for (let i = 0; i < 6; i++) {
+    const candidate = path.join(current, 'target', 'debug', 'evalforge');
+    if (fs.existsSync(candidate)) return candidate;
+    const release = path.join(current, 'target', 'release', 'evalforge');
+    if (fs.existsSync(release)) return release;
+    current = path.dirname(current);
+  }
+
+  // 3. On PATH
+  try {
+    const which = execSync('which evalforge', { encoding: 'utf8' }).trim();
+    if (which) return which;
+  } catch {}
+
+  throw new Error(
+    'EvalForge binary not found.\n' +
+    'Option 1: git clone https://github.com/heManKuMAR6/evalforge && cargo build --release\n' +
+    'Option 2: export EVALFORGE_BIN=/path/to/evalforge/binary'
+  );
+}

package/src/index.ts

@@ -0,0 +1,102 @@
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { findBinary } from './binary';
+import { EvalResult, RunOptions, Trace } from './types';
+
+export { EvalResult, MetricResult, RunOptions, Trace } from './types';
+
+function parseOutput(output: string): EvalResult {
+  const metrics = [];
+  const lines = output.split('\n');
+
+  let traceId = '';
+  let framework = '';
+  let overallPassed = true;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+
+    if (line.startsWith('Trace ID:')) traceId = line.split(':')[1].trim();
+    if (line.startsWith('Framework:')) framework = line.split(':')[1].trim();
+    if (line === 'Overall: FAIL') overallPassed = false;
+
+    const metricMatch = line.match(/^(\w+)\s+([\d.]+)\s+(PASS|FAIL)$/);
+    if (metricMatch) {
+      const reason = lines[i + 1]?.trim().startsWith('Reason:')
+        ? lines[i + 1].replace('Reason:', '').trim()
+        : '';
+      metrics.push({
+        metric: metricMatch[1],
+        score: parseFloat(metricMatch[2]),
+        passed: metricMatch[3] === 'PASS',
+        reason,
+      });
+    }
+  }
+
+  return { traceId, framework, metrics, passed: overallPassed };
+}
+
+export function run(tracePath: string, options: RunOptions): EvalResult {
+  const binary = findBinary();
+  const args = [
+    'run',
+    '--trace', tracePath,
+    '--metrics', options.metrics.join(','),
+    '--threshold', String(options.threshold ?? 0.7),
+  ];
+
+  if (options.mock) args.push('--mock');
+  if (options.rubric) { args.push('--rubric'); args.push(options.rubric); }
+
+  const env = { ...process.env };
+  if (options.apiKey) env.ANTHROPIC_API_KEY = options.apiKey;
+
+  const result = spawnSync(binary, args, { encoding: 'utf8', env });
+  if (result.error) throw result.error;
+
+  return parseOutput(result.stdout);
+}
+
+export function demo(): EvalResult {
+  const trace: Trace = {
+    evalforge_version: '0.1',
+    trace_id: 'demo-js-001',
+    timestamp: new Date().toISOString(),
+    metadata: {
+      framework: 'openai-agents',
+      model: 'gpt-4o',
+      agent_name: 'demo-agent',
+      duration_ms: 1200,
+      total_tokens: 450,
+    },
+    input: {
+      user: 'What is the capital of Australia?',
+      system: 'You are a helpful assistant.',
+    },
+    steps: [
+      { step_id: 1, type: 'thought', content: 'I know this — Canberra.' },
+      {
+        step_id: 2,
+        type: 'tool_call',
+        tool: 'web_search',
+        input: { query: 'capital of Australia' },
+        output: { result: 'Canberra is the capital of Australia.' },
+        duration_ms: 400,
+      },
+    ],
+    output: { answer: 'The capital of Australia is Canberra.' },
+    eval_hints: {
+      expected_tools: ['web_search'],
+      expected_answer: 'Canberra',
+      context_documents: [],
+    },
+  };
+
+  const tmp = path.join(os.tmpdir(), `evalforge_demo_${Date.now()}.json`);
+  fs.writeFileSync(tmp, JSON.stringify(trace));
+
+  return run(tmp, { metrics: ['faithfulness'], mock: true });
+}

package/src/types.ts

@@ -0,0 +1,56 @@
+export interface MetricResult {
+  metric: string;
+  score: number;
+  passed: boolean;
+  reason: string;
+}
+
+export interface EvalResult {
+  traceId: string;
+  framework: string;
+  metrics: MetricResult[];
+  passed: boolean;
+}
+
+export interface RunOptions {
+  metrics: string[];
+  threshold?: number;
+  mock?: boolean;
+  apiKey?: string;
+  rubric?: string;
+}
+
+export interface Trace {
+  evalforge_version: string;
+  trace_id: string;
+  timestamp: string;
+  metadata: {
+    framework: string;
+    model: string;
+    agent_name: string;
+    duration_ms: number;
+    total_tokens: number;
+  };
+  input: {
+    user: string;
+    system: string;
+  };
+  steps: Array<{
+    step_id: number;
+    type: string;
+    content?: string;
+    tool?: string;
+    input?: Record<string, unknown>;
+    output?: Record<string, unknown>;
+    duration_ms?: number;
+  }>;
+  output: {
+    answer: string;
+    finish_reason?: string;
+  };
+  eval_hints: {
+    expected_tools: string[];
+    expected_answer: string | null;
+    context_documents: string[];
+  };
+}

package/tests/index.test.js

@@ -0,0 +1,27 @@
+const { test } = require('node:test');
+const assert = require('node:assert');
+const path = require('path');
+
+// We test the compiled JS
+const evalforge = require('../dist/index.js');
+
+test('demo() returns EvalResult with passed=true', () => {
+  process.env.EVALFORGE_BIN = path.resolve(
+    __dirname, '../../../target/debug/evalforge'
+  );
+  const result = evalforge.demo();
+  assert.strictEqual(result.passed, true);
+  assert.strictEqual(result.metrics.length > 0, true);
+  assert.strictEqual(result.metrics[0].score, 0.91);
+});
+
+test('EvalResult has correct shape', () => {
+  process.env.EVALFORGE_BIN = path.resolve(
+    __dirname, '../../../target/debug/evalforge'
+  );
+  const result = evalforge.demo();
+  assert.ok('traceId' in result);
+  assert.ok('framework' in result);
+  assert.ok('metrics' in result);
+  assert.ok('passed' in result);
+});

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "tests"]
+}