evalforge 0.7.0 → 0.7.1

package/dist/adapters/index.d.ts

@@ -0,0 +1,2 @@
+export { fromMastra } from './mastra';
+//# sourceMappingURL=index.d.ts.map

package/dist/adapters/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/adapters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC"}

package/dist/adapters/index.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromMastra = void 0;
+var mastra_1 = require("./mastra");
+Object.defineProperty(exports, "fromMastra", { enumerable: true, get: function () { return mastra_1.fromMastra; } });
+//# sourceMappingURL=index.js.map

package/dist/adapters/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/adapters/index.ts"],"names":[],"mappings":";;;AAAA,mCAAsC;AAA7B,oGAAA,UAAU,OAAA"}

package/dist/adapters/mastra.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Mastra adapter for EvalForge.
+ *
+ * Usage:
+ *   import { Agent } from '@mastra/core';
+ *   import { fromMastra } from 'evalforge/adapters/mastra';
+ *   import { run } from 'evalforge';
+ *
+ *   const agent = new Agent({ name: 'my-agent', ... });
+ *   const result = await agent.generate('What is the capital of France?');
+ *
+ *   const tracePath = fromMastra(result, { agentName: 'my-agent', model: 'gpt-4o' });
+ *   const evalResult = run(tracePath, { metrics: ['faithfulness'] });
+ */
+export interface MastraAdapterOptions {
+    agentName?: string;
+    model?: string;
+    question?: string;
+    expectedTools?: string[];
+}
+export declare function fromMastra(result: any, options?: MastraAdapterOptions): string;
+//# sourceMappingURL=mastra.d.ts.map

package/dist/adapters/mastra.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mastra.d.ts","sourceRoot":"","sources":["../../src/adapters/mastra.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,wBAAgB,UAAU,CACxB,MAAM,EAAE,GAAG,EACX,OAAO,GAAE,oBAAyB,GACjC,MAAM,CA4GR"}

package/dist/adapters/mastra.js

@@ -0,0 +1,151 @@
+"use strict";
+/**
+ * Mastra adapter for EvalForge.
+ *
+ * Usage:
+ *   import { Agent } from '@mastra/core';
+ *   import { fromMastra } from 'evalforge/adapters/mastra';
+ *   import { run } from 'evalforge';
+ *
+ *   const agent = new Agent({ name: 'my-agent', ... });
+ *   const result = await agent.generate('What is the capital of France?');
+ *
+ *   const tracePath = fromMastra(result, { agentName: 'my-agent', model: 'gpt-4o' });
+ *   const evalResult = run(tracePath, { metrics: ['faithfulness'] });
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromMastra = fromMastra;
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+function fromMastra(result, options = {}) {
+    /**
+     * Convert Mastra agent.generate() result to EvalForge trace JSON file.
+     *
+     * Supports:
+     * - result.text — final answer string
+     * - result.steps — array of reasoning steps
+     * - result.toolCalls — array of tool invocations
+     * - result.usage — token usage stats
+     */
+    const { agentName = 'mastra-agent', model = 'unknown', question = '', expectedTools = [], } = options;
+    const steps = [];
+    let stepId = 1;
+    let finalAnswer = '';
+    let totalTokens = 0;
+    // Extract final answer
+    if (typeof result === 'string') {
+        finalAnswer = result;
+    }
+    else if (result?.text) {
+        finalAnswer = result.text;
+    }
+    else if (result?.content) {
+        finalAnswer = String(result.content);
+    }
+    // Extract token usage
+    if (result?.usage) {
+        totalTokens = (result.usage.promptTokens ?? 0) +
+            (result.usage.completionTokens ?? 0);
+    }
+    // Extract steps if present
+    if (Array.isArray(result?.steps)) {
+        for (const step of result.steps) {
+            if (step.text) {
+                steps.push({
+                    step_id: stepId++,
+                    type: 'thought',
+                    content: step.text,
+                });
+            }
+        }
+    }
+    // Extract tool calls if present
+    if (Array.isArray(result?.toolCalls)) {
+        for (const tc of result.toolCalls) {
+            steps.push({
+                step_id: stepId++,
+                type: 'tool_call',
+                tool: tc.toolName ?? tc.name ?? 'unknown_tool',
+                input: tc.args ?? tc.input ?? {},
+                output: { result: String(tc.result ?? '') },
+                duration_ms: 0,
+            });
+        }
+    }
+    // Extract tool results from steps if toolCalls not separate
+    if (Array.isArray(result?.toolResults)) {
+        for (const tr of result.toolResults) {
+            steps.push({
+                step_id: stepId++,
+                type: 'tool_call',
+                tool: tr.toolName ?? 'unknown_tool',
+                input: tr.args ?? {},
+                output: { result: String(tr.result ?? '') },
+                duration_ms: 0,
+            });
+        }
+    }
+    const trace = {
+        evalforge_version: '0.1',
+        trace_id: `mastra-${Date.now()}`,
+        timestamp: new Date().toISOString(),
+        metadata: {
+            framework: 'mastra',
+            model,
+            agent_name: agentName,
+            duration_ms: 0,
+            total_tokens: totalTokens,
+        },
+        input: {
+            user: question,
+            system: '',
+        },
+        steps,
+        output: {
+            answer: finalAnswer,
+        },
+        eval_hints: {
+            expected_tools: expectedTools,
+            expected_answer: null,
+            context_documents: [],
+        },
+    };
+    const tmp = path.join(os.tmpdir(), `evalforge_mastra_${Date.now()}.json`);
+    fs.writeFileSync(tmp, JSON.stringify(trace, null, 2));
+    return tmp;
+}
+//# sourceMappingURL=mastra.js.map

package/dist/adapters/mastra.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mastra.js","sourceRoot":"","sources":["../../src/adapters/mastra.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcH,gCA+GC;AA3HD,uCAAyB;AACzB,uCAAyB;AACzB,2CAA6B;AAU7B,SAAgB,UAAU,CACxB,MAAW,EACX,UAAgC,EAAE;IAElC;;;;;;;;OAQG;IAEH,MAAM,EACJ,SAAS,GAAG,cAAc,EAC1B,KAAK,GAAG,SAAS,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,EAAE,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,uBAAuB;IACvB,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,WAAW,GAAG,MAAM,CAAC;IACvB,CAAC;SAAM,IAAI,MAAM,EAAE,IAAI,EAAE,CAAC;QACxB,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,CAAC;SAAM,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QAC3B,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,sBAAsB;IACtB,IAAI,MAAM,EAAE,KAAK,EAAE,CAAC;QAClB,WAAW,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;YAChC,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,2BAA2B;IAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,KAAK,CAAC,IAAI,CAAC;oBACT,OAAO,EAAE,MAAM,EAAE;oBACjB,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI,CAAC,IAAI;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;QACrC,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,MAAM,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,IAAI,cAAc;gBAC9C,KAAK,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE;gBAChC,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE;gBAC3C,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4DAA4D;IAC5D,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,CAAC;QACvC,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,MAAM,EAAE;gBACjB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,EAAE,CAAC,QAAQ,IAAI,cAAc;gBACnC,KAAK,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE;gBACpB,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE;gBAC3C,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAU;QACnB,iBAAiB,EAAE,KAAK;QACxB,QAAQ,EAAE,UAAU,IAAI,CAAC,GAAG,EAAE,EAAE;QAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE;YACR,SAAS,EAAE,QAAQ;YACnB,KAAK;YACL,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,WAAW;SAC1B;QACD,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;SACX;QACD,KAAK;QACL,MAAM,EAAE;YACN,MAAM,EAAE,WAAW;SACpB;QACD,UAAU,EAAE;YACV,cAAc,EAAE,aAAa;YAC7B,eAAe,EAAE,IAAI;YACrB,iBAAiB,EAAE,EAAE;SACtB;KACF,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,oBAAoB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1E,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACtD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import { EvalResult, RunOptions } from './types';
 export { EvalResult, MetricResult, RunOptions, Trace } from './types';
+export * from './adapters';
 export declare function run(tracePath: string, options: RunOptions): EvalResult;
 export declare function demo(): EvalResult;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAS,MAAM,SAAS,CAAC;AAExD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAkCtE,wBAAgB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,GAAG,UAAU,CAmBtE;AAED,wBAAgB,IAAI,IAAI,UAAU,CAuCjC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAS,MAAM,SAAS,CAAC;AAExD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AACtE,cAAc,YAAY,CAAC;AAkC3B,wBAAgB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,GAAG,UAAU,CAmBtE;AAED,wBAAgB,IAAI,IAAI,UAAU,CAuCjC"}

package/dist/index.js

@@ -32,6 +32,9 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.run = run;
 exports.demo = demo;
@@ -40,6 +43,7 @@ const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
 const binary_1 = require("./binary");
+__exportStar(require("./adapters"), exports);
 function parseOutput(output) {
     const metrics = [];
     const lines = output.split('\n');

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,kBAmBC;AAED,oBAuCC;AArGD,iDAA0C;AAC1C,uCAAyB;AACzB,uCAAyB;AACzB,2CAA6B;AAC7B,qCAAsC;AAKtC,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,aAAa,GAAG,IAAI,CAAC;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7B,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;YAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzE,IAAI,IAAI,KAAK,eAAe;YAAE,aAAa,GAAG,KAAK,CAAC;QAEpD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnE,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;gBACvD,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;gBAC5C,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;gBACtB,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,MAAM;gBACjC,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;AAChE,CAAC;AAED,SAAgB,GAAG,CAAC,SAAiB,EAAE,OAAmB;IACxD,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAC5B,MAAM,IAAI,GAAG;QACX,KAAK;QACL,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;QACtC,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;KAChD,CAAC;IAEF,IAAI,OAAO,CAAC,IAAI;QAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAAC,CAAC;IAEzE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC/B,IAAI,OAAO,CAAC,MAAM;QAAE,GAAG,CAAC,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAE3D,MAAM,MAAM,GAAG,IAAA,yBAAS,EAAC,MAAM,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,KAAK;QAAE,MAAM,MAAM,CAAC,KAAK,CAAC;IAErC,OAAO,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAgB,IAAI;IAClB,MAAM,KAAK,GAAU;QACnB,iBAAiB,EAAE,KAAK;QACxB,QAAQ,EAAE,aAAa;QACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE;YACR,SAAS,EAAE,eAAe;YAC1B,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,YAAY;YACxB,WAAW,EAAE,IAAI;YACjB,YAAY,EAAE,GAAG;SAClB;QACD,KAAK,EAAE;YACL,IAAI,EAAE,mCAAmC;YACzC,MAAM,EAAE,8BAA8B;SACvC;QACD,KAAK,EAAE;YACL,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,yBAAyB,EAAE;YACnE;gBACE,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE;gBACxC,MAAM,EAAE,EAAE,MAAM,EAAE,uCAAuC,EAAE;gBAC3D,WAAW,EAAE,GAAG;aACjB;SACF;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,uCAAuC,EAAE;QAC3D,UAAU,EAAE;YACV,cAAc,EAAE,CAAC,YAAY,CAAC;YAC9B,eAAe,EAAE,UAAU;YAC3B,iBAAiB,EAAE,EAAE;SACtB;KACF,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAE7C,OAAO,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7D,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,kBAmBC;AAED,oBAuCC;AAtGD,iDAA0C;AAC1C,uCAAyB;AACzB,uCAAyB;AACzB,2CAA6B;AAC7B,qCAAsC;AAItC,6CAA2B;AAE3B,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,aAAa,GAAG,IAAI,CAAC;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7B,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;YAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzE,IAAI,IAAI,KAAK,eAAe;YAAE,aAAa,GAAG,KAAK,CAAC;QAEpD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnE,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;gBACvD,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;gBAC5C,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;gBACtB,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,MAAM;gBACjC,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;AAChE,CAAC;AAED,SAAgB,GAAG,CAAC,SAAiB,EAAE,OAAmB;IACxD,MAAM,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IAC5B,MAAM,IAAI,GAAG;QACX,KAAK;QACL,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;QACtC,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;KAChD,CAAC;IAEF,IAAI,OAAO,CAAC,IAAI;QAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAAC,CAAC;IAEzE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC/B,IAAI,OAAO,CAAC,MAAM;QAAE,GAAG,CAAC,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAE3D,MAAM,MAAM,GAAG,IAAA,yBAAS,EAAC,MAAM,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,KAAK;QAAE,MAAM,MAAM,CAAC,KAAK,CAAC;IAErC,OAAO,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAgB,IAAI;IAClB,MAAM,KAAK,GAAU;QACnB,iBAAiB,EAAE,KAAK;QACxB,QAAQ,EAAE,aAAa;QACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE;YACR,SAAS,EAAE,eAAe;YAC1B,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,YAAY;YACxB,WAAW,EAAE,IAAI;YACjB,YAAY,EAAE,GAAG;SAClB;QACD,KAAK,EAAE;YACL,IAAI,EAAE,mCAAmC;YACzC,MAAM,EAAE,8BAA8B;SACvC;QACD,KAAK,EAAE;YACL,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,yBAAyB,EAAE;YACnE;gBACE,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE;gBACxC,MAAM,EAAE,EAAE,MAAM,EAAE,uCAAuC,EAAE;gBAC3D,WAAW,EAAE,GAAG;aACjB;SACF;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,uCAAuC,EAAE;QAC3D,UAAU,EAAE;YACV,cAAc,EAAE,CAAC,YAAY,CAAC;YAC9B,eAAe,EAAE,UAAU;YAC3B,iBAAiB,EAAE,EAAE;SACtB;KACF,CAAC;IAEF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAE7C,OAAO,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7D,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "evalforge",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "Framework-agnostic LLM agent evaluation harness",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/adapters/index.ts

@@ -0,0 +1 @@
+export { fromMastra } from './mastra';

package/src/adapters/mastra.ts

@@ -0,0 +1,139 @@
+/**
+ * Mastra adapter for EvalForge.
+ *
+ * Usage:
+ *   import { Agent } from '@mastra/core';
+ *   import { fromMastra } from 'evalforge/adapters/mastra';
+ *   import { run } from 'evalforge';
+ *
+ *   const agent = new Agent({ name: 'my-agent', ... });
+ *   const result = await agent.generate('What is the capital of France?');
+ *
+ *   const tracePath = fromMastra(result, { agentName: 'my-agent', model: 'gpt-4o' });
+ *   const evalResult = run(tracePath, { metrics: ['faithfulness'] });
+ */
+
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { Trace } from '../types';
+
+export interface MastraAdapterOptions {
+  agentName?: string;
+  model?: string;
+  question?: string;
+  expectedTools?: string[];
+}
+
+export function fromMastra(
+  result: any,
+  options: MastraAdapterOptions = {}
+): string {
+  /**
+   * Convert Mastra agent.generate() result to EvalForge trace JSON file.
+   *
+   * Supports:
+   * - result.text — final answer string
+   * - result.steps — array of reasoning steps
+   * - result.toolCalls — array of tool invocations
+   * - result.usage — token usage stats
+   */
+
+  const {
+    agentName = 'mastra-agent',
+    model = 'unknown',
+    question = '',
+    expectedTools = [],
+  } = options;
+
+  const steps: Trace['steps'] = [];
+  let stepId = 1;
+  let finalAnswer = '';
+  let totalTokens = 0;
+
+  // Extract final answer
+  if (typeof result === 'string') {
+    finalAnswer = result;
+  } else if (result?.text) {
+    finalAnswer = result.text;
+  } else if (result?.content) {
+    finalAnswer = String(result.content);
+  }
+
+  // Extract token usage
+  if (result?.usage) {
+    totalTokens = (result.usage.promptTokens ?? 0) +
+                  (result.usage.completionTokens ?? 0);
+  }
+
+  // Extract steps if present
+  if (Array.isArray(result?.steps)) {
+    for (const step of result.steps) {
+      if (step.text) {
+        steps.push({
+          step_id: stepId++,
+          type: 'thought',
+          content: step.text,
+        });
+      }
+    }
+  }
+
+  // Extract tool calls if present
+  if (Array.isArray(result?.toolCalls)) {
+    for (const tc of result.toolCalls) {
+      steps.push({
+        step_id: stepId++,
+        type: 'tool_call',
+        tool: tc.toolName ?? tc.name ?? 'unknown_tool',
+        input: tc.args ?? tc.input ?? {},
+        output: { result: String(tc.result ?? '') },
+        duration_ms: 0,
+      });
+    }
+  }
+
+  // Extract tool results from steps if toolCalls not separate
+  if (Array.isArray(result?.toolResults)) {
+    for (const tr of result.toolResults) {
+      steps.push({
+        step_id: stepId++,
+        type: 'tool_call',
+        tool: tr.toolName ?? 'unknown_tool',
+        input: tr.args ?? {},
+        output: { result: String(tr.result ?? '') },
+        duration_ms: 0,
+      });
+    }
+  }
+
+  const trace: Trace = {
+    evalforge_version: '0.1',
+    trace_id: `mastra-${Date.now()}`,
+    timestamp: new Date().toISOString(),
+    metadata: {
+      framework: 'mastra',
+      model,
+      agent_name: agentName,
+      duration_ms: 0,
+      total_tokens: totalTokens,
+    },
+    input: {
+      user: question,
+      system: '',
+    },
+    steps,
+    output: {
+      answer: finalAnswer,
+    },
+    eval_hints: {
+      expected_tools: expectedTools,
+      expected_answer: null,
+      context_documents: [],
+    },
+  };
+
+  const tmp = path.join(os.tmpdir(), `evalforge_mastra_${Date.now()}.json`);
+  fs.writeFileSync(tmp, JSON.stringify(trace, null, 2));
+  return tmp;
+}

package/src/index.ts

@@ -6,6 +6,7 @@ import { findBinary } from './binary';
 import { EvalResult, RunOptions, Trace } from './types';
 
 export { EvalResult, MetricResult, RunOptions, Trace } from './types';
+export * from './adapters';
 
 function parseOutput(output: string): EvalResult {
   const metrics = [];

package/tests/adapters.test.js

@@ -0,0 +1,55 @@
+const { test } = require('node:test');
+const assert = require('node:assert');
+const path = require('path');
+const fs = require('fs');
+
+const { fromMastra } = require('../dist/adapters/index.js');
+
+test('fromMastra with text result', () => {
+  const result = { text: 'The capital of France is Paris.' };
+  const tracePath = fromMastra(result, {
+    agentName: 'test-agent',
+    model: 'gpt-4o',
+    question: 'What is the capital of France?'
+  });
+  const trace = JSON.parse(fs.readFileSync(tracePath, 'utf8'));
+  assert.strictEqual(trace.metadata.framework, 'mastra');
+  assert.strictEqual(trace.output.answer, 'The capital of France is Paris.');
+  assert.strictEqual(trace.input.user, 'What is the capital of France?');
+});
+
+test('fromMastra with tool calls', () => {
+  const result = {
+    text: 'Canberra is the capital.',
+    toolCalls: [{
+      toolName: 'web_search',
+      args: { query: 'capital of Australia' },
+      result: 'Canberra is the capital of Australia.'
+    }]
+  };
+  const tracePath = fromMastra(result, {
+    question: 'What is the capital of Australia?',
+    expectedTools: ['web_search']
+  });
+  const trace = JSON.parse(fs.readFileSync(tracePath, 'utf8'));
+  assert.strictEqual(trace.steps.length, 1);
+  assert.strictEqual(trace.steps[0].type, 'tool_call');
+  assert.strictEqual(trace.steps[0].tool, 'web_search');
+  assert.deepStrictEqual(trace.eval_hints.expected_tools, ['web_search']);
+});
+
+test('fromMastra with string result', () => {
+  const tracePath = fromMastra('Direct answer.', { agentName: 'simple' });
+  const trace = JSON.parse(fs.readFileSync(tracePath, 'utf8'));
+  assert.strictEqual(trace.output.answer, 'Direct answer.');
+});
+
+test('fromMastra with token usage', () => {
+  const result = {
+    text: 'Answer.',
+    usage: { promptTokens: 100, completionTokens: 50 }
+  };
+  const tracePath = fromMastra(result);
+  const trace = JSON.parse(fs.readFileSync(tracePath, 'utf8'));
+  assert.strictEqual(trace.metadata.total_tokens, 150);
+});