armature-agent-eval 0.1.0

package/dist/golden.d.ts

@@ -0,0 +1,7 @@
+import type { AgentMessage } from '@armature/core';
+import type { GoldenTranscriptOptions, GoldenTranscriptResult } from './types.js';
+/**
+ * Compare a transcript against a golden reference transcript.
+ */
+export declare function compareGoldenTranscript(actual: AgentMessage[], expected: AgentMessage[], options?: GoldenTranscriptOptions): GoldenTranscriptResult;
+//# sourceMappingURL=golden.d.ts.map

package/dist/golden.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden.d.ts","sourceRoot":"","sources":["../src/golden.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,KAAK,EAAc,uBAAuB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AA6B7F;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,YAAY,EAAE,EACtB,QAAQ,EAAE,YAAY,EAAE,EACxB,OAAO,GAAE,uBAA4B,GACpC,sBAAsB,CAmCxB"}

package/dist/golden.js

@@ -0,0 +1,63 @@
+import { collectToolCalls } from '@open-agent-sdk/core';
+function messageToFingerprint(msg, options) {
+    if (options.toolNamesOnly) {
+        const calls = collectToolCalls(msg);
+        if (calls.length > 0) {
+            return `${msg.role}:tools:[${calls.map(c => c.name).join(',')}]`;
+        }
+    }
+    const parts = 'parts' in msg ? (msg.parts ?? []) : [];
+    const texts = parts
+        .map(p => {
+        if (p.type === 'text')
+            return p.text;
+        if (p.type === 'tool_call')
+            return `call:${p.toolCall.name}`;
+        if (p.type === 'tool_result')
+            return `result:${p.toolResult.toolCallId}`;
+        if (p.type === 'reasoning')
+            return `reasoning`;
+        if (p.type === 'attachment')
+            return `attachment:${p.mediaType}`;
+        return '';
+    })
+        .join('|');
+    const content = options.ignoreWhitespace ? texts.replace(/\s+/g, ' ').trim() : texts;
+    return `${msg.role}:${content}`;
+}
+/**
+ * Compare a transcript against a golden reference transcript.
+ */
+export function compareGoldenTranscript(actual, expected, options = {}) {
+    const ignoreRoles = new Set(options.ignoreRoles ?? []);
+    const filteredActual = actual.filter(m => !ignoreRoles.has(m.role));
+    const filteredExpected = expected.filter(m => !ignoreRoles.has(m.role));
+    const actualFPs = filteredActual.map(m => messageToFingerprint(m, options));
+    const expectedFPs = filteredExpected.map(m => messageToFingerprint(m, options));
+    const diffs = [];
+    const maxLen = Math.max(actualFPs.length, expectedFPs.length);
+    let matches = 0;
+    for (let i = 0; i < maxLen; i++) {
+        const a = actualFPs[i];
+        const e = expectedFPs[i];
+        if (a === undefined) {
+            diffs.push({ index: i, type: 'missing', expected: e });
+        }
+        else if (e === undefined) {
+            diffs.push({ index: i, type: 'extra', actual: a });
+        }
+        else if (a !== e) {
+            diffs.push({ index: i, type: 'mismatch', expected: e, actual: a });
+        }
+        else {
+            matches++;
+        }
+    }
+    const similarity = maxLen > 0 ? matches / maxLen : 1;
+    return {
+        passed: diffs.length === 0,
+        similarity,
+        diffs,
+    };
+}
+//# sourceMappingURL=golden.js.map

package/dist/golden.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden.js","sourceRoot":"","sources":["../src/golden.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAIvD,SAAS,oBAAoB,CAC3B,GAAiB,EACjB,OAAgC;IAEhC,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAA;QACnC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,GAAG,GAAG,CAAC,IAAI,WAAW,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;QAClE,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACrD,MAAM,KAAK,GAAG,KAAK;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE;QACP,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,CAAC,CAAC,IAAI,CAAA;QACpC,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;QAC5D,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa;YAAE,OAAO,UAAU,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,CAAA;QACxE,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO,WAAW,CAAA;QAC9C,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO,cAAc,CAAC,CAAC,SAAS,EAAE,CAAA;QAC/D,OAAO,EAAE,CAAA;IACX,CAAC,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAA;IAEZ,MAAM,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;IACpF,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,OAAO,EAAE,CAAA;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,MAAsB,EACtB,QAAwB,EACxB,UAAmC,EAAE;IAErC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;IAEtD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAA0B,CAAC,CAAC,CAAA;IACzF,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAA0B,CAAC,CAAC,CAAA;IAE7F,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAA;IAC3E,MAAM,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAA;IAE/E,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;IAC7D,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QACtB,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAExB,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAA;QACxD,CAAC;aAAM,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAA;QACpD,CAAC;aAAM,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAA;QACpE,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEpD,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC;QAC1B,UAAU;QACV,KAAK;KACN,CAAA;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { assertTrajectory, extractToolCallSequence } from './trajectory.js';
+export { validateToolConstraints } from './toolValidation.js';
+export { compareGoldenTranscript } from './golden.js';
+export { runEval, runEvalSuite } from './runner.js';
+export type { EvalCase, EvalResult, GoldenDiff, GoldenTranscriptOptions, GoldenTranscriptResult, ToolCallConstraint, ToolCallValidationResult, TrajectoryAssertion, TrajectoryResult, TrajectoryStep, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAEnD,YAAY,EACV,QAAQ,EACR,UAAU,EACV,UAAU,EACV,uBAAuB,EACvB,sBAAsB,EACtB,kBAAkB,EAClB,wBAAwB,EACxB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA"}

package/dist/index.js

@@ -0,0 +1,5 @@
+export { assertTrajectory, extractToolCallSequence } from './trajectory.js';
+export { validateToolConstraints } from './toolValidation.js';
+export { compareGoldenTranscript } from './golden.js';
+export { runEval, runEvalSuite } from './runner.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA"}

package/dist/runner.d.ts

@@ -0,0 +1,10 @@
+import type { EvalCase, EvalResult } from './types.js';
+/**
+ * Run a full eval case against a session transcript.
+ */
+export declare function runEval(evalCase: EvalCase): EvalResult;
+/**
+ * Run multiple eval cases and return all results.
+ */
+export declare function runEvalSuite(cases: EvalCase[]): EvalResult[];
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAKtD;;GAEG;AACH,wBAAgB,OAAO,CAAC,QAAQ,EAAE,QAAQ,GAAG,UAAU,CA6BtD;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,CAE5D"}

package/dist/runner.js

@@ -0,0 +1,32 @@
+import { assertTrajectory } from './trajectory.js';
+import { validateToolConstraints } from './toolValidation.js';
+import { compareGoldenTranscript } from './golden.js';
+/**
+ * Run a full eval case against a session transcript.
+ */
+export function runEval(evalCase) {
+    const trajectoryResults = (evalCase.trajectories ?? []).map(assertion => assertTrajectory(evalCase.transcript, assertion));
+    const toolValidationResults = evalCase.toolConstraints
+        ? validateToolConstraints(evalCase.transcript, evalCase.toolConstraints)
+        : [];
+    const goldenResult = evalCase.goldenTranscript
+        ? compareGoldenTranscript(evalCase.transcript, evalCase.goldenTranscript, evalCase.goldenOptions)
+        : undefined;
+    const passed = trajectoryResults.every(r => r.passed) &&
+        toolValidationResults.every(r => r.passed) &&
+        (goldenResult?.passed ?? true);
+    return {
+        name: evalCase.name,
+        passed,
+        trajectoryResults,
+        toolValidationResults,
+        goldenResult,
+    };
+}
+/**
+ * Run multiple eval cases and return all results.
+ */
+export function runEvalSuite(cases) {
+    return cases.map(runEval);
+}
+//# sourceMappingURL=runner.js.map

package/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAA;AAErD;;GAEG;AACH,MAAM,UAAU,OAAO,CAAC,QAAkB;IACxC,MAAM,iBAAiB,GAAG,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CACtE,gBAAgB,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CACjD,CAAA;IAED,MAAM,qBAAqB,GAAG,QAAQ,CAAC,eAAe;QACpD,CAAC,CAAC,uBAAuB,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,eAAe,CAAC;QACxE,CAAC,CAAC,EAAE,CAAA;IAEN,MAAM,YAAY,GAAG,QAAQ,CAAC,gBAAgB;QAC5C,CAAC,CAAC,uBAAuB,CACrB,QAAQ,CAAC,UAAU,EACnB,QAAQ,CAAC,gBAAgB,EACzB,QAAQ,CAAC,aAAa,CACvB;QACH,CAAC,CAAC,SAAS,CAAA;IAEb,MAAM,MAAM,GACV,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACtC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,CAAC,YAAY,EAAE,MAAM,IAAI,IAAI,CAAC,CAAA;IAEhC,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,MAAM;QACN,iBAAiB;QACjB,qBAAqB;QACrB,YAAY;KACb,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAiB;IAC5C,OAAO,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;AAC3B,CAAC"}

package/dist/toolValidation.d.ts

@@ -0,0 +1,7 @@
+import type { AgentMessage } from '@armature/core';
+import type { ToolCallConstraint, ToolCallValidationResult } from './types.js';
+/**
+ * Validate tool-call constraints against a session transcript.
+ */
+export declare function validateToolConstraints(transcript: AgentMessage[], constraints: ToolCallConstraint[]): ToolCallValidationResult[];
+//# sourceMappingURL=toolValidation.d.ts.map

package/dist/toolValidation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolValidation.d.ts","sourceRoot":"","sources":["../src/toolValidation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAA;AAE9E;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,YAAY,EAAE,EAC1B,WAAW,EAAE,kBAAkB,EAAE,GAChC,wBAAwB,EAAE,CAqE5B"}

package/dist/toolValidation.js

@@ -0,0 +1,56 @@
+import { collectToolCalls } from '@open-agent-sdk/core';
+/**
+ * Validate tool-call constraints against a session transcript.
+ */
+export function validateToolConstraints(transcript, constraints) {
+    const allCalls = transcript.flatMap(msg => collectToolCalls(msg).map(tc => ({ name: tc.name, input: tc.input })));
+    return constraints.map(constraint => {
+        const violations = [];
+        const matchingCalls = allCalls.filter(c => c.name === constraint.toolName);
+        // Forbidden check
+        if (constraint.forbidden && matchingCalls.length > 0) {
+            violations.push(`Tool "${constraint.toolName}" was called ${matchingCalls.length} time(s) but is forbidden`);
+        }
+        // Max calls check
+        if (constraint.maxCalls !== undefined &&
+            matchingCalls.length > constraint.maxCalls) {
+            violations.push(`Tool "${constraint.toolName}" was called ${matchingCalls.length} time(s), exceeds max ${constraint.maxCalls}`);
+        }
+        // Required input check
+        if (constraint.requiredInput) {
+            for (let i = 0; i < matchingCalls.length; i++) {
+                const call = matchingCalls[i];
+                const input = call.input;
+                if (!input || typeof input !== 'object') {
+                    violations.push(`Tool "${constraint.toolName}" call #${i} has non-object input`);
+                    continue;
+                }
+                for (const [key, value] of Object.entries(constraint.requiredInput)) {
+                    if (input[key] !== value) {
+                        violations.push(`Tool "${constraint.toolName}" call #${i} missing required input ${key}=${JSON.stringify(value)}`);
+                    }
+                }
+            }
+        }
+        // Forbidden input check
+        if (constraint.forbiddenInput) {
+            for (let i = 0; i < matchingCalls.length; i++) {
+                const call = matchingCalls[i];
+                const input = call.input;
+                if (!input || typeof input !== 'object')
+                    continue;
+                for (const [key, value] of Object.entries(constraint.forbiddenInput)) {
+                    if (input[key] === value) {
+                        violations.push(`Tool "${constraint.toolName}" call #${i} contains forbidden input ${key}=${JSON.stringify(value)}`);
+                    }
+                }
+            }
+        }
+        return {
+            toolName: constraint.toolName,
+            passed: violations.length === 0,
+            violations,
+        };
+    });
+}
+//# sourceMappingURL=toolValidation.js.map

package/dist/toolValidation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolValidation.js","sourceRoot":"","sources":["../src/toolValidation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAIvD;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,UAA0B,EAC1B,WAAiC;IAEjC,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CACxC,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CACtE,CAAA;IAED,OAAO,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;QAClC,MAAM,UAAU,GAAa,EAAE,CAAA;QAC/B,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,QAAQ,CAAC,CAAA;QAE1E,kBAAkB;QAClB,IAAI,UAAU,CAAC,SAAS,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,UAAU,CAAC,IAAI,CACb,SAAS,UAAU,CAAC,QAAQ,gBAAgB,aAAa,CAAC,MAAM,2BAA2B,CAC5F,CAAA;QACH,CAAC;QAED,kBAAkB;QAClB,IACE,UAAU,CAAC,QAAQ,KAAK,SAAS;YACjC,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,QAAQ,EAC1C,CAAC;YACD,UAAU,CAAC,IAAI,CACb,SAAS,UAAU,CAAC,QAAQ,gBAAgB,aAAa,CAAC,MAAM,yBAAyB,UAAU,CAAC,QAAQ,EAAE,CAC/G,CAAA;QACH,CAAC;QAED,uBAAuB;QACvB,IAAI,UAAU,CAAC,aAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;gBAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAuC,CAAA;gBAC1D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACxC,UAAU,CAAC,IAAI,CACb,SAAS,UAAU,CAAC,QAAQ,WAAW,CAAC,uBAAuB,CAChE,CAAA;oBACD,SAAQ;gBACV,CAAC;gBACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;oBACpE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC;wBACzB,UAAU,CAAC,IAAI,CACb,SAAS,UAAU,CAAC,QAAQ,WAAW,CAAC,2BAA2B,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAClG,CAAA;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;YAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;gBAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAuC,CAAA;gBAC1D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;oBAAE,SAAQ;gBACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;oBACrE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC;wBACzB,UAAU,CAAC,IAAI,CACb,SAAS,UAAU,CAAC,QAAQ,WAAW,CAAC,6BAA6B,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CACpG,CAAA;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,MAAM,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC;YAC/B,UAAU;SACX,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}

package/dist/trajectory.d.ts

@@ -0,0 +1,14 @@
+import type { AgentMessage } from '@armature/core';
+import type { TrajectoryAssertion, TrajectoryResult } from './types.js';
+/**
+ * Extract the ordered list of tool call names from a session transcript.
+ */
+export declare function extractToolCallSequence(transcript: AgentMessage[]): Array<{
+    name: string;
+    input: unknown;
+}>;
+/**
+ * Assert that a session transcript matches an expected trajectory.
+ */
+export declare function assertTrajectory(transcript: AgentMessage[], assertion: TrajectoryAssertion): TrajectoryResult;
+//# sourceMappingURL=trajectory.d.ts.map

package/dist/trajectory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trajectory.d.ts","sourceRoot":"","sources":["../src/trajectory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,KAAK,EAAE,mBAAmB,EAAE,gBAAgB,EAAkB,MAAM,YAAY,CAAA;AAEvF;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,YAAY,EAAE,GACzB,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC,CAIzC;AAgBD;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,YAAY,EAAE,EAC1B,SAAS,EAAE,mBAAmB,GAC7B,gBAAgB,CAgElB"}

package/dist/trajectory.js

@@ -0,0 +1,80 @@
+import { collectToolCalls } from '@open-agent-sdk/core';
+/**
+ * Extract the ordered list of tool call names from a session transcript.
+ */
+export function extractToolCallSequence(transcript) {
+    return transcript.flatMap(msg => collectToolCalls(msg).map(tc => ({ name: tc.name, input: tc.input })));
+}
+function inputMatches(actual, pattern) {
+    if (typeof actual !== 'object' || actual === null)
+        return false;
+    const obj = actual;
+    return Object.entries(pattern).every(([key, value]) => {
+        if (typeof value === 'object' && value !== null) {
+            return inputMatches(obj[key], value);
+        }
+        return obj[key] === value;
+    });
+}
+/**
+ * Assert that a session transcript matches an expected trajectory.
+ */
+export function assertTrajectory(transcript, assertion) {
+    const calls = extractToolCallSequence(transcript);
+    const ordered = assertion.ordered !== false;
+    const missingSteps = [];
+    const matchedIndices = new Set();
+    let matchedSteps = 0;
+    if (ordered) {
+        let callIdx = 0;
+        for (const step of assertion.steps) {
+            let found = false;
+            const savedIdx = callIdx;
+            while (callIdx < calls.length) {
+                const call = calls[callIdx];
+                if (call.name === step.toolName &&
+                    (!step.inputMatch || inputMatches(call.input, step.inputMatch))) {
+                    matchedIndices.add(callIdx);
+                    matchedSteps++;
+                    callIdx++;
+                    found = true;
+                    break;
+                }
+                callIdx++;
+            }
+            if (!found) {
+                // Restore cursor so subsequent steps can still match
+                callIdx = savedIdx;
+                if (step.required !== false) {
+                    missingSteps.push(step);
+                }
+            }
+        }
+    }
+    else {
+        for (const step of assertion.steps) {
+            const idx = calls.findIndex((call, i) => !matchedIndices.has(i) &&
+                call.name === step.toolName &&
+                (!step.inputMatch || inputMatches(call.input, step.inputMatch)));
+            if (idx >= 0) {
+                matchedIndices.add(idx);
+                matchedSteps++;
+            }
+            else if (step.required !== false) {
+                missingSteps.push(step);
+            }
+        }
+    }
+    const extraCalls = calls
+        .filter((_, i) => !matchedIndices.has(i))
+        .map(c => c.name);
+    return {
+        name: assertion.name,
+        passed: missingSteps.length === 0,
+        matchedSteps,
+        totalSteps: assertion.steps.filter(s => s.required !== false).length,
+        missingSteps,
+        extraCalls,
+    };
+}
+//# sourceMappingURL=trajectory.js.map

package/dist/trajectory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trajectory.js","sourceRoot":"","sources":["../src/trajectory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAIvD;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,UAA0B;IAE1B,OAAO,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAC9B,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CACtE,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CACnB,MAAe,EACf,OAAgC;IAEhC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,KAAK,CAAA;IAC/D,MAAM,GAAG,GAAG,MAAiC,CAAA;IAC7C,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAChD,OAAO,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,KAAgC,CAAC,CAAA;QACjE,CAAC;QACD,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,CAAA;IAC3B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,UAA0B,EAC1B,SAA8B;IAE9B,MAAM,KAAK,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAAA;IACjD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,KAAK,KAAK,CAAA;IAE3C,MAAM,YAAY,GAAqB,EAAE,CAAA;IACzC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAA;IACxC,IAAI,YAAY,GAAG,CAAC,CAAA;IAEpB,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,KAAK,GAAG,KAAK,CAAA;YACjB,MAAM,QAAQ,GAAG,OAAO,CAAA;YACxB,OAAO,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAA;gBAC3B,IACE,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ;oBAC3B,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,EAC/D,CAAC;oBACD,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;oBAC3B,YAAY,EAAE,CAAA;oBACd,OAAO,EAAE,CAAA;oBACT,KAAK,GAAG,IAAI,CAAA;oBACZ,MAAK;gBACP,CAAC;gBACD,OAAO,EAAE,CAAA;YACX,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,qDAAqD;gBACrD,OAAO,GAAG,QAAQ,CAAA;gBAClB,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;oBAC5B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CACzB,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACV,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ;gBAC3B,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAClE,CAAA;YACD,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;gBACb,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACvB,YAAY,EAAE,CAAA;YAChB,CAAC;iBAAM,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACnC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,KAAK;SACrB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAEnB,OAAO;QACL,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,MAAM,EAAE,YAAY,CAAC,MAAM,KAAK,CAAC;QACjC,YAAY;QACZ,UAAU,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,MAAM;QACpE,YAAY;QACZ,UAAU;KACX,CAAA;AACH,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,81 @@
+import type { AgentMessage } from '@armature/core';
+export interface TrajectoryStep {
+    /** Expected tool name (e.g., 'Read', 'Bash', 'Edit') */
+    toolName: string;
+    /** Optional: partial match on tool input */
+    inputMatch?: Record<string, unknown>;
+    /** Optional: whether the step is required (default: true) */
+    required?: boolean;
+}
+export interface TrajectoryAssertion {
+    /** Human-readable name for this assertion */
+    name: string;
+    /** Expected sequence of tool calls */
+    steps: TrajectoryStep[];
+    /** Whether ordering must be exact (default: true) */
+    ordered?: boolean;
+}
+export interface TrajectoryResult {
+    name: string;
+    passed: boolean;
+    matchedSteps: number;
+    totalSteps: number;
+    missingSteps: TrajectoryStep[];
+    extraCalls: string[];
+}
+export interface ToolCallConstraint {
+    /** Tool name to constrain */
+    toolName: string;
+    /** Maximum number of times this tool may be called */
+    maxCalls?: number;
+    /** Tool must never be called */
+    forbidden?: boolean;
+    /** Partial input that must appear in every call to this tool */
+    requiredInput?: Record<string, unknown>;
+    /** Partial input that must never appear */
+    forbiddenInput?: Record<string, unknown>;
+}
+export interface ToolCallValidationResult {
+    toolName: string;
+    passed: boolean;
+    violations: string[];
+}
+export interface GoldenTranscriptOptions {
+    /** Ignore whitespace differences in text content */
+    ignoreWhitespace?: boolean;
+    /** Only compare tool call names and sequence, not arguments */
+    toolNamesOnly?: boolean;
+    /** Ignore specific message roles */
+    ignoreRoles?: Array<'system' | 'event'>;
+}
+export interface GoldenTranscriptResult {
+    passed: boolean;
+    similarity: number;
+    diffs: GoldenDiff[];
+}
+export interface GoldenDiff {
+    index: number;
+    type: 'missing' | 'extra' | 'mismatch';
+    expected?: string;
+    actual?: string;
+}
+export interface EvalCase {
+    name: string;
+    /** The session transcript to evaluate */
+    transcript: AgentMessage[];
+    /** Trajectory assertions to run */
+    trajectories?: TrajectoryAssertion[];
+    /** Tool-call constraints to validate */
+    toolConstraints?: ToolCallConstraint[];
+    /** Golden transcript to compare against */
+    goldenTranscript?: AgentMessage[];
+    goldenOptions?: GoldenTranscriptOptions;
+}
+export interface EvalResult {
+    name: string;
+    passed: boolean;
+    trajectoryResults: TrajectoryResult[];
+    toolValidationResults: ToolCallValidationResult[];
+    goldenResult?: GoldenTranscriptResult;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AAMxD,MAAM,WAAW,cAAc;IAC7B,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAA;IAChB,4CAA4C;IAC5C,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACpC,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAA;IACZ,sCAAsC;IACtC,KAAK,EAAE,cAAc,EAAE,CAAA;IACvB,qDAAqD;IACrD,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,cAAc,EAAE,CAAA;IAC9B,UAAU,EAAE,MAAM,EAAE,CAAA;CACrB;AAMD,MAAM,WAAW,kBAAkB;IACjC,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,gCAAgC;IAChC,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,gEAAgE;IAChE,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACvC,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACzC;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;IACf,UAAU,EAAE,MAAM,EAAE,CAAA;CACrB;AAMD,MAAM,WAAW,uBAAuB;IACtC,oDAAoD;IACpD,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,+DAA+D;IAC/D,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,oCAAoC;IACpC,WAAW,CAAC,EAAE,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAA;CACxC;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,OAAO,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,UAAU,EAAE,CAAA;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,SAAS,GAAG,OAAO,GAAG,UAAU,CAAA;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAMD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,yCAAyC;IACzC,UAAU,EAAE,YAAY,EAAE,CAAA;IAC1B,mCAAmC;IACnC,YAAY,CAAC,EAAE,mBAAmB,EAAE,CAAA;IACpC,wCAAwC;IACxC,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAA;IACtC,2CAA2C;IAC3C,gBAAgB,CAAC,EAAE,YAAY,EAAE,CAAA;IACjC,aAAa,CAAC,EAAE,uBAAuB,CAAA;CACxC;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,iBAAiB,EAAE,gBAAgB,EAAE,CAAA;IACrC,qBAAqB,EAAE,wBAAwB,EAAE,CAAA;IACjD,YAAY,CAAC,EAAE,sBAAsB,CAAA;CACtC"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "armature-agent-eval",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "armature-agent-core": "^0.1.0"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json"
+  },
+  "files": [
+    "dist"
+  ]
+}