@memberjunction/testing-engine 6.1.0-edge.5 → 6.1.0-edge.7

package/dist/eval/corpus.d.ts

@@ -0,0 +1,91 @@
+/**
+ * corpus.ts — the golden-file case format, its loader, and `anyOf`.
+ *
+ * Corpus cases are golden JSON files (test plan §4.2) rather than rows in a table, because inputs
+ * and expectations have to be **reviewable and diffable**: a change to what a case expects is a
+ * change to the yardstick, and that must show up in a code review rather than in a metadata push.
+ *
+ * The on-disk format is the plan's, verbatim — `{ "matcher": "regex", "pattern": … }` with an
+ * `optional` flag — while the evaluator's internal {@link ParamMatcher} is a discriminated union.
+ * This module is the translation, and it exists precisely so the two can differ: the file format
+ * is optimized for a human writing and reviewing it, the internal type for exhaustive checking.
+ *
+ * Loading VALIDATES. An unknown matcher name, a missing pattern, an unknown expectation kind — all
+ * throw here, at load time, naming the case. The alternative is a matcher that silently never
+ * matches, which would look exactly like a model failure in the results.
+ *
+ * FRAMEWORK-FREE (test plan §2.1) — imports nothing from the testing engine.
+ */
+import type { DecisionKind, ObservedDecision } from './decision.js';
+import type { ParamMatcher } from './matchers.js';
+import type { ChatMessage } from '@memberjunction/ai';
+import { type DecisionEvaluation, type DecisionExpectation } from './expectation.js';
+/** A parameter matcher as written in a golden file. */
+export interface CorpusParamMatcher {
+    matcher: 'exact' | 'regex' | 'oneOf' | 'contains' | 'nonEmpty' | 'numericTolerance' | 'typeOf' | 'absent';
+    value?: unknown;
+    values?: unknown[];
+    pattern?: string;
+    flags?: string;
+    /** For `numericTolerance`: the permitted absolute deviation from `value`. */
+    tolerance?: number;
+    /** For `typeOf`. */
+    type?: 'string' | 'number' | 'boolean' | 'object' | 'array';
+    /** Absent is acceptable; a present value is still checked. */
+    optional?: boolean;
+}
+/** One expected action in a golden file. */
+export interface CorpusExpectedAction {
+    name: string;
+    params?: Record<string, CorpusParamMatcher>;
+}
+/** One expectation in a golden file. `anyOf` nests alternatives. */
+export interface CorpusExpectation {
+    kind: DecisionKind | 'anyOf';
+    actions?: CorpusExpectedAction[];
+    subAgents?: string[];
+    message?: CorpusParamMatcher;
+    forbiddenActions?: string[];
+    ordered?: boolean;
+    allowAdditionalActions?: boolean;
+    /** Matchers keyed by dotted path into the payload change request, e.g. `newElements.findings`. */
+    payload?: Record<string, CorpusParamMatcher>;
+    /** Dotted paths the payload change must not touch (payload scoping). */
+    forbiddenPayloadPaths?: string[];
+    /** Alternatives, when `kind` is `anyOf`. The best-scoring branch wins. */
+    anyOf?: CorpusExpectation[];
+}
+/** A corpus case, as it appears on disk. */
+export interface CorpusCase {
+    id: string;
+    /** The agent whose prompt this case exercises. */
+    agent: string;
+    description: string;
+    input: {
+        payload?: Record<string, unknown>;
+        /** Prior turns. Multi-turn cases use the neutral tool form: assistant `toolCalls` + a `tool` turn of `tool_result` blocks; `encodeHistoryForArm` renders it per arm. */
+        conversationMessages?: ChatMessage[];
+        templateData?: Record<string, unknown>;
+        /** Forces the model's tool choice on native cells — 'none' for forced-control-flow cases, which production runs at 'none' on the final iteration. */
+        toolChoice?: 'auto' | 'none' | 'required';
+    };
+    expect: CorpusExpectation;
+    /** Free-form tags for slicing the corpus — category, provider focus, stage. */
+    tags?: string[];
+}
+/** Translates one on-disk matcher into the evaluator's union, rejecting anything malformed. */
+export declare function toParamMatcher(caseId: string, param: string, spec: CorpusParamMatcher): ParamMatcher;
+/** Translates a golden-file expectation into the evaluator's {@link DecisionExpectation}. */
+export declare function toDecisionExpectation(caseId: string, expect: CorpusExpectation): DecisionExpectation;
+/** Validates a parsed golden file and returns it typed. Throws with the case id on any problem. */
+export declare function parseCorpusCase(raw: unknown): CorpusCase;
+/**
+ * Scores an observation against a case's expectation, resolving `anyOf`.
+ *
+ * `anyOf` takes the **best-scoring** branch rather than the first passing one. Both give the same
+ * pass/fail, but the score differs when every branch fails, and reporting the closest miss is what
+ * makes a failure diagnosable: "expected chat OR action, got unparseable" is less useful than the
+ * branch that came nearest.
+ */
+export declare function evaluateCorpusExpectation(caseId: string, expect: CorpusExpectation, observed: ObservedDecision): DecisionEvaluation;
+//# sourceMappingURL=corpus.d.ts.map

package/dist/eval/corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../../src/eval/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAoB,YAAY,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAoB,KAAK,kBAAkB,EAAE,KAAK,mBAAmB,EAAuB,MAAM,eAAe,CAAC;AAEzH,uDAAuD;AACvD,MAAM,WAAW,kBAAkB;IAC/B,OAAO,EAAE,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC1G,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IAC5D,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,4CAA4C;AAC5C,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;CAC/C;AAED,oEAAoE;AACpE,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,oBAAoB,EAAE,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,OAAO,CAAC,EAAE,kBAAkB,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,kGAAkG;IAClG,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAC7C,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;IACjC,0EAA0E;IAC1E,KAAK,CAAC,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAED,4CAA4C;AAC5C,MAAM,WAAW,UAAU;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,kDAAkD;IAClD,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE;QACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAClC,wKAAwK;QACxK,oBAAoB,CAAC,EAAE,WAAW,EAAE,CAAC;QACrC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACvC,qJAAqJ;QACrJ,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;KAC7C,CAAC;IACF,MAAM,EAAE,iBAAiB,CAAC;IAC1B,+EAA+E;IAC/E,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAMD,+FAA+F;AAC/F,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,GAAG,YAAY,CA4CpG;AAqBD,6FAA6F;AAC7F,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,mBAAmB,CAkBpG;AAED,mGAAmG;AACnG,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU,CAsBxD;AAaD;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,QAAQ,EAAE,gBAAgB,GAAG,kBAAkB,CAUnI"}

package/dist/eval/corpus.js

@@ -0,0 +1,141 @@
+import { evaluateDecision } from './expectation.js';
+function fail(caseId, message) {
+    throw new Error(`Corpus case '${caseId}': ${message}`);
+}
+/** Translates one on-disk matcher into the evaluator's union, rejecting anything malformed. */
+export function toParamMatcher(caseId, param, spec) {
+    switch (spec.matcher) {
+        case 'exact':
+            return { kind: 'exact', value: spec.value };
+        case 'nonEmpty':
+            // A payload path may hold an array or object, not only a string.
+            return { kind: 'nonEmpty' };
+        case 'contains':
+            if (typeof spec.value !== 'string') {
+                fail(caseId, `param '${param}': 'contains' requires a string 'value'`);
+            }
+            return { kind: 'containsIgnoreCase', value: spec.value };
+        case 'oneOf':
+            if (!Array.isArray(spec.values) || spec.values.length === 0) {
+                fail(caseId, `param '${param}': 'oneOf' requires a non-empty 'values' array`);
+            }
+            return { kind: 'oneOf', values: spec.values };
+        case 'regex':
+            if (typeof spec.pattern !== 'string') {
+                fail(caseId, `param '${param}': 'regex' requires a 'pattern'`);
+            }
+            // Compile now so a bad pattern fails at load, not silently at scoring time.
+            try {
+                new RegExp(spec.pattern, spec.flags);
+            }
+            catch (error) {
+                fail(caseId, `param '${param}': invalid regex — ${error.message}`);
+            }
+            return { kind: 'regex', pattern: spec.pattern, flags: spec.flags };
+        case 'numericTolerance': {
+            if (typeof spec.value !== 'number' || typeof spec.tolerance !== 'number') {
+                fail(caseId, `param '${param}': 'numericTolerance' requires numeric 'value' and 'tolerance'`);
+            }
+            return { kind: 'numberRange', min: spec.value - spec.tolerance, max: spec.value + spec.tolerance };
+        }
+        case 'typeOf':
+            if (!spec.type) {
+                fail(caseId, `param '${param}': 'typeOf' requires a 'type'`);
+            }
+            return { kind: 'typeOf', type: spec.type };
+        case 'absent':
+            return { kind: 'absent' };
+        default:
+            fail(caseId, `param '${param}': unknown matcher '${String(spec.matcher)}'`);
+    }
+}
+const DECISION_KINDS = ['action', 'subAgent', 'chat', 'taskComplete', 'payloadChange', 'other', 'unparseable', 'empty'];
+function toParamExpectations(caseId, params) {
+    return Object.entries(params ?? {}).map(([param, spec]) => ({
+        param,
+        matcher: toParamMatcher(caseId, param, spec),
+        optional: spec.optional === true
+    }));
+}
+function toExpectedActions(caseId, actions) {
+    return actions?.map((action) => {
+        if (!action.name) {
+            fail(caseId, 'every expected action needs a name');
+        }
+        return { name: action.name, params: toParamExpectations(caseId, action.params) };
+    });
+}
+/** Translates a golden-file expectation into the evaluator's {@link DecisionExpectation}. */
+export function toDecisionExpectation(caseId, expect) {
+    if (!DECISION_KINDS.includes(expect.kind)) {
+        fail(caseId, `unknown expectation kind '${expect.kind}'`);
+    }
+    if (expect.kind === 'action' && !expect.actions?.length) {
+        fail(caseId, "kind 'action' requires at least one expected action");
+    }
+    return {
+        kind: expect.kind,
+        actions: toExpectedActions(caseId, expect.actions),
+        subAgents: expect.subAgents,
+        message: expect.message ? toParamMatcher(caseId, 'message', expect.message) : undefined,
+        forbiddenActions: expect.forbiddenActions,
+        ordered: expect.ordered,
+        allowAdditionalActions: expect.allowAdditionalActions,
+        payload: expect.payload ? toParamExpectations(caseId, expect.payload) : undefined,
+        forbiddenPayloadPaths: expect.forbiddenPayloadPaths
+    };
+}
+/** Validates a parsed golden file and returns it typed. Throws with the case id on any problem. */
+export function parseCorpusCase(raw) {
+    const record = raw;
+    if (!record || typeof record !== 'object') {
+        throw new Error('Corpus case must be a JSON object');
+    }
+    const id = record.id;
+    if (typeof id !== 'string' || id.length === 0) {
+        throw new Error("Corpus case must have a non-empty 'id'");
+    }
+    if (typeof record.agent !== 'string' || record.agent.length === 0) {
+        fail(id, "must name the 'agent' whose prompt it exercises");
+    }
+    if (!record.expect) {
+        fail(id, "must declare an 'expect'");
+    }
+    const toolChoice = record.input?.toolChoice;
+    if (toolChoice !== undefined && !['auto', 'none', 'required'].includes(String(toolChoice))) {
+        fail(id, `input.toolChoice must be 'auto' | 'none' | 'required', got '${String(toolChoice)}'`);
+    }
+    // Compile every matcher now: a corpus that loads is a corpus that can score.
+    validateExpectation(id, record.expect);
+    return { ...record, id, agent: record.agent, description: record.description ?? '', input: record.input ?? {}, expect: record.expect };
+}
+function validateExpectation(caseId, expect) {
+    if (expect.kind === 'anyOf') {
+        if (!expect.anyOf?.length) {
+            fail(caseId, "kind 'anyOf' requires a non-empty 'anyOf' array");
+        }
+        expect.anyOf.forEach((branch) => validateExpectation(caseId, branch));
+        return;
+    }
+    toDecisionExpectation(caseId, expect);
+}
+/**
+ * Scores an observation against a case's expectation, resolving `anyOf`.
+ *
+ * `anyOf` takes the **best-scoring** branch rather than the first passing one. Both give the same
+ * pass/fail, but the score differs when every branch fails, and reporting the closest miss is what
+ * makes a failure diagnosable: "expected chat OR action, got unparseable" is less useful than the
+ * branch that came nearest.
+ */
+export function evaluateCorpusExpectation(caseId, expect, observed) {
+    if (expect.kind !== 'anyOf') {
+        return evaluateDecision(toDecisionExpectation(caseId, expect), observed);
+    }
+    const branches = (expect.anyOf ?? []).map((branch) => evaluateCorpusExpectation(caseId, branch, observed));
+    const passing = branches.find((b) => b.passed);
+    if (passing) {
+        return passing;
+    }
+    return branches.reduce((best, current) => (current.score > best.score ? current : best), branches[0]);
+}
+//# sourceMappingURL=corpus.js.map

package/dist/eval/corpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../../src/eval/corpus.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,gBAAgB,EAA0E,MAAM,eAAe,CAAC;AA2DzH,SAAS,IAAI,CAAC,MAAc,EAAE,OAAe;IACzC,MAAM,IAAI,KAAK,CAAC,gBAAgB,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;AAC3D,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,KAAa,EAAE,IAAwB;IAClF,QAAQ,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,OAAO;YACR,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAChD,KAAK,UAAU;YACX,iEAAiE;YACjE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAChC,KAAK,UAAU;YACX,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACjC,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,yCAAyC,CAAC,CAAC;YAC3E,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC7D,KAAK,OAAO;YACR,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1D,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,gDAAgD,CAAC,CAAC;YAClF,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QAClD,KAAK,OAAO;YACR,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,iCAAiC,CAAC,CAAC;YACnE,CAAC;YACD,4EAA4E;YAC5E,IAAI,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,sBAAuB,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;YAClF,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QACvE,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACtB,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACvE,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,gEAAgE,CAAC,CAAC;YAClG,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACvG,CAAC;QACD,KAAK,QAAQ;YACT,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,+BAA+B,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/C,KAAK,QAAQ;YACT,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC9B;YACI,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,uBAAuB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpF,CAAC;AACL,CAAC;AAED,MAAM,cAAc,GAAmB,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;AAExI,SAAS,mBAAmB,CAAC,MAAc,EAAE,MAAsD;IAC/F,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACxD,KAAK;QACL,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC;QAC5C,QAAQ,EAAE,IAAI,CAAC,QAAQ,KAAK,IAAI;KACnC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAc,EAAE,OAA2C;IAClF,OAAO,OAAO,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,EAAE,oCAAoC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IACrF,CAAC,CAAC,CAAC;AACP,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,qBAAqB,CAAC,MAAc,EAAE,MAAyB;IAC3E,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAoB,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,MAAM,EAAE,6BAA6B,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QACtD,IAAI,CAAC,MAAM,EAAE,qDAAqD,CAAC,CAAC;IACxE,CAAC;IACD,OAAO;QACH,IAAI,EAAE,MAAM,CAAC,IAAoB;QACjC,OAAO,EAAE,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC;QAClD,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;QACvF,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,sBAAsB,EAAE,MAAM,CAAC,sBAAsB;QACrD,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;QACjF,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;KACtD,CAAC;AACN,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,eAAe,CAAC,GAAY;IACxC,MAAM,MAAM,GAAG,GAAiC,CAAC;IACjD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;IACrB,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,EAAE,EAAE,iDAAiD,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACjB,IAAI,CAAC,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,UAAU,GAAI,MAAM,CAAC,KAA8C,EAAE,UAAU,CAAC;IACtF,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QACzF,IAAI,CAAC,EAAE,EAAE,+DAA+D,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACnG,CAAC;IACD,6EAA6E;IAC7E,mBAAmB,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACvC,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAgB,CAAC;AACzJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc,EAAE,MAAyB;IAClE,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,EAAE,iDAAiD,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QACtE,OAAO;IACX,CAAC;IACD,qBAAqB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CAAC,MAAc,EAAE,MAAyB,EAAE,QAA0B;IAC3G,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC1B,OAAO,gBAAgB,CAAC,qBAAqB,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC3G,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1G,CAAC"}

package/dist/eval/decision.d.ts

@@ -0,0 +1,156 @@
+/**
+ * decision.ts — normalizing what an agent DECIDED, independently of how it said so.
+ *
+ * This is the load-bearing idea of the whole eval harness (test plan §1). A corpus case states the
+ * decision it expects — "invoke `Run Ad-hoc Query` with these params" — and never how the wire
+ * encodes it. Two encodings exist today and a third is coming:
+ *
+ *   envelope  — a `LoopAgentResponse` JSON object with `nextStep.actions[]`
+ *   native    — provider tool calls on the assistant turn
+ *
+ * Both collapse here into one {@link ObservedDecision} before anything is scored. That is what
+ * makes a baseline run and a native run *directly comparable on identical cases*:
+ * the only thing that changed between them is the model's behavior, not the yardstick.
+ *
+ * FRAMEWORK-FREE (test plan §2.1). Nothing in `src/eval/` imports from the testing engine — not
+ * `IOracle`, not `TestEngine`, not the driver types. The oracles in `src/oracles/` are thin
+ * wrappers over these functions, and a bespoke runner could call them directly. That rule exists so
+ * the documented fallback (§2.1, option B) stays cheap; it also happens to make every line here
+ * unit-testable without a database.
+ */
+/**
+ * What kind of decision the agent made, in the vocabulary the corpus uses.
+ *
+ * These are deliberately the *decision* kinds, not the `nextStep.type` values — `taskComplete` is a
+ * decision but not a step type, and `Retry`/`ForEach`/`While` are control flow the corpus does not
+ * assert on. Anything the normalizer cannot place lands in `other`, which is a legitimate
+ * observation rather than an error.
+ */
+export type DecisionKind = 'action' | 'subAgent' | 'chat' | 'taskComplete' | 'payloadChange' | 'other'
+/** Output existed but could not be read as a decision — the malformed class. */
+ | 'unparseable'
+/** No usable output at all (empty content, no tool calls). */
+ | 'empty';
+/** How the decision arrived on the wire. Recorded so a comparison can attribute a delta to it. */
+export type DecisionEncoding = 'envelope' | 'native' | 'text' | 'none';
+/** What calling a control-flow tool decides. Sub-agent entries carry the agent's name. */
+export type ControlToolRole = {
+    kind: 'subAgent';
+    name: string;
+} | {
+    kind: 'payloadChange';
+} | {
+    kind: 'chat';
+};
+/** One action invocation, normalized across both encodings. */
+export interface ObservedAction {
+    name: string;
+    params: Record<string, unknown>;
+}
+/** One sub-agent dispatch. */
+export interface ObservedSubAgent {
+    name: string;
+    message?: string;
+}
+/** The normalized decision: everything a corpus case is allowed to assert against. */
+export interface ObservedDecision {
+    kind: DecisionKind;
+    encoding: DecisionEncoding;
+    actions: ObservedAction[];
+    subAgents: ObservedSubAgent[];
+    /** The user-facing message, when the agent produced one. */
+    message?: string;
+    /** True when the envelope declared the task finished. */
+    taskComplete: boolean;
+    /** Present when the turn carried a payload change request. */
+    payloadChange?: Record<string, unknown>;
+    /** Whether an envelope was expected and successfully parsed. `null` when none was expected. */
+    envelopeParsed: boolean | null;
+    /**
+     * The raw `nextStep.type` exactly as the model wrote it, before any normalization.
+     *
+     * Kept because the DECISION and the VALIDITY are different questions. `classifyEnvelope` is
+     * lenient by design — it infers a kind from the payload when the type is missing or odd, which
+     * is what makes the decision comparable across encodings. But the runtime is not lenient: it
+     * switches case-sensitively on this string and sends anything unrecognized to a forced Retry.
+     * A response can therefore be a perfectly good decision and still be malformed, and
+     * {@link evaluateWellFormed} needs the raw value to say so.
+     */
+    stepType?: string | null;
+    /** Why the decision is `unparseable` / `empty`, for the oracle's message. */
+    diagnostic?: string;
+    /**
+     * Native turns only: the model ALSO returned a parseable Loop envelope alongside its tool calls
+     * The tool call still wins — this records that something was discarded.
+     */
+    dualChannel?: boolean;
+    /** What that discarded envelope would have decided, when `dualChannel` is true. */
+    shadowEnvelopeKind?: DecisionKind;
+    /** Native turns only: prose accompanied the call(s) and was not an envelope (Haiku: 30% of call turns in run 7). */
+    narrationWithCalls?: boolean;
+    /** Native turns only: the calls carried empty or placeholder arguments alongside prose — the Sonnet 5 pattern of results §13.3. */
+    placeholderCall?: boolean;
+}
+/** What the harness observed on the wire, before normalization. */
+export interface RawTurn {
+    /** The model's text output, if any. */
+    text?: string | null;
+    /** Native tool calls on the assistant turn, if the driver requested tools. */
+    toolCalls?: Array<{
+        name: string;
+        arguments?: Record<string, unknown>;
+    }> | null;
+    /**
+     * Sanitized tool name → the Action name the framework dispatches for it.
+     *
+     * Provider tool names are constrained (`^[a-zA-Z0-9_-]+$` and friends), so MJ's Action names
+     * are sanitized on the way out: `Execute Code` is declared as `execute_code`. A corpus case
+     * names the ACTION, per §1.2 — comparing the raw wire name against it would score every
+     * correct native call as the wrong action, measuring the sanitizer rather than the model.
+     *
+     * The driver supplies the framework's own reverse map, the same one `LoopAgentType` dispatches
+     * from. An unmapped name passes through unchanged: a model that invented a tool should be
+     * reported under the name it invented, not silently dropped.
+     */
+    toolNameMap?: Readonly<Record<string, string>> | null;
+    /**
+     * Which protocol the turn was produced under. `'implicit'` reads plain text with no call as
+     * task completion; the other two keep the envelope rules. Absent = `'envelope'`.
+     */
+    protocol?: 'envelope' | 'hybrid' | 'implicit' | null;
+    /** Control-flow tool name → what calling it decides. Absent when no control tools were declared. */
+    controlToolMap?: Readonly<Record<string, ControlToolRole>> | null;
+}
+/**
+ * Strips a markdown fence so a fenced-but-otherwise-correct envelope still counts as parsed.
+ *
+ * Fencing is a formatting habit, not a decision error. Counting it as malformed would inflate the
+ * exact metric this harness exists to measure honestly — and MJ's own loop parser tolerates it, so
+ * scoring it as a failure would also diverge from what production actually does.
+ */
+export declare function stripJsonFence(text: string): string;
+/**
+ * Counts complete top-level JSON objects in a string that failed to parse as one — the
+ * "two envelopes back to back" shape. String-aware brace counting; returns 0 unless at least two
+ * objects were found, so a single malformed object is left to the ordinary diagnostic.
+ */
+export declare function countConcatenatedObjects(text: string): number;
+/**
+ * Spec §8.2: prose accompanied the call(s) and every call has empty arguments, or every string
+ * argument is a placeholder token. The Sonnet 5 pattern of results §13.3, made countable.
+ */
+export declare function isPlaceholderCall(calls: NonNullable<RawTurn['toolCalls']>, text: string | null | undefined): boolean;
+/**
+ * Normalizes one turn into an {@link ObservedDecision}.
+ *
+ * Native tool calls win when present: a model that emitted both has made its decision natively and
+ * the text is at most a narration of it. This is the rule the corpus specifies, and the provider matrix
+ * measured why it matters — Gemini 3.7 Flash under JSON mode answers a tool question natively while
+ * ignoring the requested envelope entirely, which an envelope-only reader scores as 100% malformed
+ * even though the decision was correct.
+ *
+ * Under the implicit protocol, plain text with no call is the terminal form: it reads as
+ * task completion unless it parses as an envelope, in which case the envelope is honoured (§2.1).
+ */
+export declare function normalizeDecision(turn: RawTurn): ObservedDecision;
+//# sourceMappingURL=decision.d.ts.map

package/dist/eval/decision.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision.d.ts","sourceRoot":"","sources":["../../src/eval/decision.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;;;GAOG;AACH,MAAM,MAAM,YAAY,GAClB,QAAQ,GACR,UAAU,GACV,MAAM,GACN,cAAc,GACd,eAAe,GACf,OAAO;AACT,gFAAgF;GAC9E,aAAa;AACf,8DAA8D;GAC5D,OAAO,CAAC;AAEd,kGAAkG;AAClG,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAEvE,0FAA0F;AAC1F,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,eAAe,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhH,+DAA+D;AAC/D,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,8BAA8B;AAC9B,MAAM,WAAW,gBAAgB;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,sFAAsF;AACtF,MAAM,WAAW,gBAAgB;IAC7B,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,OAAO,CAAC;IACtB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,+FAA+F;IAC/F,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,6EAA6E;IAC7E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mFAAmF;IACnF,kBAAkB,CAAC,EAAE,YAAY,CAAC;IAClC,oHAAoH;IACpH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,mIAAmI;IACnI,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,mEAAmE;AACnE,MAAM,WAAW,OAAO;IACpB,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8EAA8E;IAC9E,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,GAAG,IAAI,CAAC;IAChF;;;;;;;;;;;OAWG;IACH,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC;IACtD;;;OAGG;IACH,QAAQ,CAAC,EAAE,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,IAAI,CAAC;IACrD,oGAAoG;IACpG,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC;CACrE;AAMD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAMnD;AA4ED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAc7D;AAkDD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAQpH;AAsCD;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,OAAO,GAAG,gBAAgB,CAYjE"}