@memberjunction/testing-engine 6.1.0-edge.5 → 6.1.0-edge.7

package/dist/eval/decision.js

@@ -0,0 +1,257 @@
+/**
+ * decision.ts — normalizing what an agent DECIDED, independently of how it said so.
+ *
+ * This is the load-bearing idea of the whole eval harness (test plan §1). A corpus case states the
+ * decision it expects — "invoke `Run Ad-hoc Query` with these params" — and never how the wire
+ * encodes it. Two encodings exist today and a third is coming:
+ *
+ *   envelope  — a `LoopAgentResponse` JSON object with `nextStep.actions[]`
+ *   native    — provider tool calls on the assistant turn
+ *
+ * Both collapse here into one {@link ObservedDecision} before anything is scored. That is what
+ * makes a baseline run and a native run *directly comparable on identical cases*:
+ * the only thing that changed between them is the model's behavior, not the yardstick.
+ *
+ * FRAMEWORK-FREE (test plan §2.1). Nothing in `src/eval/` imports from the testing engine — not
+ * `IOracle`, not `TestEngine`, not the driver types. The oracles in `src/oracles/` are thin
+ * wrappers over these functions, and a bespoke runner could call them directly. That rule exists so
+ * the documented fallback (§2.1, option B) stays cheap; it also happens to make every line here
+ * unit-testable without a database.
+ */
+function asRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value) ? value : null;
+}
+/**
+ * Strips a markdown fence so a fenced-but-otherwise-correct envelope still counts as parsed.
+ *
+ * Fencing is a formatting habit, not a decision error. Counting it as malformed would inflate the
+ * exact metric this harness exists to measure honestly — and MJ's own loop parser tolerates it, so
+ * scoring it as a failure would also diverge from what production actually does.
+ */
+export function stripJsonFence(text) {
+    const trimmed = text.trim();
+    if (!trimmed.startsWith('```')) {
+        return trimmed;
+    }
+    return trimmed.replace(/^```[a-zA-Z]*\s*\n?/, '').replace(/\n?```\s*$/, '').trim();
+}
+function readActions(nextStep) {
+    const raw = nextStep?.actions;
+    if (!Array.isArray(raw)) {
+        return [];
+    }
+    const actions = [];
+    for (const entry of raw) {
+        const record = asRecord(entry);
+        const name = record?.name;
+        if (typeof name === 'string' && name.length > 0) {
+            actions.push({ name, params: asRecord(record?.params) ?? {} });
+        }
+    }
+    return actions;
+}
+function readSubAgents(nextStep) {
+    const found = [];
+    const single = asRecord(nextStep?.subAgent);
+    const many = Array.isArray(nextStep?.subAgents) ? nextStep?.subAgents : [];
+    for (const entry of [single, ...many.map(asRecord)]) {
+        const name = entry?.name;
+        if (typeof name === 'string' && name.length > 0) {
+            found.push({ name, message: typeof entry?.message === 'string' ? entry.message : undefined });
+        }
+    }
+    return found;
+}
+/**
+ * Picks the decision kind from an envelope's contents.
+ *
+ * Order matters and mirrors what `LoopAgentType.DetermineNextStep` actually does, not what the
+ * interface suggests: a Chat step is honored even when `taskComplete` is true, and an explicit
+ * `nextStep` outranks the completion flag. Scoring against a different precedence than production
+ * uses would measure the harness, not the agent.
+ */
+function classifyEnvelope(stepType, actions, subAgents, taskComplete, hasPayloadChange, hasResponseForm) {
+    const lowered = stepType?.toLowerCase().trim();
+    if (lowered === 'chat') {
+        return 'chat';
+    }
+    if (lowered === 'actions' || (!lowered && actions.length > 0)) {
+        return 'action';
+    }
+    if (lowered === 'sub-agent' || (!lowered && subAgents.length > 0)) {
+        return 'subAgent';
+    }
+    if (taskComplete) {
+        return 'taskComplete';
+    }
+    // `responseForm` is a first-class field on LoopAgentResponse — it is HOW an MJ agent asks the
+    // user a structured question. A turn carrying one is addressing the user, which is a chat
+    // decision by every measure the corpus cares about; only the encoding differs from a plain
+    // `nextStep: { type: 'Chat' }`. Ranked below an explicit nextStep so a turn that says both
+    // still dispatches the way production dispatches it.
+    if (hasResponseForm) {
+        return 'chat';
+    }
+    // A turn whose only content is a payload write is a payloadChange decision; the loop's own
+    // validator treats that case as an implicit completion rather than an error.
+    if (hasPayloadChange) {
+        return 'payloadChange';
+    }
+    return 'other';
+}
+/**
+ * Counts complete top-level JSON objects in a string that failed to parse as one — the
+ * "two envelopes back to back" shape. String-aware brace counting; returns 0 unless at least two
+ * objects were found, so a single malformed object is left to the ordinary diagnostic.
+ */
+export function countConcatenatedObjects(text) {
+    let depth = 0, inString = false, escaped = false, objects = 0;
+    for (const ch of text) {
+        if (inString) {
+            if (escaped) {
+                escaped = false;
+            }
+            else if (ch === '\\') {
+                escaped = true;
+            }
+            else if (ch === '"') {
+                inString = false;
+            }
+            continue;
+        }
+        if (ch === '"') {
+            inString = true;
+        }
+        else if (ch === '{') {
+            depth++;
+        }
+        else if (ch === '}') {
+            depth--;
+            if (depth === 0) {
+                objects++;
+            }
+        }
+    }
+    return objects >= 2 ? objects : 0;
+}
+/** Reads an envelope out of raw text. Never throws — unparseable output is an observation. */
+function normalizeEnvelope(text) {
+    const base = { encoding: 'envelope', actions: [], subAgents: [], taskComplete: false, envelopeParsed: false };
+    if (text.trim().length === 0) {
+        return { ...base, kind: 'empty', encoding: 'none', envelopeParsed: null, diagnostic: 'no output' };
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(stripJsonFence(text));
+    }
+    catch (error) {
+        const concatenated = countConcatenatedObjects(stripJsonFence(text));
+        return {
+            ...base,
+            kind: 'unparseable',
+            diagnostic: concatenated > 0
+                ? `envelope did not parse: ${concatenated} complete JSON objects were concatenated in one turn`
+                : `envelope did not parse: ${error.message}`
+        };
+    }
+    const record = asRecord(parsed);
+    if (!record) {
+        return { ...base, kind: 'unparseable', diagnostic: 'envelope parsed but was not a JSON object' };
+    }
+    const nextStep = asRecord(record.nextStep);
+    const actions = readActions(nextStep);
+    const subAgents = readSubAgents(nextStep);
+    const payloadChange = asRecord(record.payloadChangeRequest) ?? undefined;
+    const taskComplete = record.taskComplete === true;
+    const stepType = typeof nextStep?.type === 'string' ? nextStep.type : null;
+    return {
+        kind: classifyEnvelope(stepType, actions, subAgents, taskComplete, payloadChange !== undefined, asRecord(record.responseForm) !== null),
+        encoding: 'envelope',
+        stepType,
+        actions,
+        subAgents,
+        message: typeof record.message === 'string' ? record.message : undefined,
+        taskComplete,
+        payloadChange,
+        envelopeParsed: true
+    };
+}
+const PLACEHOLDER_VALUES = /^(placeholder|none|n\/a|null|nothing|-)$/i;
+const PLACEHOLDER_PHRASES = /none needed|just checking/i;
+/**
+ * Spec §8.2: prose accompanied the call(s) and every call has empty arguments, or every string
+ * argument is a placeholder token. The Sonnet 5 pattern of results §13.3, made countable.
+ */
+export function isPlaceholderCall(calls, text) {
+    if (!text?.trim() || calls.length === 0)
+        return false;
+    return calls.every((c) => {
+        const values = Object.values(c.arguments ?? {});
+        if (values.length === 0)
+            return true;
+        const strings = values.filter((v) => typeof v === 'string');
+        return strings.length === values.length && strings.every((v) => PLACEHOLDER_VALUES.test(v.trim()) || PLACEHOLDER_PHRASES.test(v));
+    });
+}
+/** Native-turn classification: control tools first (spec §8.1), then Actions, then payload-only. */
+function normalizeNativeTurn(turn, calls, text) {
+    const toActionName = (toolName) => turn.toolNameMap?.[toolName] ?? toolName;
+    const role = (toolName) => turn.controlToolMap?.[toolName];
+    // The call wins, but if the text is ALSO a valid envelope the model gave
+    // two answers. Record it; the scorecard counts it; production does the same on NativeDualChannel.
+    const shadow = text ? normalizeEnvelope(text) : null;
+    const dualChannel = shadow?.envelopeParsed === true;
+    const base = {
+        encoding: 'native',
+        envelopeParsed: null,
+        message: text,
+        taskComplete: false,
+        dualChannel,
+        ...(dualChannel ? { shadowEnvelopeKind: shadow.kind } : {}),
+        narrationWithCalls: !!text && !dualChannel,
+        placeholderCall: isPlaceholderCall(calls, text)
+    };
+    const ask = calls.find((c) => role(c.name)?.kind === 'chat');
+    if (ask) {
+        const message = typeof ask.arguments?.message === 'string' ? ask.arguments.message : undefined;
+        return { ...base, kind: 'chat', actions: [], subAgents: [], message: message ?? text };
+    }
+    const payloadCall = calls.find((c) => role(c.name)?.kind === 'payloadChange');
+    const payloadChange = payloadCall ? (payloadCall.arguments ?? {}) : undefined;
+    const subAgents = calls.flatMap((c) => { const r = role(c.name); return r?.kind === 'subAgent' ? [{ name: r.name }] : []; });
+    const actions = calls.filter((c) => !role(c.name)).map((c) => ({ name: toActionName(c.name), params: c.arguments ?? {} }));
+    if (subAgents.length > 0) {
+        return { ...base, kind: 'subAgent', actions, subAgents, payloadChange };
+    }
+    if (actions.length > 0) {
+        return { ...base, kind: 'action', actions, subAgents: [], payloadChange };
+    }
+    return { ...base, kind: 'payloadChange', actions: [], subAgents: [], payloadChange };
+}
+/**
+ * Normalizes one turn into an {@link ObservedDecision}.
+ *
+ * Native tool calls win when present: a model that emitted both has made its decision natively and
+ * the text is at most a narration of it. This is the rule the corpus specifies, and the provider matrix
+ * measured why it matters — Gemini 3.7 Flash under JSON mode answers a tool question natively while
+ * ignoring the requested envelope entirely, which an envelope-only reader scores as 100% malformed
+ * even though the decision was correct.
+ *
+ * Under the implicit protocol, plain text with no call is the terminal form: it reads as
+ * task completion unless it parses as an envelope, in which case the envelope is honoured (§2.1).
+ */
+export function normalizeDecision(turn) {
+    const calls = turn.toolCalls ?? [];
+    const text = turn.text?.trim() ? turn.text : undefined;
+    if (calls.length > 0) {
+        return normalizeNativeTurn(turn, calls, text);
+    }
+    if (turn.protocol === 'implicit' && text) {
+        const asEnvelope = normalizeEnvelope(text);
+        if (asEnvelope.envelopeParsed)
+            return asEnvelope;
+        return { kind: 'taskComplete', encoding: 'text', actions: [], subAgents: [], message: text.trim(), taskComplete: true, envelopeParsed: null };
+    }
+    return normalizeEnvelope(turn.text ?? '');
+}
+//# sourceMappingURL=decision.js.map

package/dist/eval/decision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision.js","sourceRoot":"","sources":["../../src/eval/decision.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AA4GH,SAAS,QAAQ,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5H,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACvF,CAAC;AAED,SAAS,WAAW,CAAC,QAAwC;IACzD,MAAM,GAAG,GAAG,QAAQ,EAAE,OAAO,CAAC;IAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC;QAC1B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,QAAwC;IAC3D,MAAM,KAAK,GAAuB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3E,KAAK,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC;QACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QAClG,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CACrB,QAAuB,EACvB,OAAyB,EACzB,SAA6B,EAC7B,YAAqB,EACrB,gBAAyB,EACzB,eAAwB;IAExB,MAAM,OAAO,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAC/C,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACrB,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QAC5D,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,KAAK,WAAW,IAAI,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QAChE,OAAO,UAAU,CAAC;IACtB,CAAC;IACD,IAAI,YAAY,EAAE,CAAC;QACf,OAAO,cAAc,CAAC;IAC1B,CAAC;IACD,8FAA8F;IAC9F,0FAA0F;IAC1F,2FAA2F;IAC3F,2FAA2F;IAC3F,qDAAqD;IACrD,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,2FAA2F;IAC3F,6EAA6E;IAC7E,IAAI,gBAAgB,EAAE,CAAC;QACnB,OAAO,eAAe,CAAC;IAC3B,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAY;IACjD,IAAI,KAAK,GAAG,CAAC,EAAE,QAAQ,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,OAAO,GAAG,CAAC,CAAC;IAC9D,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACpB,IAAI,QAAQ,EAAE,CAAC;YACX,IAAI,OAAO,EAAE,CAAC;gBAAC,OAAO,GAAG,KAAK,CAAC;YAAC,CAAC;iBAC5B,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBAAC,OAAO,GAAG,IAAI,CAAC;YAAC,CAAC;iBACpC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAAC,QAAQ,GAAG,KAAK,CAAC;YAAC,CAAC;YAC1C,SAAS;QACb,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAAC,QAAQ,GAAG,IAAI,CAAC;QAAC,CAAC;aAC/B,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAAC,KAAK,EAAE,CAAC;QAAC,CAAC;aAC5B,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAAC,KAAK,EAAE,CAAC;YAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAAC,OAAO,EAAE,CAAC;YAAC,CAAC;QAAC,CAAC;IACrE,CAAC;IACD,OAAO,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,8FAA8F;AAC9F,SAAS,iBAAiB,CAAC,IAAY;IACnC,MAAM,IAAI,GAAG,EAAE,QAAQ,EAAE,UAAmB,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;IACvH,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;IACvG,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,YAAY,GAAG,wBAAwB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACpE,OAAO;YACH,GAAG,IAAI;YACP,IAAI,EAAE,aAAa;YACnB,UAAU,EAAE,YAAY,GAAG,CAAC;gBACxB,CAAC,CAAC,2BAA2B,YAAY,sDAAsD;gBAC/F,CAAC,CAAC,2BAA4B,KAAe,CAAC,OAAO,EAAE;SAC9D,CAAC;IACN,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,2CAA2C,EAAE,CAAC;IACrG,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,SAAS,CAAC;IACzE,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,KAAK,IAAI,CAAC;IAClD,MAAM,QAAQ,GAAG,OAAO,QAAQ,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAE3E,OAAO;QACH,IAAI,EAAE,gBAAgB,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,KAAK,SAAS,EAC1F,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,IAAI,CAAC;QAC3C,QAAQ,EAAE,UAAU;QACpB,QAAQ;QACR,OAAO;QACP,SAAS;QACT,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACxE,YAAY;QACZ,aAAa;QACb,cAAc,EAAE,IAAI;KACvB,CAAC;AACN,CAAC;AAED,MAAM,kBAAkB,GAAG,2CAA2C,CAAC;AACvE,MAAM,mBAAmB,GAAG,4BAA4B,CAAC;AAEzD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAwC,EAAE,IAA+B;IACvG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QACzE,OAAO,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACtI,CAAC,CAAC,CAAC;AACP,CAAC;AAED,oGAAoG;AACpG,SAAS,mBAAmB,CAAC,IAAa,EAAE,KAAwC,EAAE,IAAwB;IAC1G,MAAM,YAAY,GAAG,CAAC,QAAgB,EAAU,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC;IAC5F,MAAM,IAAI,GAAG,CAAC,QAAgB,EAA+B,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC;IAChG,yEAAyE;IACzE,kGAAkG;IAClG,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrD,MAAM,WAAW,GAAG,MAAM,EAAE,cAAc,KAAK,IAAI,CAAC;IACpD,MAAM,IAAI,GAAG;QACT,QAAQ,EAAE,QAAiB;QAC3B,cAAc,EAAE,IAAI;QACpB,OAAO,EAAE,IAAI;QACb,YAAY,EAAE,KAAK;QACnB,WAAW;QACX,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,kBAAkB,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW;QAC1C,eAAe,EAAE,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC;KAClD,CAAC;IACF,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7D,IAAI,GAAG,EAAE,CAAC;QACN,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,SAAS,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/F,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,IAAI,IAAI,EAAE,CAAC;IAC3F,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,KAAK,eAAe,CAAC,CAAC;IAC9E,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7H,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3H,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC;IAC5E,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,CAAC;IAC9E,CAAC;IACD,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,CAAC;AACzF,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACvD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,KAAK,UAAU,IAAI,IAAI,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,UAAU,CAAC,cAAc;YAAE,OAAO,UAAU,CAAC;QACjD,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;IAClJ,CAAC;IACD,OAAO,iBAAiB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;AAC9C,CAAC"}

package/dist/eval/expectation.d.ts

@@ -0,0 +1,76 @@
+/**
+ * expectation.ts — the corpus's expectation DSL, and scoring an observation against it.
+ *
+ * A case states one {@link DecisionExpectation}; {@link evaluateDecision} scores an
+ * {@link ObservedDecision} against it and returns the components **separately** rather than only a
+ * number. That decomposition is required by test plan §3.2 so the §6.1 metrics — decision accuracy,
+ * action accuracy, param fidelity — aggregate straight out of persisted `ResultDetails` without
+ * re-running a single model call. A single collapsed score would make every cross-run comparison a
+ * re-run.
+ *
+ * FRAMEWORK-FREE (test plan §2.1) — imports nothing from the testing engine.
+ */
+import type { DecisionKind, ObservedDecision } from './decision.js';
+import { type ParamExpectation, type ParamMatchResult } from './matchers.js';
+/** One action a correct answer invokes, with the checks that make the invocation usable. */
+export interface ExpectedAction {
+    name: string;
+    params?: ParamExpectation[];
+}
+/** What a correct answer looks like — stated as a decision, never as an encoding. */
+export interface DecisionExpectation {
+    kind: DecisionKind;
+    /** Required when `kind` is `action`. Compared as a set unless `ordered`. */
+    actions?: ExpectedAction[];
+    /** Sub-agent names, when `kind` is `subAgent`. */
+    subAgents?: string[];
+    /** Optional check on the user-facing message — e.g. a `chat` case that must mention a topic. */
+    message?: ParamExpectation['matcher'];
+    /**
+     * Names that must NOT appear, whatever else happens.
+     *
+     * This is the coherence probe's instrument: with tools declared, a model
+     * must *not* reach for one when chat or completion is the right answer. Expressed as a
+     * prohibition rather than a kind mismatch so a case can forbid a specific tempting action
+     * while still allowing others.
+     */
+    forbiddenActions?: string[];
+    /** Whether action order is asserted. Defaults to false — parallel calls have no inherent order. */
+    ordered?: boolean;
+    /**
+     * Whether actions BEYOND the expected set are tolerated. Defaults to false.
+     *
+     * False is the right default for a decision corpus: an agent that calls the right action plus
+     * two speculative extras has not made the right decision, it has made three. Cases that
+     * genuinely do not care set this true.
+     */
+    allowAdditionalActions?: boolean;
+    /**
+     * Matchers over dotted paths into the observed payload change request, e.g.
+     * `newElements.findings`. Scored into `paramFidelity` alongside action parameters.
+     */
+    payload?: ParamExpectation[];
+    /** Dotted paths into the payload change that must NOT be present (payload scoping). */
+    forbiddenPayloadPaths?: string[];
+}
+/** Per-component outcome, so §6.1's metrics fall out of the stored details. */
+export interface DecisionEvaluation {
+    /** Metric 3 — did the agent decide the right *kind* of thing? */
+    decisionKindMatch: boolean;
+    /** Metric 4 — right action name(s). `null` when the case expects no action. */
+    actionAccuracy: number | null;
+    /** Metric 5 — share of parameter matchers that passed. `null` when the case declares none. */
+    paramFidelity: number | null;
+    /** Individual matcher outcomes, for reporting which parameter failed and why. */
+    paramResults: ParamMatchResult[];
+    /** Forbidden names the agent invoked anyway. Non-empty is always a failure. */
+    forbiddenViolations: string[];
+    /** Overall pass — every applicable component clean. */
+    passed: boolean;
+    /** Mean of the APPLICABLE components. See the note in {@link evaluateDecision}. */
+    score: number;
+    /** Human-readable reasons, in the order they were detected. */
+    messages: string[];
+}
+export declare function evaluateDecision(expectation: DecisionExpectation, observed: ObservedDecision): DecisionEvaluation;
+//# sourceMappingURL=expectation.d.ts.map

package/dist/eval/expectation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"expectation.d.ts","sourceRoot":"","sources":["../../src/eval/expectation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAe,KAAK,gBAAgB,EAAE,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEvF,4FAA4F;AAC5F,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,qFAAqF;AACrF,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,YAAY,CAAC;IACnB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gGAAgG;IAChG,OAAO,CAAC,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACtC;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,mGAAmG;IACnG,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;OAMG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC;;;OAGG;IACH,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,uFAAuF;IACvF,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;CACpC;AAED,+EAA+E;AAC/E,MAAM,WAAW,kBAAkB;IAC/B,iEAAiE;IACjE,iBAAiB,EAAE,OAAO,CAAC;IAC3B,+EAA+E;IAC/E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,8FAA8F;IAC9F,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iFAAiF;IACjF,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,+EAA+E;IAC/E,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,uDAAuD;IACvD,MAAM,EAAE,OAAO,CAAC;IAChB,mFAAmF;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAiGD,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,gBAAgB,GAAG,kBAAkB,CAgFjH"}

package/dist/eval/expectation.js

@@ -0,0 +1,160 @@
+import { matchParams } from './matchers.js';
+function sameNameSet(expected, observed, ordered, allowAdditional = false) {
+    if (allowAdditional) {
+        // Containment, not equality: every expected name appeared somewhere.
+        const seen = [...observed];
+        return expected.every((name) => {
+            const at = seen.indexOf(name);
+            if (at < 0) {
+                return false;
+            }
+            seen.splice(at, 1); // consume, so a duplicate expectation needs a duplicate call
+            return true;
+        });
+    }
+    if (expected.length !== observed.length) {
+        return false;
+    }
+    if (ordered) {
+        return expected.every((name, i) => name === observed[i]);
+    }
+    const a = [...expected].sort();
+    const b = [...observed].sort();
+    return a.every((name, i) => name === b[i]);
+}
+function scoreActions(expectation, observed) {
+    const expectedActions = expectation.actions ?? [];
+    if (expectedActions.length === 0) {
+        return { accuracy: null, paramResults: [], messages: [] };
+    }
+    const messages = [];
+    const observedNames = observed.actions.map((a) => a.name);
+    const namesMatch = sameNameSet(expectedActions.map((a) => a.name), observedNames, expectation.ordered === true, expectation.allowAdditionalActions === true);
+    if (!namesMatch) {
+        messages.push(`expected action(s) [${expectedActions.map((a) => a.name).join(', ')}], got [${observedNames.join(', ')}]`);
+    }
+    const paramResults = [];
+    for (const expected of expectedActions) {
+        if (!expected.params || expected.params.length === 0) {
+            continue;
+        }
+        // Score against the first call of that name. A duplicate call is already a name-level
+        // failure; charging it again here would double-penalize one mistake across two metrics.
+        const match = observed.actions.find((a) => a.name === expected.name);
+        if (!match) {
+            paramResults.push(...expected.params.map((p) => ({
+                param: `${expected.name}.${p.param}`,
+                matcherKind: p.matcher.kind,
+                passed: false,
+                detail: `action '${expected.name}' was not invoked`
+            })));
+            continue;
+        }
+        paramResults.push(...matchParams(expected.params, match.params).map((r) => ({ ...r, param: `${expected.name}.${r.param}` })));
+    }
+    for (const failure of paramResults.filter((r) => !r.passed)) {
+        messages.push(`param ${failure.param}: ${failure.detail}`);
+    }
+    return { accuracy: namesMatch ? 1 : 0, paramResults, messages };
+}
+/**
+ * Scores one observation against one expectation.
+ *
+ * **On `score`:** it is the mean of the components that *apply* to this case, not a fixed weighting.
+ * A case expecting `chat` has no action or param component, and counting those absent components as
+ * either 1 (inflating) or 0 (deflating) would make chat cases and action cases incomparable — which
+ * is precisely what a corpus-wide average needs them to be. Weighting across metrics is the
+ * scorecard's job at aggregation time, where it can be stated explicitly; §6.1 treats them as
+ * separate metrics for the same reason.
+ */
+function getPath(obj, path) {
+    return path.split('.').reduce((acc, key) => (acc !== null && typeof acc === 'object' ? acc[key] : undefined), obj);
+}
+/** Payload matchers and forbidden paths, over the observed `payloadChange`. */
+function scorePayload(expectation, observed) {
+    const matchers = expectation.payload ?? [];
+    const messages = [];
+    const flat = Object.fromEntries(matchers.map((m) => [m.param, getPath(observed.payloadChange, m.param)]));
+    const paramResults = matchers.length === 0 ? [] : matchParams(matchers, flat).map((r) => ({ ...r, param: `payload ${r.param}` }));
+    for (const failure of paramResults.filter((r) => !r.passed)) {
+        messages.push(`${failure.param}: ${failure.detail}`);
+    }
+    const forbidden = (expectation.forbiddenPayloadPaths ?? []).filter((path) => getPath(observed.payloadChange, path) !== undefined);
+    if (forbidden.length > 0) {
+        messages.push(`wrote forbidden payload path(s): [${forbidden.join(', ')}]`);
+    }
+    return { paramResults, messages, forbidden };
+}
+export function evaluateDecision(expectation, observed) {
+    const messages = [];
+    // A payloadChange expectation is met by a mixed turn too (action + payload write, spec §2.1).
+    const decisionKindMatch = observed.kind === expectation.kind
+        || (expectation.kind === 'payloadChange' && observed.payloadChange !== undefined);
+    if (!decisionKindMatch) {
+        const why = observed.diagnostic ? ` (${observed.diagnostic})` : '';
+        messages.push(`expected decision kind '${expectation.kind}', got '${observed.kind}'${why}`);
+    }
+    const actionScore = scoreActions(expectation, observed);
+    const payloadScore = scorePayload(expectation, observed);
+    const accuracy = actionScore.accuracy;
+    const paramResults = [...actionScore.paramResults, ...payloadScore.paramResults];
+    messages.push(...actionScore.messages, ...payloadScore.messages);
+    const expectedSubAgents = expectation.subAgents ?? [];
+    let subAgentsMatch = true;
+    if (expectedSubAgents.length > 0) {
+        subAgentsMatch = sameNameSet(expectedSubAgents, observed.subAgents.map((s) => s.name), expectation.ordered === true);
+        if (!subAgentsMatch) {
+            messages.push(`expected sub-agent(s) [${expectedSubAgents.join(', ')}], got [${observed.subAgents.map((s) => s.name).join(', ')}]`);
+        }
+    }
+    // Forbidden names are checked against BOTH channels: the point is that the agent did not reach
+    // for the capability, and it is equally wrong whether it did so natively or through the envelope.
+    const invoked = new Set([...observed.actions.map((a) => a.name), ...observed.subAgents.map((s) => s.name)]);
+    const forbiddenViolations = [
+        ...(expectation.forbiddenActions ?? []).filter((name) => invoked.has(name)),
+        ...payloadScore.forbidden.map((path) => `payload:${path}`)
+    ];
+    if (forbiddenViolations.length > 0) {
+        messages.push(`invoked forbidden: [${forbiddenViolations.join(', ')}]`);
+    }
+    let messageMatch = true;
+    if (expectation.message) {
+        const result = matchParams([{ param: 'message', matcher: expectation.message }], { message: observed.message });
+        messageMatch = result[0].passed;
+        if (!messageMatch) {
+            messages.push(`message: ${result[0].detail}`);
+        }
+    }
+    const paramFidelity = paramResults.length === 0 ? null : paramResults.filter((r) => r.passed).length / paramResults.length;
+    const applicable = [decisionKindMatch ? 1 : 0];
+    if (accuracy !== null) {
+        applicable.push(accuracy);
+    }
+    if (paramFidelity !== null) {
+        applicable.push(paramFidelity);
+    }
+    if (expectedSubAgents.length > 0) {
+        applicable.push(subAgentsMatch ? 1 : 0);
+    }
+    if (expectation.message) {
+        applicable.push(messageMatch ? 1 : 0);
+    }
+    applicable.push(forbiddenViolations.length === 0 ? 1 : 0);
+    const passed = decisionKindMatch
+        && (accuracy === null || accuracy === 1)
+        && (paramFidelity === null || paramFidelity === 1)
+        && subAgentsMatch
+        && messageMatch
+        && forbiddenViolations.length === 0;
+    return {
+        decisionKindMatch,
+        actionAccuracy: accuracy,
+        paramFidelity,
+        paramResults,
+        forbiddenViolations,
+        passed,
+        score: applicable.reduce((a, b) => a + b, 0) / applicable.length,
+        messages
+    };
+}
+//# sourceMappingURL=expectation.js.map

package/dist/eval/expectation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"expectation.js","sourceRoot":"","sources":["../../src/eval/expectation.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,WAAW,EAAgD,MAAM,YAAY,CAAC;AAiEvF,SAAS,WAAW,CAAC,QAAkB,EAAE,QAAkB,EAAE,OAAgB,EAAE,eAAe,GAAG,KAAK;IAClG,IAAI,eAAe,EAAE,CAAC;QAClB,qEAAqE;QACrE,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC3B,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACT,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAG,6DAA6D;YACnF,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,YAAY,CAAC,WAAgC,EAAE,QAA0B;IAK9E,MAAM,eAAe,GAAG,WAAW,CAAC,OAAO,IAAI,EAAE,CAAC;IAClD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC9D,CAAC;IACD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,WAAW,CAAC,OAAO,KAAK,IAAI,EAAE,WAAW,CAAC,sBAAsB,KAAK,IAAI,CAAC,CAAC;IAC7J,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CAAC,uBAAuB,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9H,CAAC;IAED,MAAM,YAAY,GAAuB,EAAE,CAAC;IAC5C,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,SAAS;QACb,CAAC;QACD,sFAAsF;QACtF,wFAAwF;QACxF,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrE,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,YAAY,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7C,KAAK,EAAE,GAAG,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE;gBACpC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI;gBAC3B,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,WAAW,QAAQ,CAAC,IAAI,mBAAmB;aACtD,CAAC,CAAC,CAAC,CAAC;YACL,SAAS;QACb,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAClI,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,QAAQ,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;AACpE,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,OAAO,CAAC,GAAY,EAAE,IAAY;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAU,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,GAA+B,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,CAAC;AAC7J,CAAC;AAED,+EAA+E;AAC/E,SAAS,YAAY,CAAC,WAAgC,EAAE,QAA0B;IAC9E,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,IAAI,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAClI,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,WAAW,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC;IAClI,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,qCAAqC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,WAAgC,EAAE,QAA0B;IACzF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,8FAA8F;IAC9F,MAAM,iBAAiB,GAAG,QAAQ,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI;WACrD,CAAC,WAAW,CAAC,IAAI,KAAK,eAAe,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC;IACtF,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,QAAQ,CAAC,IAAI,CAAC,2BAA2B,WAAW,CAAC,IAAI,WAAW,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;IACtC,MAAM,YAAY,GAAG,CAAC,GAAG,WAAW,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;IACjF,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEjE,MAAM,iBAAiB,GAAG,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC;IACtD,IAAI,cAAc,GAAG,IAAI,CAAC;IAC1B,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,cAAc,GAAG,WAAW,CAAC,iBAAiB,EAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;QACrH,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC,0BAA0B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxI,CAAC;IACL,CAAC;IAED,+FAA+F;IAC/F,kGAAkG;IAClG,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5G,MAAM,mBAAmB,GAAG;QACxB,GAAG,CAAC,WAAW,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3E,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;KAC7D,CAAC;IACF,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,CAAC;IACxB,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QAChH,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,QAAQ,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAClD,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IAE3H,MAAM,UAAU,GAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QACzB,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACtB,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,MAAM,MAAM,GAAG,iBAAiB;WACzB,CAAC,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,CAAC,CAAC;WACrC,CAAC,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,CAAC,CAAC;WAC/C,cAAc;WACd,YAAY;WACZ,mBAAmB,CAAC,MAAM,KAAK,CAAC,CAAC;IAExC,OAAO;QACH,iBAAiB;QACjB,cAAc,EAAE,QAAQ;QACxB,aAAa;QACb,YAAY;QACZ,mBAAmB;QACnB,MAAM;QACN,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;QAChE,QAAQ;KACX,CAAC;AACN,CAAC"}

package/dist/eval/history.d.ts

@@ -0,0 +1,17 @@
+/**
+ * history.ts — one multi-turn case, two history encodings.
+ *
+ * A multi-turn case is written ONCE, in the neutral tool form (an assistant turn carrying
+ * `toolCalls`, then a `tool` turn of `tool_result` blocks). An arm that returns results natively
+ * sends it as is. Every other arm sees what the loop shows those models today — the assistant's
+ * call turn dropped (the loop never appends the model's own turn on the envelope path) and each
+ * result as the corpus's `[Action Result] <tool> succeeded|failed. <content>` user message.
+ */
+import { type ChatMessage } from '@memberjunction/ai';
+/**
+ * The text rendering is Core's `EncodeToolTurnsAsText` — the same encoding the prompt runner's
+ * native→envelope fallback applies, so the harness's envelope arms and the runner's fallback can
+ * never drift apart.
+ */
+export declare function encodeHistoryForArm(messages: ChatMessage[], nativeResults: boolean): ChatMessage[];
+//# sourceMappingURL=history.d.ts.map

package/dist/eval/history.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../src/eval/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAyB,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE7E;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,aAAa,EAAE,OAAO,GAAG,WAAW,EAAE,CAElG"}

package/dist/eval/history.js

@@ -0,0 +1,19 @@
+/**
+ * history.ts — one multi-turn case, two history encodings.
+ *
+ * A multi-turn case is written ONCE, in the neutral tool form (an assistant turn carrying
+ * `toolCalls`, then a `tool` turn of `tool_result` blocks). An arm that returns results natively
+ * sends it as is. Every other arm sees what the loop shows those models today — the assistant's
+ * call turn dropped (the loop never appends the model's own turn on the envelope path) and each
+ * result as the corpus's `[Action Result] <tool> succeeded|failed. <content>` user message.
+ */
+import { EncodeToolTurnsAsText } from '@memberjunction/ai';
+/**
+ * The text rendering is Core's `EncodeToolTurnsAsText` — the same encoding the prompt runner's
+ * native→envelope fallback applies, so the harness's envelope arms and the runner's fallback can
+ * never drift apart.
+ */
+export function encodeHistoryForArm(messages, nativeResults) {
+    return nativeResults ? messages : EncodeToolTurnsAsText(messages);
+}
+//# sourceMappingURL=history.js.map

package/dist/eval/history.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/eval/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,qBAAqB,EAAoB,MAAM,oBAAoB,CAAC;AAE7E;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAuB,EAAE,aAAsB;IAC/E,OAAO,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;AACtE,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * The framework-free evaluation core (test plan §2.1).
+ *
+ * Everything exported here is a pure function over plain data. Nothing imports the testing engine,
+ * so the same logic runs behind `IOracle` wrappers today and would run unchanged behind a bespoke
+ * runner if the framework turns out to be a poor fit — which is the documented fallback the rule
+ * exists to keep cheap.
+ */
+export * from './decision.js';
+export * from './matchers.js';
+export * from './expectation.js';
+export * from './wellFormed.js';
+export * from './corpus.js';
+export * from './history.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC"}

package/dist/eval/index.js

@@ -0,0 +1,15 @@
+/**
+ * The framework-free evaluation core (test plan §2.1).
+ *
+ * Everything exported here is a pure function over plain data. Nothing imports the testing engine,
+ * so the same logic runs behind `IOracle` wrappers today and would run unchanged behind a bespoke
+ * runner if the framework turns out to be a poor fit — which is the documented fallback the rule
+ * exists to keep cheap.
+ */
+export * from './decision.js';
+export * from './matchers.js';
+export * from './expectation.js';
+export * from './wellFormed.js';
+export * from './corpus.js';
+export * from './history.js';
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC"}