@memberjunction/testing-engine 6.1.0-edge.6 → 6.1.0-edge.7

package/dist/eval/matchers.d.ts

@@ -0,0 +1,88 @@
+/**
+ * matchers.ts — per-parameter matchers for LLM-authored values.
+ *
+ * Exact string equality is the wrong test for most action parameters, and using it would make the
+ * harness measure phrasing instead of correctness. A SQL string, a search query, a summary
+ * instruction — all are correct in many forms. But enums, ids and booleans genuinely are exact.
+ * So matching is declarative and chosen per parameter by whoever writes the case (§3.2).
+ *
+ * Every variant is JSON-serializable, because a matcher arrives inside a `MJ: Tests`
+ * `Configuration` blob and has to survive that round trip. That constraint is why there is no
+ * `predicate` variant, however tempting: a function cannot be stored as metadata, and a corpus that
+ * needs code to express an expectation is a corpus that cannot be generated or diffed.
+ *
+ * FRAMEWORK-FREE (test plan §2.1) — imports nothing from the testing engine.
+ */
+/** One check against one parameter value. */
+export type ParamMatcher = 
+/** Deep equality against a literal. For enums, ids, booleans, numbers. */
+{
+    kind: 'exact';
+    value: unknown;
+}
+/** The value is a string with non-whitespace content. The weakest useful check. */
+ | {
+    kind: 'nonEmptyString';
+} | {
+    kind: 'nonEmpty';
+}
+/** Case-insensitive substring. For free text where a keyword must appear. */
+ | {
+    kind: 'containsIgnoreCase';
+    value: string;
+}
+/** Membership in a fixed set. For value-list parameters. */
+ | {
+    kind: 'oneOf';
+    values: unknown[];
+}
+/** A regular expression, supplied as a string with optional flags. */
+ | {
+    kind: 'regex';
+    pattern: string;
+    flags?: string;
+}
+/** Numeric bound, either end optional. */
+ | {
+    kind: 'numberRange';
+    min?: number;
+    max?: number;
+}
+/** The value's JSON type. For asserting shape without asserting content. */
+ | {
+    kind: 'typeOf';
+    type: 'string' | 'number' | 'boolean' | 'object' | 'array';
+}
+/** The parameter must be absent or null — for asserting a default was left alone. */
+ | {
+    kind: 'absent';
+};
+/** One matcher bound to the parameter it checks. */
+export interface ParamExpectation {
+    param: string;
+    matcher: ParamMatcher;
+    /**
+     * When true, an ABSENT parameter passes; a present one is still checked.
+     *
+     * This is how a case asserts "if the model sets `Format`, it must be `json`" without
+     * requiring it to set `Format` at all — the common shape for optional action params that
+     * carry a server-side default.
+     */
+    optional?: boolean;
+}
+/** The outcome of one matcher, kept per-parameter so param fidelity aggregates without a re-run. */
+export interface ParamMatchResult {
+    param: string;
+    matcherKind: ParamMatcher['kind'];
+    passed: boolean;
+    /** Why it failed, phrased for a human reading a test report. */
+    detail?: string;
+}
+/** Applies one matcher to one value, returning why it failed rather than just that it did. */
+export declare function applyMatcher(matcher: ParamMatcher, value: unknown): {
+    passed: boolean;
+    detail?: string;
+};
+/** Runs every expectation against one params object. */
+export declare function matchParams(expectations: ParamExpectation[], params: Record<string, unknown>): ParamMatchResult[];
+//# sourceMappingURL=matchers.d.ts.map

package/dist/eval/matchers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/eval/matchers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,6CAA6C;AAC7C,MAAM,MAAM,YAAY;AACpB,0EAA0E;AACxE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE;AACnC,mFAAmF;GACjF;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,UAAU,CAAA;CAAE;AACtB,6EAA6E;GAC3E;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE;AAC/C,4DAA4D;GAC1D;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAA;CAAE;AACtC,sEAAsE;GACpE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE;AACpD,0CAA0C;GACxC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE;AACrD,4EAA4E;GAC1E;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAA;CAAE;AAChF,qFAAqF;GACnF;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAC;AAEzB,oDAAoD;AACpD,MAAM,WAAW,gBAAgB;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,YAAY,CAAC;IACtB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,oGAAoG;AACpG,MAAM,WAAW,gBAAgB;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AA8BD,8FAA8F;AAC9F,wBAAgB,YAAY,CAAC,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,GAAG;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CA+DxG;AAED,wDAAwD;AACxD,wBAAgB,WAAW,CAAC,YAAY,EAAE,gBAAgB,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,gBAAgB,EAAE,CASjH"}

package/dist/eval/matchers.js

@@ -0,0 +1,119 @@
+/**
+ * matchers.ts — per-parameter matchers for LLM-authored values.
+ *
+ * Exact string equality is the wrong test for most action parameters, and using it would make the
+ * harness measure phrasing instead of correctness. A SQL string, a search query, a summary
+ * instruction — all are correct in many forms. But enums, ids and booleans genuinely are exact.
+ * So matching is declarative and chosen per parameter by whoever writes the case (§3.2).
+ *
+ * Every variant is JSON-serializable, because a matcher arrives inside a `MJ: Tests`
+ * `Configuration` blob and has to survive that round trip. That constraint is why there is no
+ * `predicate` variant, however tempting: a function cannot be stored as metadata, and a corpus that
+ * needs code to express an expectation is a corpus that cannot be generated or diffed.
+ *
+ * FRAMEWORK-FREE (test plan §2.1) — imports nothing from the testing engine.
+ */
+function describe(value) {
+    if (value === undefined) {
+        return 'absent';
+    }
+    const rendered = typeof value === 'string' ? value : JSON.stringify(value);
+    return rendered !== undefined && rendered.length > 80 ? `${rendered.slice(0, 80)}…` : String(rendered);
+}
+function deepEqual(a, b) {
+    // JSON round-trip equality is the right notion here: both sides originate as JSON (one from
+    // the model, one from a test record), so key order is the only difference it could miss.
+    if (a === b) {
+        return true;
+    }
+    if (typeof a !== typeof b || a === null || b === null) {
+        return false;
+    }
+    if (typeof a !== 'object') {
+        return false;
+    }
+    const aKeys = Object.keys(a).sort();
+    const bKeys = Object.keys(b).sort();
+    if (Array.isArray(a) !== Array.isArray(b) || aKeys.length !== bKeys.length) {
+        return false;
+    }
+    return aKeys.every((k, i) => k === bKeys[i] && deepEqual(a[k], b[k]));
+}
+/** Applies one matcher to one value, returning why it failed rather than just that it did. */
+export function applyMatcher(matcher, value) {
+    switch (matcher.kind) {
+        case 'absent':
+            return value === undefined || value === null
+                ? { passed: true }
+                : { passed: false, detail: `expected absent, got ${describe(value)}` };
+        case 'exact':
+            return deepEqual(value, matcher.value)
+                ? { passed: true }
+                : { passed: false, detail: `expected ${describe(matcher.value)}, got ${describe(value)}` };
+        case 'nonEmpty': {
+            // Generic non-emptiness (payload paths hold arrays and objects, not only strings).
+            const ok = typeof value === 'string' ? value.trim().length > 0
+                : Array.isArray(value) ? value.length > 0
+                    : value !== null && typeof value === 'object' ? Object.keys(value).length > 0
+                        : value !== null && value !== undefined;
+            return ok ? { passed: true } : { passed: false, detail: `expected a non-empty value, got ${describe(value)}` };
+        }
+        case 'nonEmptyString':
+            return typeof value === 'string' && value.trim().length > 0
+                ? { passed: true }
+                : { passed: false, detail: `expected a non-empty string, got ${describe(value)}` };
+        case 'containsIgnoreCase':
+            return typeof value === 'string' && value.toLowerCase().includes(matcher.value.toLowerCase())
+                ? { passed: true }
+                : { passed: false, detail: `expected text containing "${matcher.value}", got ${describe(value)}` };
+        case 'oneOf':
+            return matcher.values.some((candidate) => deepEqual(candidate, value))
+                ? { passed: true }
+                : { passed: false, detail: `expected one of ${describe(matcher.values)}, got ${describe(value)}` };
+        case 'regex': {
+            // A bad pattern in a test record is a authoring bug, and reporting it as a failed
+            // match would hide it. Surface it as its own message instead.
+            let expression;
+            try {
+                expression = new RegExp(matcher.pattern, matcher.flags);
+            }
+            catch (error) {
+                return { passed: false, detail: `invalid regex in expectation: ${error.message}` };
+            }
+            return typeof value === 'string' && expression.test(value)
+                ? { passed: true }
+                : { passed: false, detail: `expected /${matcher.pattern}/${matcher.flags ?? ''}, got ${describe(value)}` };
+        }
+        case 'numberRange': {
+            if (typeof value !== 'number' || Number.isNaN(value)) {
+                return { passed: false, detail: `expected a number, got ${describe(value)}` };
+            }
+            const belowMin = matcher.min !== undefined && value < matcher.min;
+            const aboveMax = matcher.max !== undefined && value > matcher.max;
+            return belowMin || aboveMax
+                ? { passed: false, detail: `expected ${matcher.min ?? '-∞'}..${matcher.max ?? '∞'}, got ${value}` }
+                : { passed: true };
+        }
+        case 'typeOf': {
+            const actual = Array.isArray(value) ? 'array' : typeof value;
+            return actual === matcher.type
+                ? { passed: true }
+                : { passed: false, detail: `expected type ${matcher.type}, got ${actual}` };
+        }
+        default:
+            // Exhaustive today; a `default` keeps the function total if a variant is added.
+            return { passed: false, detail: 'unknown matcher kind' };
+    }
+}
+/** Runs every expectation against one params object. */
+export function matchParams(expectations, params) {
+    return expectations.map((expectation) => {
+        const value = params[expectation.param];
+        if (expectation.optional && (value === undefined || value === null)) {
+            return { param: expectation.param, matcherKind: expectation.matcher.kind, passed: true };
+        }
+        const { passed, detail } = applyMatcher(expectation.matcher, value);
+        return { param: expectation.param, matcherKind: expectation.matcher.kind, passed, detail };
+    });
+}
+//# sourceMappingURL=matchers.js.map

package/dist/eval/matchers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/eval/matchers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA6CH,SAAS,QAAQ,CAAC,KAAc;IAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3E,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,SAAS,CAAC,CAAU,EAAE,CAAU;IACrC,4FAA4F;IAC5F,yFAAyF;IACzF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACzE,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,SAAS,CAAE,CAA6B,CAAC,CAAC,CAAC,EAAG,CAA6B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpI,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,YAAY,CAAC,OAAqB,EAAE,KAAc;IAC9D,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,QAAQ;YACT,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBACxC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,wBAAwB,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC/E,KAAK,OAAO;YACR,OAAO,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;gBAClC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnG,KAAK,UAAU,CAAC,CAAC,CAAC;YACd,mFAAmF;YACnF,MAAM,EAAE,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBAC1D,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;oBACzC,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC;wBAC7E,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;YAC5C,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mCAAmC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC;QACD,KAAK,gBAAgB;YACjB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACvD,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,oCAAoC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3F,KAAK,oBAAoB;YACrB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBACzF,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3G,KAAK,OAAO;YACR,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAClE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3G,KAAK,OAAO,CAAC,CAAC,CAAC;YACX,kFAAkF;YAClF,8DAA8D;YAC9D,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACD,UAAU,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAAE,CAAC;YAClG,CAAC;YACD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBACtD,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACjB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;YAClF,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;YAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;YAClE,OAAO,QAAQ,IAAI,QAAQ;gBACvB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,KAAK,EAAE,EAAE;gBACnG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACZ,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC;YAC7D,OAAO,MAAM,KAAK,OAAO,CAAC,IAAI;gBAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,OAAO,CAAC,IAAI,SAAS,MAAM,EAAE,EAAE,CAAC;QACpF,CAAC;QACD;YACI,gFAAgF;YAChF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;IACjE,CAAC;AACL,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,WAAW,CAAC,YAAgC,EAAE,MAA+B;IACzF,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,WAAW,CAAC,QAAQ,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC7F,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC/F,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/eval/subAgentTrace.d.ts

@@ -0,0 +1,52 @@
+/**
+ * subAgentTrace.ts — the judgement half of the `trace-validate-sub-agents` oracle.
+ *
+ * Split from the traversal so the decision logic is a pure function over "which sub-agents ran and
+ * how many iterations happened" (test plan §2.1). The traversal needs a database; this does not,
+ * and this is the part with the semantics worth pinning in a unit test.
+ *
+ * FRAMEWORK-FREE — imports nothing from the testing engine.
+ */
+/** What the traversal found. */
+export interface SubAgentTraceFacts {
+    /** Every sub-agent dispatched anywhere in the run tree, in dispatch order, duplicates kept. */
+    dispatchedAgents: string[];
+    /** Loop iterations on the top-level run. See {@link SubAgentTraceConfig.minIterations}. */
+    iterations: number;
+}
+/** The oracle's configuration, exactly as the shipped research-agent tests already supply it. */
+export interface SubAgentTraceConfig {
+    /** Every one of these must have been dispatched. Compared case-insensitively. */
+    requiredAgents?: string[];
+    /** None of these may have been dispatched — the "didn't over-delegate" check. */
+    forbiddenAgents?: string[];
+    /**
+     * Minimum loop iterations.
+     *
+     * An "iteration" is one Prompt step on the top-level run: the Loop agent type runs exactly one
+     * prompt per pass, so prompt steps are the only honest proxy the run tree offers. Sub-agent
+     * prompt steps are excluded — they are the *child's* iterations, not the parent's.
+     */
+    minIterations?: number;
+}
+export interface SubAgentTraceResult {
+    passed: boolean;
+    score: number;
+    message: string;
+    details: {
+        dispatchedAgents: string[];
+        missingAgents: string[];
+        forbiddenDispatched: string[];
+        iterations: number;
+        minIterations?: number;
+    };
+}
+/**
+ * Judges a run's sub-agent dispatch against the expectation.
+ *
+ * Names are compared case-insensitively and after trimming, because an agent name in a test record
+ * is typed by a human while the one in the trace comes from the catalog — a casing difference is
+ * never the failure anyone means to report.
+ */
+export declare function evaluateSubAgentTrace(facts: SubAgentTraceFacts, config: SubAgentTraceConfig): SubAgentTraceResult;
+//# sourceMappingURL=subAgentTrace.d.ts.map

package/dist/eval/subAgentTrace.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"subAgentTrace.d.ts","sourceRoot":"","sources":["../../src/eval/subAgentTrace.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,gCAAgC;AAChC,MAAM,WAAW,kBAAkB;IAC/B,+FAA+F;IAC/F,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,iGAAiG;AACjG,MAAM,WAAW,mBAAmB;IAChC,iFAAiF;IACjF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iFAAiF;IACjF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAChC,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,mBAAmB,EAAE,MAAM,EAAE,CAAC;QAC9B,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACL;AAID;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,GAAG,mBAAmB,CA+CjH"}

package/dist/eval/subAgentTrace.js

@@ -0,0 +1,61 @@
+/**
+ * subAgentTrace.ts — the judgement half of the `trace-validate-sub-agents` oracle.
+ *
+ * Split from the traversal so the decision logic is a pure function over "which sub-agents ran and
+ * how many iterations happened" (test plan §2.1). The traversal needs a database; this does not,
+ * and this is the part with the semantics worth pinning in a unit test.
+ *
+ * FRAMEWORK-FREE — imports nothing from the testing engine.
+ */
+const normalize = (name) => name.trim().toLowerCase();
+/**
+ * Judges a run's sub-agent dispatch against the expectation.
+ *
+ * Names are compared case-insensitively and after trimming, because an agent name in a test record
+ * is typed by a human while the one in the trace comes from the catalog — a casing difference is
+ * never the failure anyone means to report.
+ */
+export function evaluateSubAgentTrace(facts, config) {
+    const dispatched = new Set(facts.dispatchedAgents.map(normalize));
+    const missingAgents = (config.requiredAgents ?? []).filter((name) => !dispatched.has(normalize(name)));
+    const forbiddenDispatched = (config.forbiddenAgents ?? []).filter((name) => dispatched.has(normalize(name)));
+    const iterationsShort = config.minIterations !== undefined && facts.iterations < config.minIterations;
+    const details = {
+        dispatchedAgents: facts.dispatchedAgents,
+        missingAgents,
+        forbiddenDispatched,
+        iterations: facts.iterations,
+        minIterations: config.minIterations
+    };
+    const problems = [];
+    if (missingAgents.length > 0) {
+        problems.push(`never dispatched required sub-agent(s): ${missingAgents.join(', ')}`);
+    }
+    if (forbiddenDispatched.length > 0) {
+        problems.push(`dispatched forbidden sub-agent(s): ${forbiddenDispatched.join(', ')}`);
+    }
+    if (iterationsShort) {
+        problems.push(`expected at least ${config.minIterations} iteration(s), saw ${facts.iterations}`);
+    }
+    if (problems.length > 0) {
+        // Partial credit, so a run that got two of three required sub-agents scores above one that
+        // got none. The three checks are weighted equally and only the ones configured count.
+        const checks = [];
+        if (config.requiredAgents?.length) {
+            checks.push(missingAgents.length === 0);
+        }
+        if (config.forbiddenAgents?.length) {
+            checks.push(forbiddenDispatched.length === 0);
+        }
+        if (config.minIterations !== undefined) {
+            checks.push(!iterationsShort);
+        }
+        const score = checks.length === 0 ? 0 : checks.filter(Boolean).length / checks.length;
+        return { passed: false, score, message: problems.join('; '), details };
+    }
+    const summary = facts.dispatchedAgents.length > 0
+        ? `dispatched ${facts.dispatchedAgents.length} sub-agent step(s): ${[...new Set(facts.dispatchedAgents)].join(', ')}`
+        : 'no sub-agents dispatched (none required)';
+    return { passed: true, score: 1, message: `${summary}; ${facts.iterations} iteration(s)`, details };
+}
+//# sourceMappingURL=subAgentTrace.js.map

package/dist/eval/subAgentTrace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"subAgentTrace.js","sourceRoot":"","sources":["../../src/eval/subAgentTrace.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAuCH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAEtE;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAyB,EAAE,MAA2B;IACxF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvG,MAAM,mBAAmB,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,eAAe,GAAG,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC;IAEtG,MAAM,OAAO,GAAG;QACZ,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,aAAa;QACb,mBAAmB;QACnB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAC;IAEF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC,2CAA2C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,sCAAsC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,eAAe,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,aAAa,sBAAsB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,2FAA2F;QAC3F,sFAAsF;QACtF,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACtF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;IAC3E,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC7C,CAAC,CAAC,cAAc,KAAK,CAAC,gBAAgB,CAAC,MAAM,uBAAuB,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACrH,CAAC,CAAC,0CAA0C,CAAC;IACjD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,KAAK,KAAK,CAAC,UAAU,eAAe,EAAE,OAAO,EAAE,CAAC;AACxG,CAAC"}

package/dist/eval/wellFormed.d.ts

@@ -0,0 +1,41 @@
+import type { ObservedDecision } from './decision.js';
+/** What the harness knows about the turn beyond the decision itself. */
+export interface WellFormedInput {
+    decision: ObservedDecision;
+    /** The provider's `finish_reason`, when the driver captured one. */
+    finishReason?: string | null;
+    /** Set when the prompt run itself failed (provider error, timeout, cancellation). */
+    executionError?: string | null;
+}
+export interface WellFormedConfig {
+    /**
+     * Whether an envelope was required on this turn.
+     *
+     * Native-mode cells set this false: a clean tool call with no text is a *complete* answer
+     * there, and text never accompanies a call on Google — so requiring an
+     * envelope would fail every correct native turn.
+     */
+    requireEnvelope?: boolean;
+    /**
+     * Finish reasons that indicate the model was cut off or refused rather than finishing.
+     *
+     * Defaults cover the values seen across the provider matrix plus the documented failure modes. Values are
+     * compared case-insensitively because providers disagree on casing — see MJ#4335.
+     */
+    unacceptableFinishReasons?: string[];
+}
+export interface WellFormedResult {
+    passed: boolean;
+    score: number;
+    message: string;
+    details: {
+        kind: ObservedDecision['kind'];
+        encoding: ObservedDecision['encoding'];
+        envelopeParsed: boolean | null;
+        stepType: string | null;
+        finishReason: string | null;
+    };
+}
+/** Judges whether a turn produced usable output. */
+export declare function evaluateWellFormed(input: WellFormedInput, config?: WellFormedConfig): WellFormedResult;
+//# sourceMappingURL=wellFormed.d.ts.map

package/dist/eval/wellFormed.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wellFormed.d.ts","sourceRoot":"","sources":["../../src/eval/wellFormed.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAanD,wEAAwE;AACxE,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,qFAAqF;IACrF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,gBAAgB;IAC7B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;;OAKG;IACH,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;CACxC;AAWD,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC/B,QAAQ,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACvC,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;QAC/B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC;CACL;AAED,oDAAoD;AACpD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,GAAE,gBAAqB,GAAG,gBAAgB,CAqD1G"}

package/dist/eval/wellFormed.js

@@ -0,0 +1,83 @@
+/**
+ * wellFormed.ts — metrics 1 and 2: was the output usable at all?
+ *
+ * This is the oracle that measures the `MALFORMED_FUNCTION_CALL` class directly (test plan §3.2,
+ * §6.1) — the headline number the whole native-tool-calling effort exists to move. It answers a
+ * narrower question than {@link evaluateDecision}: not "was the decision right" but "did the model
+ * produce something the framework could read." A run can be well-formed and wrong; it cannot be
+ * malformed and right.
+ *
+ * Keeping the two separate matters for attribution. A comparison run weighs baseline against native, and
+ * "malformed rate fell" and "decision accuracy rose" are different claims with different causes.
+ *
+ * FRAMEWORK-FREE (test plan §2.1) — imports nothing from the testing engine.
+ */
+import { LOOP_NEXT_STEP_TYPES } from '@memberjunction/ai-agents';
+/**
+ * The step types the runtime will actually dispatch, derived from the agent framework rather than
+ * restated here — a hand-copied list would drift the moment a step type is added.
+ *
+ * Casing is load-bearing: `LoopAgentType.DetermineNextStep` switches on the raw string, so a
+ * lower-cased `'chat'` passes the validator (which compares case-insensitively) and then falls
+ * through to `default:` — a forced Retry that costs a whole iteration. That is a malformed
+ * response by any measure the issue cares about, so it is checked here.
+ */
+const DISPATCHABLE_STEP_TYPES = new Set(LOOP_NEXT_STEP_TYPES);
+const DEFAULT_UNACCEPTABLE_FINISH_REASONS = [
+    'malformed_function_call',
+    'content_filter',
+    'safety',
+    'recitation',
+    'length',
+    'max_tokens'
+];
+/** Judges whether a turn produced usable output. */
+export function evaluateWellFormed(input, config = {}) {
+    const { decision } = input;
+    const details = {
+        kind: decision.kind,
+        encoding: decision.encoding,
+        envelopeParsed: decision.envelopeParsed,
+        stepType: decision.stepType ?? null,
+        finishReason: input.finishReason ?? null
+    };
+    const fail = (message) => ({ passed: false, score: 0, message, details });
+    // An execution error outranks everything: there is no output to judge.
+    if (input.executionError) {
+        return fail(`execution failed: ${input.executionError}`);
+    }
+    if (decision.kind === 'empty') {
+        return fail('no output received from model');
+    }
+    if (decision.kind === 'unparseable') {
+        return fail(decision.diagnostic ?? 'output could not be read as a decision');
+    }
+    const unacceptable = (config.unacceptableFinishReasons ?? DEFAULT_UNACCEPTABLE_FINISH_REASONS).map((r) => r.toLowerCase());
+    const finish = input.finishReason?.toLowerCase().trim();
+    if (finish && unacceptable.includes(finish)) {
+        return fail(`unacceptable finish reason: ${input.finishReason}`);
+    }
+    if (decision.stepType && !DISPATCHABLE_STEP_TYPES.has(decision.stepType)) {
+        const canonical = LOOP_NEXT_STEP_TYPES.find((t) => t.toLowerCase() === decision.stepType?.toLowerCase().trim());
+        return fail(canonical
+            ? `nextStep.type '${decision.stepType}' is the wrong case — the dispatcher switches on '${canonical}' and would force a Retry`
+            : `nextStep.type '${decision.stepType}' is not a dispatchable step type`);
+    }
+    // Parsed JSON that is not a LoopAgentResponse at all: no dispatchable nextStep, not complete,
+    // no payload write, no responseForm. Production has nothing to act on and forces a Retry — the
+    // same cost as a bad step type above, so it belongs in the same bucket. Counting it as usable
+    // is what let Cerebras's `{"weather":"unavailable", ...}` score as a usable envelope response.
+    // Guarded on stepType being absent: a valid-but-contentless step ('Retry') IS dispatchable.
+    if (decision.kind === 'other' && !decision.stepType) {
+        return fail('parsed JSON is not a LoopAgentResponse — nothing for the loop to dispatch');
+    }
+    if (config.requireEnvelope === true && decision.envelopeParsed !== true) {
+        return fail(decision.encoding === 'native'
+            // Worth naming precisely: this is the exact mechanism the provider matrix reproduced, where a
+            // correct decision arrives through a channel the caller was not reading.
+            ? 'answered with a native tool call while an envelope was required'
+            : 'envelope required but not parsed');
+    }
+    return { passed: true, score: 1, message: `usable ${decision.encoding} response (${decision.kind})`, details };
+}
+//# sourceMappingURL=wellFormed.js.map

package/dist/eval/wellFormed.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wellFormed.js","sourceRoot":"","sources":["../../src/eval/wellFormed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAGjE;;;;;;;;GAQG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAS,oBAAoB,CAAC,CAAC;AA6BtE,MAAM,mCAAmC,GAAG;IACxC,yBAAyB;IACzB,gBAAgB;IAChB,QAAQ;IACR,YAAY;IACZ,QAAQ;IACR,YAAY;CACf,CAAC;AAeF,oDAAoD;AACpD,MAAM,UAAU,kBAAkB,CAAC,KAAsB,EAAE,SAA2B,EAAE;IACpF,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAC3B,MAAM,OAAO,GAAG;QACZ,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,IAAI;QACnC,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;KAC3C,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,OAAe,EAAoB,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAEpG,uEAAuE;IACvE,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,qBAAqB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,wCAAwC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,yBAAyB,IAAI,mCAAmC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3H,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACxD,IAAI,MAAM,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,+BAA+B,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvE,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAChH,OAAO,IAAI,CAAC,SAAS;YACjB,CAAC,CAAC,kBAAkB,QAAQ,CAAC,QAAQ,qDAAqD,SAAS,2BAA2B;YAC9H,CAAC,CAAC,kBAAkB,QAAQ,CAAC,QAAQ,mCAAmC,CAAC,CAAC;IAClF,CAAC;IAED,8FAA8F;IAC9F,+FAA+F;IAC/F,8FAA8F;IAC9F,+FAA+F;IAC/F,4FAA4F;IAC5F,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,2EAA2E,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,MAAM,CAAC,eAAe,KAAK,IAAI,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;YACtC,8FAA8F;YAC9F,yEAAyE;YACzE,CAAC,CAAC,iEAAiE;YACnE,CAAC,CAAC,kCAAkC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,UAAU,QAAQ,CAAC,QAAQ,cAAc,QAAQ,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,CAAC;AACnH,CAAC"}

package/dist/index.d.ts

@@ -7,9 +7,12 @@
 export * from './engine/TestEngine.js';
 export * from './drivers/BaseTestDriver.js';
 export * from './drivers/AgentEvalDriver.js';
+export * from './drivers/PromptEvalDriver.js';
 export * from './oracles/IOracle.js';
 export * from './oracles/SchemaValidatorOracle.js';
 export * from './oracles/TraceValidatorOracle.js';
+export * from './oracles/TraceSubAgentValidatorOracle.js';
+export * from './oracles/AgentDecisionOracle.js';
 export * from './oracles/LLMJudgeOracle.js';
 export * from './oracles/ExactMatchOracle.js';
 export * from './oracles/SQLValidatorOracle.js';
@@ -19,4 +22,6 @@ export * from './utils/cost-calculator.js';
 export * from './utils/result-formatter.js';
 export * from './utils/execution-context.js';
 export * from './utils/variable-resolver.js';
+export * from './eval/index.js';
+export * from './drivers/AgentPromptComposer.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,cAAc,qBAAqB,CAAC;AAGpC,cAAc,0BAA0B,CAAC;AAGzC,cAAc,2BAA2B,CAAC;AAG1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,8BAA8B,CAAC;AAG7C,cAAc,SAAS,CAAC;AAGxB,cAAc,iBAAiB,CAAC;AAChC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,cAAc,qBAAqB,CAAC;AAGpC,cAAc,0BAA0B,CAAC;AAGzC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,4BAA4B,CAAC;AAG3C,cAAc,mBAAmB,CAAC;AAClC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,wCAAwC,CAAC;AACvD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,8BAA8B,CAAC;AAG7C,cAAc,SAAS,CAAC;AAGxB,cAAc,iBAAiB,CAAC;AAChC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,QAAQ,CAAC;AACvB,cAAc,+BAA+B,CAAC"}

package/dist/index.js

@@ -10,10 +10,13 @@ export * from './engine/TestEngine.js';
 export * from './drivers/BaseTestDriver.js';
 // Concrete drivers
 export * from './drivers/AgentEvalDriver.js';
+export * from './drivers/PromptEvalDriver.js';
 // Oracle interface and implementations
 export * from './oracles/IOracle.js';
 export * from './oracles/SchemaValidatorOracle.js';
 export * from './oracles/TraceValidatorOracle.js';
+export * from './oracles/TraceSubAgentValidatorOracle.js';
+export * from './oracles/AgentDecisionOracle.js';
 export * from './oracles/LLMJudgeOracle.js';
 export * from './oracles/ExactMatchOracle.js';
 export * from './oracles/SQLValidatorOracle.js';
@@ -25,4 +28,6 @@ export * from './utils/cost-calculator.js';
 export * from './utils/result-formatter.js';
 export * from './utils/execution-context.js';
 export * from './utils/variable-resolver.js';
+export * from './eval/index.js';
+export * from './drivers/AgentPromptComposer.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc;AACd,cAAc,qBAAqB,CAAC;AAEpC,eAAe;AACf,cAAc,0BAA0B,CAAC;AAEzC,mBAAmB;AACnB,cAAc,2BAA2B,CAAC;AAE1C,uCAAuC;AACvC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,8BAA8B,CAAC;AAE7C,uBAAuB;AACvB,cAAc,SAAS,CAAC;AAExB,YAAY;AACZ,cAAc,iBAAiB,CAAC;AAChC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,2BAA2B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc;AACd,cAAc,qBAAqB,CAAC;AAEpC,eAAe;AACf,cAAc,0BAA0B,CAAC;AAEzC,mBAAmB;AACnB,cAAc,2BAA2B,CAAC;AAC1C,cAAc,4BAA4B,CAAC;AAE3C,uCAAuC;AACvC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,wCAAwC,CAAC;AACvD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,8BAA8B,CAAC;AAE7C,uBAAuB;AACvB,cAAc,SAAS,CAAC;AAExB,YAAY;AACZ,cAAc,iBAAiB,CAAC;AAChC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,QAAQ,CAAC;AACvB,cAAc,+BAA+B,CAAC"}

package/dist/oracles/AgentDecisionOracle.d.ts

@@ -0,0 +1,54 @@
+/**
+ * @fileoverview Agent-decision oracles — the centrepiece of the prompt-eval corpus (test plan §3.2).
+ * @module @memberjunction/testing-engine
+ */
+import { IOracle } from './IOracle.js';
+import { OracleInput, OracleConfig, OracleResult } from '../types.js';
+import { type RawTurn } from '../eval/decision.js';
+/**
+ * The shape a driver hands oracles as `actualOutput` for a prompt-eval test.
+ *
+ * Deliberately minimal and encoding-neutral: the raw text and, when the driver requested tools, the
+ * normalized calls. Anything richer would couple the oracles to a driver, and the whole point of
+ * §2.1 is that the same evaluation runs under a different runner unchanged.
+ */
+export interface PromptEvalActualOutput {
+    turn: RawTurn;
+    finishReason?: string | null;
+    executionError?: string | null;
+}
+/**
+ * `agent-decision-match` — did the agent decide the right thing?
+ *
+ * Reads the decision off whichever channel the model used (envelope or native tool calls) and
+ * compares it against an encoding-independent expectation. That normalization is what lets a
+ * baseline cell and a native cell be compared on an identical corpus case: the
+ * yardstick does not move between them, only the model's behavior does.
+ *
+ * The result's `details` carry the components separately — decision kind, action accuracy, param
+ * fidelity, per-matcher outcomes — so §6.1's metrics aggregate out of persisted `ResultDetails`
+ * without re-running anything.
+ *
+ * Configuration is the {@link DecisionExpectation} itself.
+ */
+export declare class AgentDecisionOracle implements IOracle {
+    readonly type = "agent-decision-match";
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+}
+/**
+ * `response-well-formed` — was the output usable at all?
+ *
+ * This is the oracle that measures the malformed/discard class directly — the issue's headline
+ * number. It asks strictly less than {@link AgentDecisionOracle}: a turn can be well-formed and
+ * wrong, but never malformed and right. Keeping them apart is what lets a comparison say *which* of
+ * "malformed rate fell" and "decision accuracy rose" actually happened.
+ *
+ * Configuration is a {@link WellFormedConfig}. Note `requireEnvelope` defaults to **off**: measurement
+ * measured that a correct native tool call carries no text at all on Google, so requiring an
+ * envelope unconditionally would fail every correct native turn.
+ */
+export declare class ResponseWellFormedOracle implements IOracle {
+    readonly type = "response-well-formed";
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+}
+//# sourceMappingURL=AgentDecisionOracle.d.ts.map

package/dist/oracles/AgentDecisionOracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgentDecisionOracle.d.ts","sourceRoot":"","sources":["../../src/oracles/AgentDecisionOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAqB,KAAK,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAKnE;;;;;;GAMG;AACH,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,OAAO,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AASD;;;;;;;;;;;;;GAaG;AACH,qBAAa,mBAAoB,YAAW,OAAO;IAC/C,QAAQ,CAAC,IAAI,0BAA0B;IAEjC,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;CA+ClF;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,wBAAyB,YAAW,OAAO;IACpD,QAAQ,CAAC,IAAI,0BAA0B;IAEjC,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;CAalF"}