@itaila/archetype 0.3.30

package/dist/evals/index.js

@@ -0,0 +1,11 @@
+export { runEvalConversation } from './runtime.js';
+export { auditBrainBloat } from './brain-bloat.js';
+export { auditOperationalPromptContract } from './operational-contract.js';
+export { auditPromptContent } from './prompt-content.js';
+export { auditCrossLayerDuplicates } from './cross-layer-duplicates.js';
+export { auditActionContracts } from './action-contracts.js';
+export { auditBrainPrescriptions } from './brain-prescriptions.js';
+export { auditEntityVisibility } from './entity-visibility.js';
+export { SAMPLE_PROJECTS, coachProject, nutritionProject, fitnessProject, languageTutorProject, chiefOfStaffProject, } from './sample-projects.js';
+export { judgeEvalTurn, judgeEvalConversation, judgePairwiseConversations, } from './judge.js';
+//# sourceMappingURL=index.js.map

package/dist/evals/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,EAAE,8BAA8B,EAAE,MAAM,2BAA2B,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAA;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAA;AAC5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AAClE,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAwC9D,OAAO,EACL,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,cAAc,EACd,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,0BAA0B,GAC3B,MAAM,YAAY,CAAA"}

package/dist/evals/judge.d.ts

@@ -0,0 +1,22 @@
+import type { EvalJudgeScenario, EvalJudgeVerdict, EvalPairwiseVerdict, EvalTurnResult } from './types.js';
+export declare function judgeEvalTurn(apiKey: string, projectName: string, failureSurface: string, scenario: EvalJudgeScenario, transcript: string, turn: EvalTurnResult, options?: {
+    model?: string;
+    fallbackModels?: string[];
+}): Promise<EvalJudgeVerdict>;
+export declare function judgeEvalConversation(apiKey: string, projectName: string, failureSurface: string, scenario: EvalJudgeScenario, transcript: string, stateBefore: string, stateAfter: string, options?: {
+    model?: string;
+    fallbackModels?: string[];
+}): Promise<EvalJudgeVerdict>;
+export declare function judgePairwiseConversations(apiKey: string, scenario: EvalJudgeScenario, conversationA: {
+    label: string;
+    transcript: string;
+    stateAfter: string;
+}, conversationB: {
+    label: string;
+    transcript: string;
+    stateAfter: string;
+}, options?: {
+    model?: string;
+    fallbackModels?: string[];
+}): Promise<EvalPairwiseVerdict>;
+//# sourceMappingURL=judge.d.ts.map

package/dist/evals/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,cAAc,EACf,MAAM,YAAY,CAAA;AA8HnB,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,EACtB,QAAQ,EAAE,iBAAiB,EAC3B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,cAAc,EACpB,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACtD,OAAO,CAAC,gBAAgB,CAAC,CAuF3B;AAED,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,EACtB,QAAQ,EAAE,iBAAiB,EAC3B,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACtD,OAAO,CAAC,gBAAgB,CAAC,CA8D3B;AAED,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,iBAAiB,EAC3B,aAAa,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EACxE,aAAa,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EACxE,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACtD,OAAO,CAAC,mBAAmB,CAAC,CAsE9B"}

package/dist/evals/judge.js

@@ -0,0 +1,337 @@
+import { GoogleGenerativeAI, SchemaType } from '@google/generative-ai';
+const JUDGE_MODEL = 'gemini-3.5-flash';
+const JUDGE_FALLBACK_CHAIN = ['gemini-3.1-pro-preview', 'gemini-3.1-flash-lite'];
+const JUDGE_MAX_RETRIES = 2;
+/**
+ * Resolve the model chain for judge calls: caller override → env → default.
+ * Returns [primary, ...fallbacks] with duplicates removed.
+ */
+function resolveJudgeChain(primaryOverride, fallbackOverride) {
+    const primary = primaryOverride ?? process.env.ARCHETYPE_JUDGE_MODEL ?? JUDGE_MODEL;
+    const envFallbacks = (process.env.ARCHETYPE_JUDGE_FALLBACK_MODELS ?? '')
+        .split(',')
+        .map((s) => s.trim())
+        .filter(Boolean);
+    const fallbacks = fallbackOverride ?? (envFallbacks.length > 0 ? envFallbacks : JUDGE_FALLBACK_CHAIN);
+    const chain = [primary, ...fallbacks];
+    const seen = new Set();
+    return chain.filter((m) => {
+        if (seen.has(m))
+            return false;
+        seen.add(m);
+        return true;
+    });
+}
+function isRetryable(error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return /\b(429|5\d\d|timeout|timed out|overloaded|unavailable|resource exhausted)\b/i.test(msg);
+}
+async function callJudgeWithFallback(apiKey, modelChain, makeModel, input) {
+    const genAI = new GoogleGenerativeAI(apiKey);
+    let lastError;
+    for (let m = 0; m < modelChain.length; m++) {
+        const current = modelChain[m];
+        if (m > 0) {
+            console.warn(`[archetype:judge] ${modelChain[m - 1]} exhausted — falling back to ${current}`);
+        }
+        const model = makeModel(genAI, current);
+        for (let attempt = 0; attempt <= JUDGE_MAX_RETRIES; attempt++) {
+            try {
+                const result = await model.generateContent(input);
+                return result.response.text();
+            }
+            catch (err) {
+                lastError = err instanceof Error ? err : new Error(String(err));
+                if (!isRetryable(lastError))
+                    throw lastError;
+                if (attempt < JUDGE_MAX_RETRIES) {
+                    const delayMs = Math.min(500 * Math.pow(2, attempt), 4000);
+                    console.warn(`[archetype:judge] ${current} attempt ${attempt + 1} failed: ${lastError.message} — retrying in ${delayMs}ms`);
+                    await new Promise((r) => setTimeout(r, delayMs));
+                }
+            }
+        }
+    }
+    throw lastError ?? new Error('[archetype:judge] all models exhausted');
+}
+const JUDGE_CRITERIA = [
+    {
+        name: 'Human voice',
+        desc2: 'Reads like a capable human in this role, not like an LLM performing a persona.',
+        desc0: 'Feels robotic, generic, or obviously prompt-shaped.',
+    },
+    {
+        name: 'Relationship fit',
+        desc2: 'The trust posture matches the domain: warm where needed, firm where needed, never off-key.',
+        desc0: 'The relationship archetype feels mismatched to the domain or moment.',
+    },
+    {
+        name: 'Judgment',
+        desc2: 'Takes or withholds action appropriately; good sense of when to talk, when to act, and when to confirm.',
+        desc0: 'Acts too eagerly, stays too passive, or clearly mis-times action.',
+    },
+    {
+        name: 'Invisible operations',
+        desc2: 'Side-effects happen naturally without awkward tool narration or self-consciousness.',
+        desc0: 'Operations feel clanky, overly explicit, or disrupt the conversation.',
+    },
+    {
+        name: 'Memory hygiene',
+        desc2: 'Remembers the right thing, updates instead of duplicating, and avoids storing junk.',
+        desc0: 'Misses durable signal or creates noisy/duplicated memory.',
+    },
+    {
+        name: 'Goal advancement',
+        desc2: 'Meaningfully moves the user toward the persona north star in this turn.',
+        desc0: 'Sounds okay but does not create real forward motion.',
+    },
+    {
+        name: 'Specificity',
+        desc2: 'Grounded in the provided context and says something particular rather than interchangeable.',
+        desc0: 'Could have been said to almost anyone in almost any situation.',
+    },
+    {
+        name: 'Come-back test',
+        desc2: 'Makes the user want another turn because it is useful, alive, and well-judged.',
+        desc0: 'The user would be less likely to come back after this response.',
+    },
+];
+const JUDGE_PROMPT = `You are evaluating whether an Archetype persona actually works in practice.
+
+Score each criterion 0, 1, or 2.
+
+${JUDGE_CRITERIA.map((criterion, index) => `${index + 1}. ${criterion.name}
+- 2: ${criterion.desc2}
+- 1: Partially works but leaves meaningful room for improvement.
+- 0: ${criterion.desc0}`).join('\n\n')}
+
+You are not only judging the response quality. You are also stress-testing the concept:
+- Is the persona itself well-shaped for this domain?
+- Does the SDK seem to make the behavior easier or harder?
+- Do the side-effects feel invisible and coherent?
+
+Use the transcript, executed/proposed actions, and state delta.
+Return ONLY valid JSON.`;
+export async function judgeEvalTurn(apiKey, projectName, failureSurface, scenario, transcript, turn, options) {
+    const modelChain = resolveJudgeChain(options?.model, options?.fallbackModels);
+    const schema = {
+        type: SchemaType.OBJECT,
+        properties: {
+            scores: {
+                type: SchemaType.ARRAY,
+                items: {
+                    type: SchemaType.OBJECT,
+                    properties: {
+                        criterion: { type: SchemaType.STRING },
+                        score: { type: SchemaType.NUMBER },
+                        reasoning: { type: SchemaType.STRING },
+                    },
+                    required: ['criterion', 'score', 'reasoning'],
+                },
+            },
+            promptFixes: {
+                type: SchemaType.ARRAY,
+                items: { type: SchemaType.STRING },
+            },
+            sdkGaps: {
+                type: SchemaType.ARRAY,
+                items: { type: SchemaType.STRING },
+            },
+            conceptGaps: {
+                type: SchemaType.ARRAY,
+                items: { type: SchemaType.STRING },
+            },
+        },
+        required: ['scores', 'promptFixes', 'sdkGaps', 'conceptGaps'],
+    };
+    const makeModel = (genAI, modelName) => genAI.getGenerativeModel({
+        model: modelName,
+        systemInstruction: JUDGE_PROMPT,
+        generationConfig: {
+            temperature: 0.2,
+            responseMimeType: 'application/json',
+            responseSchema: schema,
+        },
+    });
+    const actionLines = [];
+    for (const record of turn.actionRecords) {
+        const mode = record.proposed ? 'proposed' : record.success ? 'executed' : 'failed';
+        actionLines.push(`- ${mode}: ${record.action.name}${record.error ? ` (${record.error})` : ''}`);
+    }
+    for (const crud of turn.trace.crudActions) {
+        const mode = crud.status === 'invalid' ? 'invalid' : 'executed';
+        const params = Object.keys(crud.params ?? {}).length > 0 ? ` ${JSON.stringify(crud.params)}` : '';
+        actionLines.push(`- ${mode}: crud ${crud.operation} ${crud.entity}${crud.id ? ` ${crud.id}` : ''}${params}${crud.error ? ` (${crud.error})` : ''}`);
+    }
+    const actionSummary = actionLines.length === 0 ? 'No actions' : actionLines.join('\n');
+    const input = `PROJECT: ${projectName}
+FAILURE SURFACE: ${failureSurface}
+SCENARIO: ${scenario.name}
+DESCRIPTION: ${scenario.description}
+TESTS: ${scenario.tests.join(', ')}
+${scenario.expectedBehavior?.length ? `EXPECTED BEHAVIOR:\n${scenario.expectedBehavior.map(item => `- ${item}`).join('\n')}` : ''}
+
+TRANSCRIPT SO FAR:
+${transcript}
+
+CURRENT TURN:
+USER: ${turn.userMessage}
+ASSISTANT: ${turn.assistantMessage}
+
+ACTIONS:
+${actionSummary}
+
+STATE BEFORE:
+${turn.stateBefore}
+
+STATE AFTER:
+${turn.stateAfter}
+
+Score the current turn.`;
+    const text = await callJudgeWithFallback(apiKey, modelChain, makeModel, input);
+    return parseJudgeVerdict(text);
+}
+export async function judgeEvalConversation(apiKey, projectName, failureSurface, scenario, transcript, stateBefore, stateAfter, options) {
+    const modelChain = resolveJudgeChain(options?.model, options?.fallbackModels);
+    const schema = {
+        type: SchemaType.OBJECT,
+        properties: {
+            scores: {
+                type: SchemaType.ARRAY,
+                items: {
+                    type: SchemaType.OBJECT,
+                    properties: {
+                        criterion: { type: SchemaType.STRING },
+                        score: { type: SchemaType.NUMBER },
+                        reasoning: { type: SchemaType.STRING },
+                    },
+                    required: ['criterion', 'score', 'reasoning'],
+                },
+            },
+            promptFixes: { type: SchemaType.ARRAY, items: { type: SchemaType.STRING } },
+            sdkGaps: { type: SchemaType.ARRAY, items: { type: SchemaType.STRING } },
+            conceptGaps: { type: SchemaType.ARRAY, items: { type: SchemaType.STRING } },
+        },
+        required: ['scores', 'promptFixes', 'sdkGaps', 'conceptGaps'],
+    };
+    const makeModel = (genAI, modelName) => genAI.getGenerativeModel({
+        model: modelName,
+        systemInstruction: `${JUDGE_PROMPT}
+
+In addition to judging the latest turn, judge the entire conversation arc:
+- Does the persona stay coherent across turns?
+- Does it compound value rather than resetting every turn?
+- Does memory use help rather than create drift?
+- Do the actions across turns feel clean and well-timed?`,
+        generationConfig: {
+            temperature: 0.2,
+            responseMimeType: 'application/json',
+            responseSchema: schema,
+        },
+    });
+    const input = `PROJECT: ${projectName}
+FAILURE SURFACE: ${failureSurface}
+SCENARIO: ${scenario.name}
+DESCRIPTION: ${scenario.description}
+TESTS: ${scenario.tests.join(', ')}
+${scenario.expectedBehavior?.length ? `EXPECTED BEHAVIOR:\n${scenario.expectedBehavior.map(item => `- ${item}`).join('\n')}` : ''}
+
+FULL TRANSCRIPT:
+${transcript}
+
+STATE BEFORE:
+${stateBefore}
+
+STATE AFTER:
+${stateAfter}
+
+Score the overall conversation, not just a single turn.`;
+    const text = await callJudgeWithFallback(apiKey, modelChain, makeModel, input);
+    return parseJudgeVerdict(text);
+}
+export async function judgePairwiseConversations(apiKey, scenario, conversationA, conversationB, options) {
+    const modelChain = resolveJudgeChain(options?.model, options?.fallbackModels);
+    const schema = {
+        type: SchemaType.OBJECT,
+        properties: {
+            winner: {
+                type: SchemaType.STRING,
+                enum: ['a', 'b', 'tie'],
+            },
+            reasoning: { type: SchemaType.STRING },
+            promptFixes: { type: SchemaType.ARRAY, items: { type: SchemaType.STRING } },
+            sdkGaps: { type: SchemaType.ARRAY, items: { type: SchemaType.STRING } },
+            conceptGaps: { type: SchemaType.ARRAY, items: { type: SchemaType.STRING } },
+        },
+        required: ['winner', 'reasoning', 'promptFixes', 'sdkGaps', 'conceptGaps'],
+    };
+    const makeModel = (genAI, modelName) => genAI.getGenerativeModel({
+        model: modelName,
+        systemInstruction: `You are comparing two persona conversations for the same scenario.
+
+Decide which conversation is better overall on:
+- human voice
+- relationship fit
+- invisible operations
+- goal advancement
+- state hygiene across turns
+
+Choose:
+- "a" if A is better
+- "b" if B is better
+- "tie" only if they are genuinely comparable
+
+Return ONLY valid JSON.`,
+        generationConfig: {
+            temperature: 0.2,
+            responseMimeType: 'application/json',
+            responseSchema: schema,
+        },
+    });
+    const input = `SCENARIO: ${scenario.name}
+DESCRIPTION: ${scenario.description}
+TESTS: ${scenario.tests.join(', ')}
+
+CONVERSATION A (${conversationA.label}):
+${conversationA.transcript}
+
+STATE AFTER A:
+${conversationA.stateAfter}
+
+CONVERSATION B (${conversationB.label}):
+${conversationB.transcript}
+
+STATE AFTER B:
+${conversationB.stateAfter}
+
+Which conversation is better overall?`;
+    const text = await callJudgeWithFallback(apiKey, modelChain, makeModel, input);
+    const parsed = JSON.parse(text);
+    return {
+        winner: parsed.winner,
+        reasoning: parsed.reasoning,
+        promptFixes: parsed.promptFixes ?? [],
+        sdkGaps: parsed.sdkGaps ?? [],
+        conceptGaps: parsed.conceptGaps ?? [],
+    };
+}
+function parseJudgeVerdict(rawText) {
+    const parsed = JSON.parse(rawText);
+    const scores = parsed.scores.map(score => ({
+        criterion: score.criterion,
+        score: Math.round(score.score),
+        reasoning: score.reasoning,
+    }));
+    const average = scores.reduce((sum, score) => sum + score.score, 0) / scores.length;
+    const hasZero = scores.some(score => score.score === 0);
+    return {
+        scores,
+        average: Math.round(average * 100) / 100,
+        hasZero,
+        pass: average >= 1.5 && !hasZero,
+        promptFixes: parsed.promptFixes ?? [],
+        sdkGaps: parsed.sdkGaps ?? [],
+        conceptGaps: parsed.conceptGaps ?? [],
+    };
+}
+//# sourceMappingURL=judge.js.map

package/dist/evals/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAe,MAAM,uBAAuB,CAAA;AAQnF,MAAM,WAAW,GAAG,kBAAkB,CAAA;AACtC,MAAM,oBAAoB,GAAG,CAAC,wBAAwB,EAAE,uBAAuB,CAAC,CAAA;AAChF,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAE3B;;;GAGG;AACH,SAAS,iBAAiB,CAAC,eAAwB,EAAE,gBAA2B;IAC9E,MAAM,OAAO,GAAG,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,WAAW,CAAA;IACnF,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,EAAE,CAAC;SACrE,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAA;IAClB,MAAM,SAAS,GAAG,gBAAgB,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAA;IACrG,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,CAAA;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAA;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QACX,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAClE,OAAO,8EAA8E,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjG,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,MAAc,EACd,UAAoB,EACpB,SAAkJ,EAClJ,KAAa;IAEb,MAAM,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAC5C,IAAI,SAA4B,CAAA;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,OAAO,CAAC,IAAI,CAAC,qBAAqB,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAA;QAC/F,CAAC;QACD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAEvC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,iBAAiB,EAAE,OAAO,EAAE,EAAE,CAAC;YAC9D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;gBACjD,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAkB,CAAA;YAC/C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;oBAAE,MAAM,SAAS,CAAA;gBAC5C,IAAI,OAAO,GAAG,iBAAiB,EAAE,CAAC;oBAChC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,CAAA;oBAC1D,OAAO,CAAC,IAAI,CAAC,qBAAqB,OAAO,YAAY,OAAO,GAAG,CAAC,YAAY,SAAS,CAAC,OAAO,kBAAkB,OAAO,IAAI,CAAC,CAAA;oBAC3H,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAA;gBAClD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,cAAc,GAAG;IACrB;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,gFAAgF;QACvF,KAAK,EAAE,qDAAqD;KAC7D;IACD;QACE,IAAI,EAAE,kBAAkB;QACxB,KAAK,EAAE,4FAA4F;QACnG,KAAK,EAAE,sEAAsE;KAC9E;IACD;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,wGAAwG;QAC/G,KAAK,EAAE,mEAAmE;KAC3E;IACD;QACE,IAAI,EAAE,sBAAsB;QAC5B,KAAK,EAAE,qFAAqF;QAC5F,KAAK,EAAE,uEAAuE;KAC/E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,qFAAqF;QAC5F,KAAK,EAAE,2DAA2D;KACnE;IACD;QACE,IAAI,EAAE,kBAAkB;QACxB,KAAK,EAAE,yEAAyE;QAChF,KAAK,EAAE,sDAAsD;KAC9D;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,6FAA6F;QACpG,KAAK,EAAE,gEAAgE;KACxE;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,gFAAgF;QACvF,KAAK,EAAE,iEAAiE;KACzE;CACF,CAAA;AAED,MAAM,YAAY,GAAG;;;;EAInB,cAAc,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI;OACnE,SAAS,CAAC,KAAK;;OAEf,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;;;;;;;;wBAQd,CAAA;AAExB,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,WAAmB,EACnB,cAAsB,EACtB,QAA2B,EAC3B,UAAkB,EAClB,IAAoB,EACpB,OAAuD;IAEvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;IAE7E,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,UAAU,CAAC,MAAM;QACvB,UAAU,EAAE;YACV,MAAM,EAAE;gBACN,IAAI,EAAE,UAAU,CAAC,KAAK;gBACtB,KAAK,EAAE;oBACL,IAAI,EAAE,UAAU,CAAC,MAAM;oBACvB,UAAU,EAAE;wBACV,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;wBACtC,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;wBAClC,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;qBACvC;oBACD,QAAQ,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,WAAW,CAAC;iBAC9C;aACF;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,UAAU,CAAC,KAAK;gBACtB,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;aACnC;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU,CAAC,KAAK;gBACtB,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;aACnC;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,UAAU,CAAC,KAAK;gBACtB,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;aACnC;SACF;QACD,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,CAAC;KAC9D,CAAA;IAED,MAAM,SAAS,GAAG,CAAC,KAAyB,EAAE,SAAiB,EAAE,EAAE,CACjE,KAAK,CAAC,kBAAkB,CAAC;QACvB,KAAK,EAAE,SAAS;QAChB,iBAAiB,EAAE,YAAY;QAC/B,gBAAgB,EAAE;YAChB,WAAW,EAAE,GAAG;YAChB,gBAAgB,EAAE,kBAAkB;YACpC,cAAc,EAAE,MAA2B;SAC5C;KACF,CAAC,CAAA;IAEJ,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAA;QAClF,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACjG,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;QAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;QACjG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,UAAU,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACrJ,CAAC;IAED,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEtF,MAAM,KAAK,GAAG,YAAY,WAAW;mBACpB,cAAc;YACrB,QAAQ,CAAC,IAAI;eACV,QAAQ,CAAC,WAAW;SAC1B,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;EAChC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,uBAAuB,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;;;EAG/H,UAAU;;;QAGJ,IAAI,CAAC,WAAW;aACX,IAAI,CAAC,gBAAgB;;;EAGhC,aAAa;;;EAGb,IAAI,CAAC,WAAW;;;EAGhB,IAAI,CAAC,UAAU;;wBAEO,CAAA;IAEtB,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAS,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;IACtF,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAAc,EACd,WAAmB,EACnB,cAAsB,EACtB,QAA2B,EAC3B,UAAkB,EAClB,WAAmB,EACnB,UAAkB,EAClB,OAAuD;IAEvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;IAE7E,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,UAAU,CAAC,MAAM;QACvB,UAAU,EAAE;YACV,MAAM,EAAE;gBACN,IAAI,EAAE,UAAU,CAAC,KAAK;gBACtB,KAAK,EAAE;oBACL,IAAI,EAAE,UAAU,CAAC,MAAM;oBACvB,UAAU,EAAE;wBACV,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;wBACtC,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;wBAClC,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;qBACvC;oBACD,QAAQ,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,WAAW,CAAC;iBAC9C;aACF;YACD,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,EAAE;YAC3E,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,EAAE;YACvE,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,EAAE;SAC5E;QACD,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,CAAC;KAC9D,CAAA;IAED,MAAM,SAAS,GAAG,CAAC,KAAyB,EAAE,SAAiB,EAAE,EAAE,CACjE,KAAK,CAAC,kBAAkB,CAAC;QACvB,KAAK,EAAE,SAAS;QAChB,iBAAiB,EAAE,GAAG,YAAY;;;;;;yDAMiB;QACnD,gBAAgB,EAAE;YAChB,WAAW,EAAE,GAAG;YAChB,gBAAgB,EAAE,kBAAkB;YACpC,cAAc,EAAE,MAA2B;SAC5C;KACF,CAAC,CAAA;IAEJ,MAAM,KAAK,GAAG,YAAY,WAAW;mBACpB,cAAc;YACrB,QAAQ,CAAC,IAAI;eACV,QAAQ,CAAC,WAAW;SAC1B,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;EAChC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,uBAAuB,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;;;EAG/H,UAAU;;;EAGV,WAAW;;;EAGX,UAAU;;wDAE4C,CAAA;IAEtD,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAS,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;IACtF,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,MAAc,EACd,QAA2B,EAC3B,aAAwE,EACxE,aAAwE,EACxE,OAAuD;IAEvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;IAE7E,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,UAAU,CAAC,MAAM;QACvB,UAAU,EAAE;YACV,MAAM,EAAE;gBACN,IAAI,EAAE,UAAU,CAAC,MAAM;gBACvB,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC;aACxB;YACD,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE;YACtC,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,EAAE;YAC3E,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,EAAE;YACvE,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,EAAE;SAC5E;QACD,QAAQ,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,CAAC;KAC3E,CAAA;IAED,MAAM,SAAS,GAAG,CAAC,KAAyB,EAAE,SAAiB,EAAE,EAAE,CACjE,KAAK,CAAC,kBAAkB,CAAC;QACvB,KAAK,EAAE,SAAS;QAChB,iBAAiB,EAAE;;;;;;;;;;;;;;wBAcD;QAClB,gBAAgB,EAAE;YAChB,WAAW,EAAE,GAAG;YAChB,gBAAgB,EAAE,kBAAkB;YACpC,cAAc,EAAE,MAA2B;SAC5C;KACF,CAAC,CAAA;IAEJ,MAAM,KAAK,GAAG,aAAa,QAAQ,CAAC,IAAI;eAC3B,QAAQ,CAAC,WAAW;SAC1B,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;;kBAEhB,aAAa,CAAC,KAAK;EACnC,aAAa,CAAC,UAAU;;;EAGxB,aAAa,CAAC,UAAU;;kBAER,aAAa,CAAC,KAAK;EACnC,aAAa,CAAC,UAAU;;;EAGxB,aAAa,CAAC,UAAU;;sCAEY,CAAA;IAEpC,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAS,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;IACtF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAwB,CAAA;IACtD,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;QACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;KACtC,CAAA;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAKhC,CAAA;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;QAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;KAC3B,CAAC,CAAC,CAAA;IACH,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACnF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAA;IAEvD,OAAO;QACL,MAAM;QACN,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,GAAG;QACxC,OAAO;QACP,IAAI,EAAE,OAAO,IAAI,GAAG,IAAI,CAAC,OAAO;QAChC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;QACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;KACtC,CAAA;AACH,CAAC"}

package/dist/evals/operational-contract.d.ts

@@ -0,0 +1,40 @@
+import type { LLMProviderRequest, PromptMode, TurnTrace } from '../types.js';
+export interface OperationalPromptRequirement {
+    label: string;
+    /**
+     * Concrete tokens that should appear in the prompt when the surface is directly exposed.
+     * Example: actual entity ids, role ids, enum values.
+     */
+    tokens: string[];
+    /**
+     * Tokens that must all survive into the prompt. Use this for continuity
+     * contracts where "some mention happened" is not enough and the next turn
+     * truly needs specific carried-forward information.
+     */
+    requiredTokens?: string[];
+    /**
+     * Alternative retrieval/documentation tokens that satisfy the requirement when the full
+     * surface is intentionally not in working memory.
+     */
+    retrievalTokens?: string[];
+}
+export interface OperationalPromptContractInput {
+    request: Pick<LLMProviderRequest, 'systemPrompt' | 'message'>;
+    trace?: TurnTrace;
+    expectedMode: PromptMode;
+    ids?: OperationalPromptRequirement[];
+    enums?: OperationalPromptRequirement[];
+    recipients?: OperationalPromptRequirement[];
+    continuity?: OperationalPromptRequirement[];
+    allowRepair?: boolean;
+}
+export interface OperationalPromptContractIssue {
+    severity: 'error' | 'warn';
+    message: string;
+}
+export interface OperationalPromptContractResult {
+    pass: boolean;
+    issues: OperationalPromptContractIssue[];
+}
+export declare function auditOperationalPromptContract(input: OperationalPromptContractInput): OperationalPromptContractResult;
+//# sourceMappingURL=operational-contract.d.ts.map

package/dist/evals/operational-contract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"operational-contract.d.ts","sourceRoot":"","sources":["../../src/evals/operational-contract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAE5E,MAAM,WAAW,4BAA4B;IAC3C,KAAK,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,IAAI,CAAC,kBAAkB,EAAE,cAAc,GAAG,SAAS,CAAC,CAAA;IAC7D,KAAK,CAAC,EAAE,SAAS,CAAA;IACjB,YAAY,EAAE,UAAU,CAAA;IACxB,GAAG,CAAC,EAAE,4BAA4B,EAAE,CAAA;IACpC,KAAK,CAAC,EAAE,4BAA4B,EAAE,CAAA;IACtC,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAA;IAC3C,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAA;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAA;CACtB;AAED,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAA;IAC1B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,+BAA+B;IAC9C,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,8BAA8B,EAAE,CAAA;CACzC;AA0CD,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,8BAA8B,GACpC,+BAA+B,CAmFjC"}

package/dist/evals/operational-contract.js

@@ -0,0 +1,115 @@
+const CONVERSATION_MARKERS = [
+    'real person in front of you',
+    'Momentum:',
+    'This is a desktop side panel.',
+    'This is a mobile chat interface.',
+];
+const FOCUS_OUTPUT_MARKER = 'Return one raw JSON object: { "message": "...", "actions": [...] }. No markdown.';
+function containsAny(haystack, needles) {
+    return needles.some(needle => haystack.includes(needle));
+}
+function countOccurrences(haystack, needle) {
+    if (!needle)
+        return 0;
+    return haystack.split(needle).length - 1;
+}
+function checkSurface(issues, prompt, kind, requirements) {
+    for (const requirement of requirements ?? []) {
+        const requiredTokensPresent = (requirement.requiredTokens ?? []).every(token => prompt.includes(token));
+        const hasDirectSurface = containsAny(prompt, requirement.tokens);
+        const hasRetrievalSurface = requirement.retrievalTokens?.length
+            ? containsAny(prompt, requirement.retrievalTokens)
+            : false;
+        if (!requiredTokensPresent || (!hasDirectSurface && !hasRetrievalSurface)) {
+            issues.push({
+                severity: 'error',
+                message: `${kind} surface missing for ${requirement.label}`,
+            });
+        }
+    }
+}
+export function auditOperationalPromptContract(input) {
+    const issues = [];
+    const prompt = input.request.systemPrompt;
+    if (input.expectedMode === 'operational') {
+        const hasActionContract = prompt.includes('--- ACTION RESPONSE CONTRACT ---');
+        const hasEntityCrudContract = prompt.includes('--- ENTITY CRUD RESPONSE CONTRACT ---');
+        if (!prompt.includes('Operational reality:')) {
+            issues.push({ severity: 'error', message: 'Operational prompt is missing operational reality guidance' });
+        }
+        if (!hasActionContract && !hasEntityCrudContract) {
+            issues.push({ severity: 'error', message: 'Operational prompt is missing the canonical action or entity response contract block' });
+        }
+        else if (hasActionContract && countOccurrences(prompt, '--- ACTION RESPONSE CONTRACT ---') > 1) {
+            issues.push({ severity: 'warn', message: 'Operational prompt contains duplicate action API contract blocks' });
+        }
+        if (!prompt.includes('Return exactly one raw JSON object and nothing else.')) {
+            issues.push({ severity: 'error', message: 'Operational prompt is missing the raw JSON output contract' });
+        }
+        if (!prompt.includes('Do not wrap the response in markdown code fences.')) {
+            issues.push({ severity: 'error', message: 'Operational prompt is missing the no-code-fences output rule' });
+        }
+        for (const marker of CONVERSATION_MARKERS) {
+            if (prompt.includes(marker)) {
+                issues.push({ severity: 'error', message: `Operational prompt still contains conversational scaffold: ${marker}` });
+            }
+        }
+    }
+    else if (input.expectedMode === 'focus') {
+        if (!prompt.includes(FOCUS_OUTPUT_MARKER)) {
+            issues.push({ severity: 'error', message: 'Focus prompt is missing the focus JSON output contract' });
+        }
+        if (prompt.includes('Operational reality:')) {
+            issues.push({ severity: 'error', message: 'Focus prompt still contains operational reality guidance' });
+        }
+        for (const marker of CONVERSATION_MARKERS) {
+            if (prompt.includes(marker)) {
+                issues.push({ severity: 'error', message: `Focus prompt still contains conversational scaffold: ${marker}` });
+            }
+        }
+        if (prompt.includes('"followUps":')) {
+            issues.push({ severity: 'warn', message: 'Focus prompt still contains follow-up guidance' });
+        }
+        if (prompt.includes('Outcome notes:')) {
+            issues.push({ severity: 'warn', message: 'Focus prompt still contains outcome-notes guidance' });
+        }
+        const sequentialGuidanceCount = countOccurrences(prompt, 'actions is a list')
+            + countOccurrences(prompt, '"actions" is a list');
+        if (sequentialGuidanceCount > 1) {
+            issues.push({ severity: 'warn', message: 'Focus prompt contains duplicate sequential action guidance' });
+        }
+    }
+    else {
+        if (!prompt.includes('"followUps":')) {
+            issues.push({ severity: 'warn', message: 'Conversation prompt is missing follow-up guidance' });
+        }
+    }
+    checkSurface(issues, prompt, 'ID', input.ids);
+    checkSurface(issues, prompt, 'enum', input.enums);
+    checkSurface(issues, prompt, 'recipient', input.recipients);
+    checkSurface(issues, prompt, 'continuity', input.continuity);
+    if (input.trace) {
+        if (!input.trace.parseOk) {
+            issues.push({ severity: 'error', message: 'Turn trace shows JSON parse failure' });
+        }
+        if (!input.allowRepair && input.trace.repairAttempted) {
+            issues.push({ severity: 'warn', message: 'Turn trace needed repair to satisfy the action contract' });
+        }
+        for (const action of input.trace.actions) {
+            if (action.status === 'unknown_action') {
+                issues.push({ severity: 'error', message: `Turn trace contains unknown action: ${action.name}` });
+            }
+            else if (action.status === 'invalid') {
+                issues.push({ severity: 'warn', message: `Turn trace contains invalid action params: ${action.name}` });
+            }
+        }
+        for (const error of input.trace.errors) {
+            issues.push({ severity: 'warn', message: `Turn trace error: ${error}` });
+        }
+    }
+    return {
+        pass: issues.every(issue => issue.severity !== 'error'),
+        issues,
+    };
+}
+//# sourceMappingURL=operational-contract.js.map

package/dist/evals/operational-contract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operational-contract.js","sourceRoot":"","sources":["../../src/evals/operational-contract.ts"],"names":[],"mappings":"AA2CA,MAAM,oBAAoB,GAAG;IAC3B,6BAA6B;IAC7B,WAAW;IACX,+BAA+B;IAC/B,kCAAkC;CACnC,CAAA;AAED,MAAM,mBAAmB,GAAG,kFAAkF,CAAA;AAE9G,SAAS,WAAW,CAAC,QAAgB,EAAE,OAAiB;IACtD,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAE,MAAc;IACxD,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,CAAA;IACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;AAC1C,CAAC;AAED,SAAS,YAAY,CACnB,MAAwC,EACxC,MAAc,EACd,IAAgD,EAChD,YAA6C;IAE7C,KAAK,MAAM,WAAW,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;QAC7C,MAAM,qBAAqB,GAAG,CAAC,WAAW,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;QACvG,MAAM,gBAAgB,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;QAChE,MAAM,mBAAmB,GAAG,WAAW,CAAC,eAAe,EAAE,MAAM;YAC7D,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,eAAe,CAAC;YAClD,CAAC,CAAC,KAAK,CAAA;QAET,IAAI,CAAC,qBAAqB,IAAI,CAAC,CAAC,gBAAgB,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,GAAG,IAAI,wBAAwB,WAAW,CAAC,KAAK,EAAE;aAC5D,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,KAAqC;IAErC,MAAM,MAAM,GAAqC,EAAE,CAAA;IACnD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,CAAA;IAEzC,IAAI,KAAK,CAAC,YAAY,KAAK,aAAa,EAAE,CAAC;QACzC,MAAM,iBAAiB,GAAG,MAAM,CAAC,QAAQ,CAAC,kCAAkC,CAAC,CAAA;QAC7E,MAAM,qBAAqB,GAAG,MAAM,CAAC,QAAQ,CAAC,uCAAuC,CAAC,CAAA;QACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,4DAA4D,EAAE,CAAC,CAAA;QAC3G,CAAC;QACD,IAAI,CAAC,iBAAiB,IAAI,CAAC,qBAAqB,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,sFAAsF,EAAE,CAAC,CAAA;QACrI,CAAC;aAAM,IAAI,iBAAiB,IAAI,gBAAgB,CAAC,MAAM,EAAE,kCAAkC,CAAC,GAAG,CAAC,EAAE,CAAC;YACjG,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,kEAAkE,EAAE,CAAC,CAAA;QAChH,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,sDAAsD,CAAC,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,4DAA4D,EAAE,CAAC,CAAA;QAC3G,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,mDAAmD,CAAC,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,8DAA8D,EAAE,CAAC,CAAA;QAC7G,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,oBAAoB,EAAE,CAAC;YAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,8DAA8D,MAAM,EAAE,EAAE,CAAC,CAAA;YACrH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,KAAK,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,wDAAwD,EAAE,CAAC,CAAA;QACvG,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,0DAA0D,EAAE,CAAC,CAAA;QACzG,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,oBAAoB,EAAE,CAAC;YAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,wDAAwD,MAAM,EAAE,EAAE,CAAC,CAAA;YAC/G,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,gDAAgD,EAAE,CAAC,CAAA;QAC9F,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC,CAAA;QAClG,CAAC;QACD,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,MAAM,EAAE,mBAAmB,CAAC;cACzE,gBAAgB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnD,IAAI,uBAAuB,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,4DAA4D,EAAE,CAAC,CAAA;QAC1G,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,mDAAmD,EAAE,CAAC,CAAA;QACjG,CAAC;IACH,CAAC;IAED,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;IAC7C,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACjD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;IAC3D,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;IAE5D,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAA;QACpF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,yDAAyD,EAAE,CAAC,CAAA;QACvG,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,uCAAuC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YACnG,CAAC;iBAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,8CAA8C,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YACzG,CAAC;QACH,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,KAAK,EAAE,EAAE,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC;QACvD,MAAM;KACP,CAAA;AACH,CAAC"}

package/dist/evals/prompt-content.d.ts

@@ -0,0 +1,14 @@
+export interface PromptContentAuditInput {
+    prompt: string;
+    declaredEntities?: string[];
+}
+export interface PromptContentAuditIssue {
+    severity: 'error' | 'warn';
+    message: string;
+}
+export interface PromptContentAuditResult {
+    pass: boolean;
+    issues: PromptContentAuditIssue[];
+}
+export declare function auditPromptContent(input: PromptContentAuditInput): PromptContentAuditResult;
+//# sourceMappingURL=prompt-content.d.ts.map

package/dist/evals/prompt-content.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-content.d.ts","sourceRoot":"","sources":["../../src/evals/prompt-content.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAA;IAC1B,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,uBAAuB,EAAE,CAAA;CAClC;AAkCD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,uBAAuB,GAAG,wBAAwB,CAiF3F"}