@xoxoai/checkmate 0.4.28 → 0.5.1

package/dist/runtime/step-execution.d.ts

@@ -1,17 +1,27 @@
-import { Step } from './types.js';
-import { AiClient } from '../ai/client.js';
-import { ExtensionHost } from './extension.js';
+import type { AiClient } from '../ai/client.js';
+import type { RuntimeConfig } from './config.js';
+import type { DriverSession, StepIntent } from '../driver.js';
+import type { RuntimeLogger } from '../logging/types.js';
+import { DiagnosticSanitizer } from '../redaction/diagnostic-sanitizer.js';
+import type { ToolRegistry } from '../tools/registry.js';
+import type { StepControl } from './scenario-control.js';
+import type { InternalStepReport } from './types.js';
+import { ScenarioUsageTracker } from './usage-tracker.js';
+export type StepExecutionDependencies = {
+    config: RuntimeConfig;
+    aiClient: AiClient;
+    toolRegistry: ToolRegistry;
+    driverSession: DriverSession;
+    usageTracker: ScenarioUsageTracker;
+    driverId: string;
+    redact: boolean;
+    diagnosticSanitizer: DiagnosticSanitizer;
+    logger: RuntimeLogger;
+};
+/** Authoritative driver-backed model/tool loop. */
 export declare class StepExecution {
-    private readonly aiClient;
-    private readonly extensionHost;
-    private stepResult;
-    private resolveStepResult;
-    private readonly stepResultPromise;
-    constructor(aiClient: AiClient, extensionHost: ExtensionHost);
-    run(step: Step): Promise<void>;
-    private logStepStart;
-    private logStepFinish;
-    private assertStepResult;
-    private assertionMessage;
+    private readonly dependencies;
+    constructor(dependencies: StepExecutionDependencies);
+    run(step: StepIntent, control: StepControl): Promise<InternalStepReport>;
 }
 //# sourceMappingURL=step-execution.d.ts.map

package/dist/runtime/step-execution.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"step-execution.d.ts","sourceRoot":"","sources":["../../src/runtime/step-execution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAoD,MAAM,YAAY,CAAA;AAGnF,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAE9C,qBAAa,aAAa;IAMxB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAN/B,OAAO,CAAC,UAAU,CAA4C;IAC9D,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;gBAGnC,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,aAAa;IAOxC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBpC,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,gBAAgB;CAGxB"}
+{"version":3,"file":"step-execution.d.ts","sourceRoot":"","sources":["../../src/runtime/step-execution.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAI/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAA;AAG1E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AAIxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,KAAK,EAAE,kBAAkB,EAA6B,MAAM,YAAY,CAAA;AAC/E,OAAO,EAGN,oBAAoB,EAEpB,MAAM,oBAAoB,CAAA;AAE3B,MAAM,MAAM,yBAAyB,GAAG;IACvC,MAAM,EAAE,aAAa,CAAA;IACrB,QAAQ,EAAE,QAAQ,CAAA;IAClB,YAAY,EAAE,YAAY,CAAA;IAC1B,aAAa,EAAE,aAAa,CAAA;IAC5B,YAAY,EAAE,oBAAoB,CAAA;IAClC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;IACf,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,MAAM,EAAE,aAAa,CAAA;CACrB,CAAA;AAED,mDAAmD;AACnD,qBAAa,aAAa;IACb,OAAO,CAAC,QAAQ,CAAC,YAAY;gBAAZ,YAAY,EAAE,yBAAyB;IAE9D,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAoI9E"}

package/dist/runtime/step-execution.js

@@ -1,49 +1,143 @@
-import { expect } from '@playwright/test';
-import { logger } from '../logging/index.js';
-import { STEP_START_USER_PROMPT, STEP_SYSTEM_PROMPT } from '../ai/prompts.js';
 import { MessageHistory } from '../ai/message-history.js';
+import { STEP_START_USER_PROMPT, STEP_SYSTEM_PROMPT } from '../ai/prompts.js';
+import { TurnProcessor } from '../ai/turn-processor.js';
+import { ToolDispatchError } from '../tools/dispatcher.js';
+import { LoopDetector } from '../tools/loop-detector.js';
+import { DriverBoundaryError } from './driver-boundary.js';
+import { buildInitialDriverContext, buildPostToolDriverContext, isEphemeralDriverMessage } from './driver-session.js';
+import { InternalStepEvidence } from './internal-step-evidence.js';
+import { ProviderUsageInvalidError, ProviderUsageUnavailableError, TokenBudgetExceededError, } from './usage-tracker.js';
+/** Authoritative driver-backed model/tool loop. */
 export class StepExecution {
-    aiClient;
-    extensionHost;
-    stepResult = { passed: false, actual: '' };
-    resolveStepResult;
-    stepResultPromise;
-    constructor(aiClient, extensionHost) {
-        this.aiClient = aiClient;
-        this.extensionHost = extensionHost;
-        this.stepResultPromise = new Promise((resolve) => {
-            this.resolveStepResult = resolve;
-        });
+    dependencies;
+    constructor(dependencies) {
+        this.dependencies = dependencies;
     }
-    async run(step) {
-        this.logStepStart(step);
+    async run(step, control) {
+        const { config, aiClient, toolRegistry, driverSession, usageTracker, driverId, redact, diagnosticSanitizer, logger, } = this.dependencies;
+        const evidence = new InternalStepEvidence(step, driverId, redact, diagnosticSanitizer, control.now);
+        const checkpoint = usageTracker.beginStep();
+        const messages = [];
+        const ephemeralMessages = new Set();
+        const turnProcessor = new TurnProcessor({
+            config,
+            toolRegistry,
+            loopDetector: new LoopDetector(config.loopMaxRepetitions),
+            evidence,
+            logger,
+        });
+        messages.push(...new MessageHistory().buildInitialMessages({
+            systemPrompt: STEP_SYSTEM_PROMPT([...driverSession.instructions]),
+            userPrompt: STEP_START_USER_PROMPT(step),
+        }));
+        logger.info(`step started: ${step.id}`);
+        let turns = 0;
         try {
-            await this.aiClient.initialize(step, this.resolveStepResult);
-            const initialMessages = new MessageHistory().buildInitialMessages({
-                systemPrompt: STEP_SYSTEM_PROMPT(this.extensionHost.getInstructions()),
-                userPrompt: STEP_START_USER_PROMPT(step),
-                contextMessages: await this.extensionHost.buildInitialMessages(step),
-            });
-            await this.aiClient.sendMessage(initialMessages);
-            this.stepResult = await this.stepResultPromise;
-            this.assertStepResult(step);
+            try {
+                appendDriverContext(messages, ephemeralMessages, await buildInitialDriverContext(driverSession, step, control));
+            }
+            catch (error) {
+                return finish(evidence, usageTracker, checkpoint, failureFrom(error, control, turns, diagnosticSanitizer), logger);
+            }
+            for (;;) {
+                if (turns >= config.turnCap) {
+                    return finish(evidence, usageTracker, checkpoint, { outcome: 'failed', reason: 'turn-cap-exceeded', turns }, logger);
+                }
+                const expired = control.poll();
+                if (expired.expired) {
+                    return finish(evidence, usageTracker, checkpoint, { outcome: 'failed', reason: expired.reason, turns }, logger);
+                }
+                turns++;
+                try {
+                    const { response, assistantMessages } = await aiClient.send(messages, {
+                        step,
+                        signal: control.signal,
+                    });
+                    usageTracker.record(response.usage);
+                    assertControlLive(control, 'provider-response');
+                    messages.push(...assistantMessages);
+                    const outcome = await turnProcessor.process({ response, step, turn: turns, control });
+                    assertControlLive(control, 'turn-processing');
+                    if (outcome.kind === 'assertion') {
+                        return finish(evidence, usageTracker, checkpoint, {
+                            outcome: outcome.passed ? 'passed' : 'failed',
+                            reason: outcome.passed ? 'met-expectation' : 'failed-expectation',
+                            actual: outcome.actual,
+                            turns,
+                        }, logger);
+                    }
+                    if (outcome.kind === 'stuck') {
+                        return finish(evidence, usageTracker, checkpoint, {
+                            outcome: 'failed',
+                            reason: 'loop-detected',
+                            actual: 'the model repeated the same tool calls without reaching a result',
+                            turns,
+                        }, logger);
+                    }
+                    dropEphemeralMessages(messages, ephemeralMessages);
+                    messages.push(...outcome.messages);
+                    appendDriverContext(messages, ephemeralMessages, await buildPostToolDriverContext(driverSession, step, turns, outcome.toolResults, control));
+                }
+                catch (error) {
+                    return finish(evidence, usageTracker, checkpoint, failureFrom(error, control, turns, diagnosticSanitizer), logger);
+                }
+            }
         }
-        catch (error) {
-            throw new Error(`\nFailed to execute action:\n${step.action}`, { cause: error });
+        finally {
+            control.dispose();
         }
-        this.logStepFinish();
     }
-    logStepStart(step) {
-        logger.info(`step started:\n${JSON.stringify(step, null, 2).replaceAll('  ', '').trim()}`);
+}
+function finish(evidence, usageTracker, checkpoint, termination, logger) {
+    if (termination.reason === 'step-timeout' || termination.reason === 'scenario-timeout') {
+        evidence.recordDiagnostic({
+            code: 'expired-boundary',
+            path: '',
+            message: termination.actual ?? `Step execution expired with ${termination.reason}`,
+        });
     }
-    logStepFinish() {
-        logger.info('step finished');
+    const report = evidence.buildReport(termination, usageTracker.stepUsage(checkpoint));
+    logger.info(`step finished: ${report.step.id} ${report.outcome} (${report.category} / ${report.reason})`);
+    return report;
+}
+function appendDriverContext(messages, ephemeralMessages, context) {
+    for (const message of context) {
+        messages.push(message);
+        if (isEphemeralDriverMessage(message))
+            ephemeralMessages.add(message);
     }
-    assertStepResult(step) {
-        expect(this.stepResult.passed, this.assertionMessage(step)).toBeTruthy();
+}
+function dropEphemeralMessages(messages, ephemeralMessages) {
+    if (ephemeralMessages.size === 0)
+        return;
+    const retained = messages.filter((message) => !ephemeralMessages.has(message));
+    messages.length = 0;
+    messages.push(...retained);
+    ephemeralMessages.clear();
+}
+function assertControlLive(control, operation) {
+    const expired = control.poll();
+    if (expired.expired)
+        throw new DriverBoundaryError(operation, expired.reason);
+}
+function failureFrom(error, control, turns, sanitizer) {
+    const actual = sanitizer.error(error);
+    const expired = control.poll();
+    if (expired.expired)
+        return { outcome: 'failed', reason: expired.reason, actual, turns };
+    if (error instanceof DriverBoundaryError)
+        return { outcome: 'failed', reason: error.reason, actual, turns };
+    if (error instanceof TokenBudgetExceededError) {
+        return { outcome: 'failed', reason: 'token-budget-exceeded', actual, turns };
     }
-    assertionMessage(step) {
-        return this.stepResult.passed ? step.expect : `\nExpected: ${step.expect}\nActual: ${this.stepResult.actual}`;
+    if (error instanceof ProviderUsageUnavailableError || error instanceof ProviderUsageInvalidError) {
+        return { outcome: 'failed', reason: 'provider-error', actual, turns };
     }
+    if (error instanceof ToolDispatchError)
+        return { outcome: 'failed', reason: 'tool-error', actual, turns };
+    return { outcome: 'failed', reason: driverInfraReason(error), actual, turns };
+}
+function driverInfraReason(error) {
+    return error instanceof ToolDispatchError ? 'tool-error' : 'provider-error';
 }
 //# sourceMappingURL=step-execution.js.map

package/dist/runtime/step-execution.js.map

@@ -1 +1 @@
-{"version":3,"file":"step-execution.js","sourceRoot":"","sources":["../../src/runtime/step-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAIzD,MAAM,OAAO,aAAa;IAMP;IACA;IANV,UAAU,GAAe,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAA;IACtD,iBAAiB,CAAoB;IAC5B,iBAAiB,CAAmB;IAErD,YACkB,QAAkB,EAClB,aAA4B;QAD5B,aAAQ,GAAR,QAAQ,CAAU;QAClB,kBAAa,GAAb,aAAa,CAAe;QAE7C,IAAI,CAAC,iBAAiB,GAAG,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YAC5D,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAA;QACjC,CAAC,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;QAEvB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAA;YAC5D,MAAM,eAAe,GAAG,IAAI,cAAc,EAAE,CAAC,oBAAoB,CAAC;gBACjE,YAAY,EAAE,kBAAkB,CAAC,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,CAAC;gBACtE,UAAU,EAAE,sBAAsB,CAAC,IAAI,CAAC;gBACxC,eAAe,EAAE,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,IAAI,CAAC;aACpE,CAAC,CAAA;YACF,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,eAAe,CAAC,CAAA;YAChD,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAA;YAC9C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAA;QACjF,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAA;IACrB,CAAC;IAEO,YAAY,CAAC,IAAU;QAC9B,MAAM,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAC3F,CAAC;IAEO,aAAa;QACpB,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IAC7B,CAAC;IAEO,gBAAgB,CAAC,IAAU;QAClC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAA;IACzE,CAAC;IAEO,gBAAgB,CAAC,IAAU;QAClC,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAA;IAC9G,CAAC;CACD"}
+{"version":3,"file":"step-execution.js","sourceRoot":"","sources":["../../src/runtime/step-execution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC7E,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAA;AAKvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AAExD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAA;AACrH,OAAO,EAAE,oBAAoB,EAAgC,MAAM,6BAA6B,CAAA;AAGhG,OAAO,EACN,yBAAyB,EACzB,6BAA6B,EAE7B,wBAAwB,GACxB,MAAM,oBAAoB,CAAA;AAc3B,mDAAmD;AACnD,MAAM,OAAO,aAAa;IACI;IAA7B,YAA6B,YAAuC;QAAvC,iBAAY,GAAZ,YAAY,CAA2B;IAAG,CAAC;IAExE,KAAK,CAAC,GAAG,CAAC,IAAgB,EAAE,OAAoB;QAC/C,MAAM,EACL,MAAM,EACN,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,QAAQ,EACR,MAAM,EACN,mBAAmB,EACnB,MAAM,GACN,GAAG,IAAI,CAAC,YAAY,CAAA;QACrB,MAAM,QAAQ,GAAG,IAAI,oBAAoB,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,OAAO,CAAC,GAAG,CAAC,CAAA;QACnG,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QAC3C,MAAM,QAAQ,GAAiC,EAAE,CAAA;QACjD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAA;QAC/D,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC;YACvC,MAAM;YACN,YAAY;YACZ,YAAY,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC,kBAAkB,CAAC;YACzD,QAAQ;YACR,MAAM;SACN,CAAC,CAAA;QACF,QAAQ,CAAC,IAAI,CACZ,GAAG,IAAI,cAAc,EAAE,CAAC,oBAAoB,CAAC;YAC5C,YAAY,EAAE,kBAAkB,CAAC,CAAC,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;YACjE,UAAU,EAAE,sBAAsB,CAAC,IAAI,CAAC;SACxC,CAAC,CACF,CAAA;QACD,MAAM,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAA;QAEvC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,CAAC;YACJ,IAAI,CAAC;gBACJ,mBAAmB,CAClB,QAAQ,EACR,iBAAiB,EACjB,MAAM,yBAAyB,CAAC,aAAa,EAAE,IAAI,EAAE,OAAO,CAAC,CAC7D,CAAA;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,MAAM,CACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,mBAAmB,CAAC,EACvD,MAAM,CACN,CAAA;YACF,CAAC;YAED,SAAS,CAAC;gBACT,IAAI,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,MAAM,CACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,EACzD,MAAM,CACN,CAAA;gBACF,CAAC;gBACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;gBAC9B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,OAAO,MAAM,CACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EACpD,MAAM,CACN,CAAA;gBACF,CAAC;gBAED,KAAK,EAAE,CAAA;gBACP,IAAI,CAAC;oBACJ,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACrE,IAAI;wBACJ,MAAM,EAAE,OAAO,CAAC,MAAM;qBACtB,CAAC,CAAA;oBACF,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;oBACnC,iBAAiB,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAA;oBAC/C,QAAQ,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,CAAA;oBACnC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;oBACrF,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAA;oBAE7C,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;wBAClC,OAAO,MAAM,CACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV;4BACC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;4BAC7C,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,oBAAoB;4BACjE,MAAM,EAAE,OAAO,CAAC,MAAM;4BACtB,KAAK;yBACL,EACD,MAAM,CACN,CAAA;oBACF,CAAC;oBACD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBAC9B,OAAO,MAAM,CACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV;4BACC,OAAO,EAAE,QAAQ;4BACjB,MAAM,EAAE,eAAe;4BACvB,MAAM,EAAE,kEAAkE;4BAC1E,KAAK;yBACL,EACD,MAAM,CACN,CAAA;oBACF,CAAC;oBAED,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAA;oBAClD,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;oBAClC,mBAAmB,CAClB,QAAQ,EACR,iBAAiB,EACjB,MAAM,0BAA0B,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAC1F,CAAA;gBACF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,OAAO,MAAM,CACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,mBAAmB,CAAC,EACvD,MAAM,CACN,CAAA;gBACF,CAAC;YACF,CAAC;QACF,CAAC;gBAAS,CAAC;YACV,OAAO,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;IACF,CAAC;CACD;AAED,SAAS,MAAM,CACd,QAA8B,EAC9B,YAAkC,EAClC,UAAyD,EACzD,WAAoC,EACpC,MAAqB;IAErB,IAAI,WAAW,CAAC,MAAM,KAAK,cAAc,IAAI,WAAW,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;QACxF,QAAQ,CAAC,gBAAgB,CAAC;YACzB,IAAI,EAAE,kBAAkB;YACxB,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,WAAW,CAAC,MAAM,IAAI,+BAA+B,WAAW,CAAC,MAAM,EAAE;SAClF,CAAC,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;IACpF,MAAM,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;IACzG,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,mBAAmB,CAC3B,QAAsC,EACtC,iBAAkD,EAClD,OAAqC;IAErC,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACtB,IAAI,wBAAwB,CAAC,OAAO,CAAC;YAAE,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IACtE,CAAC;AACF,CAAC;AAED,SAAS,qBAAqB,CAC7B,QAAsC,EACtC,iBAAkD;IAElD,IAAI,iBAAiB,CAAC,IAAI,KAAK,CAAC;QAAE,OAAM;IACxC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;IAC9E,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;IACnB,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;IAC1B,iBAAiB,CAAC,KAAK,EAAE,CAAA;AAC1B,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAoB,EAAE,SAAiB;IACjE,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;IAC9B,IAAI,OAAO,CAAC,OAAO;QAAE,MAAM,IAAI,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;AAC9E,CAAC;AAED,SAAS,WAAW,CACnB,KAAc,EACd,OAAoB,EACpB,KAAa,EACb,SAA8B;IAE9B,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACrC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;IAC9B,IAAI,OAAO,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IACxF,IAAI,KAAK,YAAY,mBAAmB;QAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IAC3G,IAAI,KAAK,YAAY,wBAAwB,EAAE,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,uBAAuB,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IAC7E,CAAC;IACD,IAAI,KAAK,YAAY,6BAA6B,IAAI,KAAK,YAAY,yBAAyB,EAAE,CAAC;QAClG,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IACtE,CAAC;IACD,IAAI,KAAK,YAAY,iBAAiB;QAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IACzG,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,iBAAiB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC9E,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACxC,OAAO,KAAK,YAAY,iBAAiB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAA;AAC5E,CAAC"}

package/dist/runtime/text.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Removes the common leading indentation from a multi-line string.
+ *
+ * Steps are written as template literals in a spec file, so `action` and `expect` arrive
+ * carrying whatever indentation the surrounding code happened to have. That indentation is
+ * an artifact of the source layout, not of the step, and it should not reach the report or
+ * the failure message.
+ */
+export declare function dedent(value: string): string;
+//# sourceMappingURL=text.d.ts.map

package/dist/runtime/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../src/runtime/text.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAoB5C"}

package/dist/runtime/text.js

@@ -0,0 +1,25 @@
+/**
+ * Removes the common leading indentation from a multi-line string.
+ *
+ * Steps are written as template literals in a spec file, so `action` and `expect` arrive
+ * carrying whatever indentation the surrounding code happened to have. That indentation is
+ * an artifact of the source layout, not of the step, and it should not reach the report or
+ * the failure message.
+ */
+export function dedent(value) {
+    const lines = value.split('\n').map((line) => line.replace(/\s+$/, ''));
+    while (lines.length > 0 && lines[0].trim() === '') {
+        lines.shift();
+    }
+    while (lines.length > 0 && lines[lines.length - 1].trim() === '') {
+        lines.pop();
+    }
+    const indent = lines
+        .filter((line) => line.trim() !== '')
+        .reduce((smallest, line) => Math.min(smallest, line.length - line.trimStart().length), Number.MAX_SAFE_INTEGER);
+    if (indent === Number.MAX_SAFE_INTEGER) {
+        return '';
+    }
+    return lines.map((line) => line.slice(indent)).join('\n');
+}
+//# sourceMappingURL=text.js.map

package/dist/runtime/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/runtime/text.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,UAAU,MAAM,CAAC,KAAa;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAA;IAEvE,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,KAAK,CAAC,KAAK,EAAE,CAAA;IACd,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAClE,KAAK,CAAC,GAAG,EAAE,CAAA;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,KAAK;SAClB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAA;IAEhH,IAAI,MAAM,KAAK,MAAM,CAAC,gBAAgB,EAAE,CAAC;QACxC,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC"}

package/dist/runtime/types.d.ts

@@ -1,79 +1,53 @@
-/**
- * A single natural-language test step executed by Checkmate.
- *
- * `action` should describe what the agent needs to do.
- * `expect` should describe the expected result after the action finishes.
- * `search` can bias snapshot filtering toward specific keywords.
- * `topPercent` controls how much of the scored page snapshot is kept.
- *
- * @example
- * ```ts
- * const step: Step = {
- *   action: "Open the pricing page and click the Pro plan",
- *   expect: "The checkout page for the Pro plan is displayed",
- *   topPercent: 15,
- * }
- * ```
- */
-export interface Step {
-    /**
-     * What the agent should do in the browser.
-     */
-    action: string;
-    /**
-     * What should be true after the action is complete.
-     */
-    expect: string;
-    /**
-     * Optional keyword hints for snapshot filtering.
-     *
-     * When provided, these terms are prioritized over semantic `action + expect` matching.
-     */
-    search?: string[];
-    /**
-     * Optional percentage of the highest-scoring snapshot elements to keep.
-     *
-     * This value is expressed as a real percent from `1` to `100`.
-     * For example, `10` keeps the top 10% of scored elements.
-     *
-     * @example
-     * ```ts
-     * topPercent: 20
-     * ```
-     */
-    topPercent?: number;
-}
-/**
- * Final result returned by a step execution.
- *
- * @example
- * ```ts
- * const result: StepResult = {
- *   passed: true,
- *   actual: 'Checkout page is visible',
- * }
- * ```
- */
-export interface StepResult {
-    /**
-     * Whether the step passed.
-     */
+import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
+import type { Diagnostic, Usage } from '../contracts/types.js';
+import type { StepIntent } from '../driver.js';
+import type { ToolExecution } from '../tools/types.js';
+export type Step = Pick<StepIntent, 'action' | 'expect'>;
+export type StepAssertion = {
     passed: boolean;
-    /**
-     * The observed result collected during execution.
-     */
     actual: string;
-}
-export type StepResultPromise = Promise<StepResult>;
-/**
- * Callback used internally to resolve a running step result.
- *
- * @example
- * ```ts
- * const resolve: ResolveStepResult = (result) => {
- *   console.log(result.actual)
- * }
- * ```
- */
-export type ResolveStepResult = (result: StepResult) => void;
+};
+export type InternalTerminationReason = 'met-expectation' | 'failed-expectation' | 'loop-detected' | 'turn-cap-exceeded' | 'step-timeout' | 'scenario-timeout' | 'tool-error' | 'provider-error' | 'token-budget-exceeded' | 'interrupted' | 'internal-error';
+export type InternalStepToolCall = {
+    turn: number;
+    driverId: string;
+    name: string;
+    arguments: unknown;
+    status: 'ok' | 'error';
+};
+export type InternalStepReport = {
+    step: StepIntent;
+    outcome: 'passed' | 'failed';
+    category: StepCategory;
+    reason: InternalTerminationReason;
+    actual?: string;
+    turns: number;
+    durationMs: number;
+    usage: Usage;
+    toolCalls: InternalStepToolCall[];
+    transcript: TranscriptEntry[];
+    diagnostics: Diagnostic[];
+};
+export type StepCategory = 'app' | 'model' | 'infra';
+export type TranscriptEntry = {
+    turn: number;
+    role: 'assistant' | 'tool';
+    content: string;
+};
+export type ContextMessage = {
+    message: ChatCompletionMessageParam;
+    ephemeral?: boolean;
+};
+export type TurnOutcome = {
+    kind: 'continue';
+    toolResults: ToolExecution[];
+    messages: ChatCompletionMessageParam[];
+} | {
+    kind: 'assertion';
+    passed: boolean;
+    actual: string;
+} | {
+    kind: 'stuck';
+    reason: 'loop-detected';
+};
 //# sourceMappingURL=types.d.ts.map

package/dist/runtime/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,IAAI;IACpB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IAEjB;;;;;;;;;;OAUG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,UAAU;IAC1B;;OAEG;IACH,MAAM,EAAE,OAAO,CAAA;IAEf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;CACd;AAED,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;AAEnD;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runtime/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,mCAAmC,CAAA;AACnF,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAEtD,MAAM,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,CAAC,CAAA;AAExD,MAAM,MAAM,aAAa,GAAG;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAA;AAE/D,MAAM,MAAM,yBAAyB,GAClC,iBAAiB,GACjB,oBAAoB,GACpB,eAAe,GACf,mBAAmB,GACnB,cAAc,GACd,kBAAkB,GAClB,YAAY,GACZ,gBAAgB,GAChB,uBAAuB,GACvB,aAAa,GACb,gBAAgB,CAAA;AAEnB,MAAM,MAAM,oBAAoB,GAAG;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,OAAO,CAAA;IAClB,MAAM,EAAE,IAAI,GAAG,OAAO,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG;IAChC,IAAI,EAAE,UAAU,CAAA;IAChB,OAAO,EAAE,QAAQ,GAAG,QAAQ,CAAA;IAC5B,QAAQ,EAAE,YAAY,CAAA;IACtB,MAAM,EAAE,yBAAyB,CAAA;IACjC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,KAAK,CAAA;IACZ,SAAS,EAAE,oBAAoB,EAAE,CAAA;IACjC,UAAU,EAAE,eAAe,EAAE,CAAA;IAC7B,WAAW,EAAE,UAAU,EAAE,CAAA;CACzB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAA;AAEpD,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAA;AAE3F,MAAM,MAAM,cAAc,GAAG;IAAE,OAAO,EAAE,0BAA0B,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,CAAA;AAEzF,MAAM,MAAM,WAAW,GACpB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,WAAW,EAAE,aAAa,EAAE,CAAC;IAAC,QAAQ,EAAE,0BAA0B,EAAE,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE,CAAA"}

package/dist/runtime/usage-tracker.d.ts

@@ -0,0 +1,37 @@
+import type { ChatCompletion } from 'openai/resources/chat/completions';
+import type { Usage } from '../contracts/types.js';
+export type UsageCheckpoint = Readonly<{
+    promptTokens: number;
+    cachedPromptTokens: number;
+    completionTokens: number;
+    reportedResponses: number;
+    missingResponses: number;
+}>;
+export declare class TokenBudgetExceededError extends Error {
+    readonly budgetTokens: number;
+    readonly totalTokens: number;
+    constructor(budgetTokens: number, totalTokens: number);
+}
+export declare class ProviderUsageUnavailableError extends Error {
+    constructor();
+}
+export declare class ProviderUsageInvalidError extends Error {
+    constructor(message: string);
+}
+export declare class ScenarioUsageTracker {
+    private readonly budgetTokens?;
+    private promptTokens;
+    private cachedPromptTokens;
+    private completionTokens;
+    private reportedResponses;
+    private missingResponses;
+    constructor(budgetTokens?: number);
+    beginStep(): UsageCheckpoint;
+    record(usage: ChatCompletion['usage']): void;
+    stepUsage(checkpoint: UsageCheckpoint): Usage;
+    usage(): Usage & {
+        state: 'complete' | 'partial' | 'unavailable';
+    };
+    private checkpoint;
+}
+//# sourceMappingURL=usage-tracker.d.ts.map

package/dist/runtime/usage-tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage-tracker.d.ts","sourceRoot":"","sources":["../../src/runtime/usage-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAA;AACvE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAElD,MAAM,MAAM,eAAe,GAAG,QAAQ,CAAC;IACtC,YAAY,EAAE,MAAM,CAAA;IACpB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,gBAAgB,EAAE,MAAM,CAAA;IACxB,iBAAiB,EAAE,MAAM,CAAA;IACzB,gBAAgB,EAAE,MAAM,CAAA;CACxB,CAAC,CAAA;AAEF,qBAAa,wBAAyB,SAAQ,KAAK;IAEjD,QAAQ,CAAC,YAAY,EAAE,MAAM;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM;gBADnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM;CAK7B;AAED,qBAAa,6BAA8B,SAAQ,KAAK;;CAKvD;AAED,qBAAa,yBAA0B,SAAQ,KAAK;gBACvC,OAAO,EAAE,MAAM;CAI3B;AAED,qBAAa,oBAAoB;IAOpB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;IAN1C,OAAO,CAAC,YAAY,CAAI;IACxB,OAAO,CAAC,kBAAkB,CAAI;IAC9B,OAAO,CAAC,gBAAgB,CAAI;IAC5B,OAAO,CAAC,iBAAiB,CAAI;IAC7B,OAAO,CAAC,gBAAgB,CAAI;gBAEC,YAAY,CAAC,EAAE,MAAM;IAMlD,SAAS,IAAI,eAAe;IAI5B,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,GAAG,IAAI;IAgC5C,SAAS,CAAC,UAAU,EAAE,eAAe,GAAG,KAAK;IAQ7C,KAAK,IAAI,KAAK,GAAG;QAAE,KAAK,EAAE,UAAU,GAAG,SAAS,GAAG,aAAa,CAAA;KAAE;IAYlE,OAAO,CAAC,UAAU;CASlB"}

package/dist/runtime/usage-tracker.js

@@ -0,0 +1,110 @@
+export class TokenBudgetExceededError extends Error {
+    budgetTokens;
+    totalTokens;
+    constructor(budgetTokens, totalTokens) {
+        super(`Scenario token budget of ${budgetTokens} was exceeded by response total ${totalTokens}`);
+        this.budgetTokens = budgetTokens;
+        this.totalTokens = totalTokens;
+        this.name = 'TokenBudgetExceededError';
+    }
+}
+export class ProviderUsageUnavailableError extends Error {
+    constructor() {
+        super('Provider response omitted usage while a scenario token budget is active');
+        this.name = 'ProviderUsageUnavailableError';
+    }
+}
+export class ProviderUsageInvalidError extends Error {
+    constructor(message) {
+        super(`Provider response contained invalid usage: ${message}`);
+        this.name = 'ProviderUsageInvalidError';
+    }
+}
+export class ScenarioUsageTracker {
+    budgetTokens;
+    promptTokens = 0;
+    cachedPromptTokens = 0;
+    completionTokens = 0;
+    reportedResponses = 0;
+    missingResponses = 0;
+    constructor(budgetTokens) {
+        this.budgetTokens = budgetTokens;
+        if (budgetTokens !== undefined && (!Number.isSafeInteger(budgetTokens) || budgetTokens < 1)) {
+            throw new Error('Scenario token budget must be a positive integer');
+        }
+    }
+    beginStep() {
+        return this.checkpoint();
+    }
+    record(usage) {
+        if (!usage) {
+            this.missingResponses++;
+            if (this.budgetTokens !== undefined)
+                throw new ProviderUsageUnavailableError();
+            return;
+        }
+        const promptTokens = requiredTokenCount(usage.prompt_tokens, 'prompt_tokens');
+        const completionTokens = requiredTokenCount(usage.completion_tokens, 'completion_tokens');
+        const totalTokens = requiredTokenCount(usage.total_tokens, 'total_tokens');
+        if (totalTokens !== promptTokens + completionTokens) {
+            throw new ProviderUsageInvalidError('total_tokens must equal prompt_tokens plus completion_tokens');
+        }
+        const cachedPromptTokens = optionalTokenCount(usage.prompt_tokens_details?.cached_tokens, 'prompt_tokens_details.cached_tokens');
+        if (cachedPromptTokens > promptTokens) {
+            throw new ProviderUsageInvalidError('cached_tokens cannot exceed prompt_tokens');
+        }
+        this.promptTokens += promptTokens;
+        this.cachedPromptTokens += cachedPromptTokens;
+        this.completionTokens += completionTokens;
+        this.reportedResponses++;
+        const scenarioTotalTokens = this.promptTokens + this.completionTokens;
+        if (this.budgetTokens !== undefined && scenarioTotalTokens > this.budgetTokens) {
+            throw new TokenBudgetExceededError(this.budgetTokens, scenarioTotalTokens);
+        }
+    }
+    stepUsage(checkpoint) {
+        return usageFrom({
+            promptTokens: this.promptTokens - checkpoint.promptTokens,
+            cachedPromptTokens: this.cachedPromptTokens - checkpoint.cachedPromptTokens,
+            completionTokens: this.completionTokens - checkpoint.completionTokens,
+        });
+    }
+    usage() {
+        const state = this.reportedResponses === 0 ? 'unavailable' : this.missingResponses > 0 ? 'partial' : 'complete';
+        return {
+            ...usageFrom({
+                promptTokens: this.promptTokens,
+                cachedPromptTokens: this.cachedPromptTokens,
+                completionTokens: this.completionTokens,
+            }),
+            state,
+        };
+    }
+    checkpoint() {
+        return {
+            promptTokens: this.promptTokens,
+            cachedPromptTokens: this.cachedPromptTokens,
+            completionTokens: this.completionTokens,
+            reportedResponses: this.reportedResponses,
+            missingResponses: this.missingResponses,
+        };
+    }
+}
+function requiredTokenCount(value, field) {
+    if (!Number.isSafeInteger(value) || value < 0) {
+        throw new ProviderUsageInvalidError(`${field} must be a non-negative safe integer`);
+    }
+    return value;
+}
+function optionalTokenCount(value, field) {
+    return value === undefined ? 0 : requiredTokenCount(value, field);
+}
+function usageFrom(input) {
+    return {
+        promptTokens: input.promptTokens,
+        cachedPromptTokens: Math.min(input.cachedPromptTokens, input.promptTokens),
+        completionTokens: input.completionTokens,
+        totalTokens: input.promptTokens + input.completionTokens,
+    };
+}
+//# sourceMappingURL=usage-tracker.js.map

package/dist/runtime/usage-tracker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage-tracker.js","sourceRoot":"","sources":["../../src/runtime/usage-tracker.ts"],"names":[],"mappings":"AAWA,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IAExC;IACA;IAFV,YACU,YAAoB,EACpB,WAAmB;QAE5B,KAAK,CAAC,4BAA4B,YAAY,mCAAmC,WAAW,EAAE,CAAC,CAAA;QAHtF,iBAAY,GAAZ,YAAY,CAAQ;QACpB,gBAAW,GAAX,WAAW,CAAQ;QAG5B,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAA;IACvC,CAAC;CACD;AAED,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACvD;QACC,KAAK,CAAC,yEAAyE,CAAC,CAAA;QAChF,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAA;IAC5C,CAAC;CACD;AAED,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IACnD,YAAY,OAAe;QAC1B,KAAK,CAAC,8CAA8C,OAAO,EAAE,CAAC,CAAA;QAC9D,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAA;IACxC,CAAC;CACD;AAED,MAAM,OAAO,oBAAoB;IAOH;IANrB,YAAY,GAAG,CAAC,CAAA;IAChB,kBAAkB,GAAG,CAAC,CAAA;IACtB,gBAAgB,GAAG,CAAC,CAAA;IACpB,iBAAiB,GAAG,CAAC,CAAA;IACrB,gBAAgB,GAAG,CAAC,CAAA;IAE5B,YAA6B,YAAqB;QAArB,iBAAY,GAAZ,YAAY,CAAS;QACjD,IAAI,YAAY,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,SAAS;QACR,OAAO,IAAI,CAAC,UAAU,EAAE,CAAA;IACzB,CAAC;IAED,MAAM,CAAC,KAA8B;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,IAAI,CAAC,gBAAgB,EAAE,CAAA;YACvB,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS;gBAAE,MAAM,IAAI,6BAA6B,EAAE,CAAA;YAC9E,OAAM;QACP,CAAC;QAED,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,aAAa,EAAE,eAAe,CAAC,CAAA;QAC7E,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,KAAK,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAA;QACzF,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC,CAAA;QAC1E,IAAI,WAAW,KAAK,YAAY,GAAG,gBAAgB,EAAE,CAAC;YACrD,MAAM,IAAI,yBAAyB,CAAC,8DAA8D,CAAC,CAAA;QACpG,CAAC;QAED,MAAM,kBAAkB,GAAG,kBAAkB,CAC5C,KAAK,CAAC,qBAAqB,EAAE,aAAa,EAC1C,qCAAqC,CACrC,CAAA;QACD,IAAI,kBAAkB,GAAG,YAAY,EAAE,CAAC;YACvC,MAAM,IAAI,yBAAyB,CAAC,2CAA2C,CAAC,CAAA;QACjF,CAAC;QACD,IAAI,CAAC,YAAY,IAAI,YAAY,CAAA;QACjC,IAAI,CAAC,kBAAkB,IAAI,kBAAkB,CAAA;QAC7C,IAAI,CAAC,gBAAgB,IAAI,gBAAgB,CAAA;QACzC,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAExB,MAAM,mBAAmB,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAA;QACrE,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,IAAI,mBAAmB,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAChF,MAAM,IAAI,wBAAwB,CAAC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,SAAS,CAAC,UAA2B;QACpC,OAAO,SAAS,CAAC;YAChB,YAAY,EAAE,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY;YACzD,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC,kBAAkB;YAC3E,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB;SACrE,CAAC,CAAA;IACH,CAAC;IAED,KAAK;QACJ,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;QAC/G,OAAO;YACN,GAAG,SAAS,CAAC;gBACZ,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACvC,CAAC;YACF,KAAK;SACL,CAAA;IACF,CAAC;IAEO,UAAU;QACjB,OAAO;YACN,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAA;IACF,CAAC;CACD;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,KAAa;IACxD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,yBAAyB,CAAC,GAAG,KAAK,sCAAsC,CAAC,CAAA;IACpF,CAAC;IACD,OAAO,KAAe,CAAA;AACvB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,KAAa;IACxD,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;AAClE,CAAC;AAED,SAAS,SAAS,CAAC,KAAqF;IACvG,OAAO;QACN,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC,YAAY,CAAC;QAC1E,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,WAAW,EAAE,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,gBAAgB;KACxD,CAAA;AACF,CAAC"}