@qvac/harness 0.7.1 → 1.0.0-rc.1

package/dist-lib/lib/machine/engine.js

@@ -0,0 +1,483 @@
+import { waitFor } from 'xstate';
+import { createCompletionMachine } from './completion/machine.js';
+import { createInvokeTracker } from './completion/ports.js';
+import { isRecord } from '../run-log/patch.js';
+import { replayFinalRound, replaySuccessOutput } from './replay-output.js';
+import { AbortedError, didDrainTimeOut, raceDrain, rehydrateError, tagError } from './errors.js';
+import { createPersistedActor } from '../run-log/persisted-actor.js';
+import { stableStringify } from './stable-stringify.js';
+import { applyEngineStep, captureRunRecord, captureEngineStep, createEngineStep, prepareEngineBoot, transitionName } from '../run-log/protocol.js';
+export { stableStringify } from './stable-stringify.js';
+import { MACHINE_VERSION, XSTATE_VERSION, RUN_LOG_VERSION, ENGINE_BOUNDARY_BOOT, RunJournal, acquireRun, copy } from '../run-log/run-journal.js';
+function nameOf(named) {
+    return transitionName(named);
+}
+function stripTransition(transition) {
+    const out = { eventType: transition.eventType };
+    if (transition.target?.length) {
+        out.target = transition.target.map((node) => node.id);
+    }
+    if (transition.guard) {
+        out.guard = nameOf(transition.guard);
+    }
+    if (transition.actions.length) {
+        out.actions = transition.actions.map(nameOf);
+    }
+    return out;
+}
+let cachedDefinition;
+function machineDefinition() {
+    cachedDefinition ??= stripNode(createCompletionMachine().root);
+    return cachedDefinition;
+}
+export function completionMachineDefinition() {
+    return copy(machineDefinition());
+}
+function stripNode(node) {
+    const out = {
+        id: node.id,
+        type: node.type,
+        tags: [...node.tags],
+        entry: node.entry.map(nameOf),
+        exit: node.exit.map(nameOf),
+        invoke: node.invoke.map((invoke) => ({
+            id: invoke.id,
+            src: typeof invoke.src === 'string' ? invoke.src : nameOf(invoke.src)
+        }))
+    };
+    const initial = node.initial?.target?.[0]?.id;
+    if (initial) {
+        const prefix = `${node.id}.`;
+        out.initial = initial.startsWith(prefix) ? initial.slice(prefix.length) : initial;
+    }
+    const children = Object.entries(node.states);
+    if (children.length) {
+        out.states = Object.fromEntries(children.map(([key, child]) => [key, stripNode(child)]));
+    }
+    if (node.transitions.size) {
+        out.on = Object.fromEntries([...node.transitions].map(([event, list]) => [event, list.map(stripTransition)]));
+    }
+    if (node.always?.length) {
+        out.always = node.always.map(stripTransition);
+    }
+    return out;
+}
+export const HOST_APPEND_TIMEOUT_MS = 5000;
+function readRestoredSnapshot(value) {
+    if (!isRecord(value) || Array.isArray(value))
+        return undefined;
+    const context = isRecord(value.context) && !Array.isArray(value.context) ? value.context : undefined;
+    return { status: value.status, output: value.output, context };
+}
+function isActiveIdle(snapshot) {
+    const value = snapshot.value;
+    return isRecord(value) && value.active === 'idle';
+}
+function isPausedOrAwaitingRecovery(snapshot) {
+    return snapshot.hasTag('paused');
+}
+function wasInActiveState(snapshot, name) {
+    if (isRecord(snapshot.value) && snapshot.value.active === name) {
+        return true;
+    }
+    const historyValue = Reflect.get(snapshot, 'historyValue');
+    if (!isRecord(historyValue)) {
+        return false;
+    }
+    return Object.values(historyValue).some((nodes) => Array.isArray(nodes) &&
+        nodes.some((node) => isRecord(node) && node.id === `completion.active.${name}`));
+}
+export async function* runEngine(run, options = {}) {
+    const { runId } = run;
+    const log = boundaryLog(options, runId);
+    const journal = new RunJournal(log, {
+        boot: log[ENGINE_BOUNDARY_BOOT](),
+        redact: options.redact,
+        redactShared: options.redactShared,
+        redactOperationResult: options.redactOperationResult,
+        redactRecord: options.redactRecord
+    });
+    const core = journal.core;
+    const release = acquireRun(run.lockOwner ?? options.lockOwner ?? core, runId);
+    try {
+        if (!journal.restored) {
+            await journal.restore();
+        }
+        else if (journal.knownLength !== core.length) {
+            throw new Error(`Run ${runId} changed while opening`);
+        }
+        if (run.resume) {
+            if (!journal.metadata) {
+                throw new Error(`Run ${runId} does not exist`);
+            }
+            const requestMatches = run.request === undefined ||
+                stableStringify(journal.metadata.request) === stableStringify(run.request);
+            if (journal.metadata.runId !== runId ||
+                (run.requestDigest !== undefined
+                    ? journal.metadata.requestDigest !== run.requestDigest
+                    : !requestMatches)) {
+                throw new Error('Resume cannot replace the original model, conversation, or execution configuration');
+            }
+        }
+        else {
+            if (run.request === undefined) {
+                throw new Error('A new run requires a request identity');
+            }
+            if (core.length) {
+                throw new Error(`Run ${runId} already exists; use resume mode`);
+            }
+            if (run.recovery?.length) {
+                throw new Error('Recovery resolutions require resume mode');
+            }
+            const metadata = {
+                version: RUN_LOG_VERSION,
+                type: 'metadata',
+                machine: MACHINE_VERSION,
+                definition: machineDefinition(),
+                xstate: XSTATE_VERSION,
+                runId,
+                request: run.request,
+                requestDigest: run.requestDigest,
+                digestRunId: run.requestDigest ? runId : undefined,
+                budget: run.budget,
+                startedAt: run.now ? run.now() : undefined
+            };
+            await journal.append(metadata);
+        }
+        const restored = readRestoredSnapshot(journal.snapshot);
+        if (restored && restored.context?.runId !== runId) {
+            throw new Error('Run checkpoint identity does not match its metadata');
+        }
+        await journal.resolve(run.recovery ?? []);
+        if (restored?.status === 'done') {
+            yield { type: run.resume ? 'run-resumed' : 'run-started', runId };
+            const round = restored.context?.round;
+            if (run.resume && typeof round === 'number') {
+                yield { type: 'round-reset', runId, round, replacedTypes: ['content', 'thinking'] };
+            }
+            const output = restored.output;
+            if (isRecord(output) && output.status === 'failure') {
+                const failure = restored.context?.failure;
+                const fallback = typeof output.error === 'string' ? output.error : 'Completion failed';
+                throw rehydrateError(failure ? tagError(failure) : undefined, fallback);
+            }
+            yield* replaySuccessOutput({ runId, round, context: restored.context, output });
+            return { status: 'replayed' };
+        }
+        yield { type: run.resume ? 'run-resumed' : 'run-started', runId };
+        const invokes = createInvokeTracker();
+        const boot = immutableBoot(getCurrentBoot(log, journal));
+        const mailbox = createEventMailbox();
+        const { ports, context } = await run.start({
+            operations: createOperationPort(journal),
+            invokes,
+            boot
+        });
+        return yield* drive({
+            journal,
+            run,
+            machine: createCompletionMachine(context, ports.now, {
+                ports,
+                journal,
+                invokes,
+                emitContent: (event) => mailbox.emit(event)
+            }),
+            invokes,
+            now: ports.now,
+            durable: typeof options.onStep === 'function',
+            onDetached: options.onDetached,
+            mailbox
+        });
+    }
+    finally {
+        if (journal.detachedEffects) {
+            journal.sealed = true;
+        }
+        release();
+    }
+}
+async function* drive({ journal, run, machine, invokes, now, durable, onDetached, mailbox }) {
+    const { runId, signal } = run;
+    const pausedMs = run.resume && journal.snapshotSavedAt !== undefined
+        ? Math.max(0, now() - journal.snapshotSavedAt)
+        : 0;
+    const { queue } = mailbox;
+    let detached = false;
+    const detach = () => {
+        detached = true;
+        if (!journal.detachedEffects) {
+            journal.detachedEffects = invokes.waitForInvokes();
+            onDetached?.(journal.detachedEffects, runId);
+        }
+    };
+    let wake = () => { };
+    let wakeVersion = 0;
+    const notify = () => {
+        wakeVersion++;
+        wake();
+    };
+    mailbox.notify = notify;
+    const { actor, persistence } = await createPersistedActor(journal, machine, notify, now);
+    actor.on('*', (event) => {
+        queue.push(event);
+        notify();
+    });
+    actor.subscribe({ next: notify, error: notify });
+    const pause = () => {
+        if (actor.getSnapshot().status === 'active') {
+            actor.send({ type: 'PAUSE' });
+        }
+        notify();
+    };
+    signal?.addEventListener('abort', pause);
+    try {
+        await persistence.checkpoint();
+        const restored = actor.getSnapshot();
+        actor.start();
+        if (run.resume) {
+            if (!actor.getSnapshot().hasTag('paused')) {
+                pause();
+            }
+            const result = await drainResume({ actor, invokes, persistence, durable, detach });
+            if (result) {
+                return result;
+            }
+            yield* replayInterruptedGeneration(runId, restored);
+            const uncertain = journal.uncertainWrite();
+            if (uncertain) {
+                actor.send({ type: 'RECOVERY_REQUIRED', operation: uncertain });
+            }
+            else if (!signal?.aborted) {
+                actor.send({ type: 'RESUME', pausedMs });
+            }
+        }
+        if (signal?.aborted) {
+            pause();
+        }
+        else if (isActiveIdle(actor.getSnapshot())) {
+            actor.send({ type: 'START' });
+        }
+        while (true) {
+            if (journal.hasPendingFlush()) {
+                await persistence.flush();
+            }
+            yield* drainQueue(queue);
+            if (isActiveIdle(actor.getSnapshot()) && !signal?.aborted) {
+                actor.send({ type: 'START' });
+            }
+            const snapshot = actor.getSnapshot();
+            if (snapshot.status === 'error') {
+                return { status: 'error', snapshot, error: snapshot.error };
+            }
+            if (snapshot.status === 'done') {
+                break;
+            }
+            const isSettling = settling(snapshot);
+            const settlingResult = isSettling
+                ? await drainSettling({ actor, persistence, durable, detach })
+                : undefined;
+            if (settlingResult) {
+                yield* drainQueue(queue);
+                return settlingResult;
+            }
+            if (isSettling) {
+                continue;
+            }
+            if (isPausedOrAwaitingRecovery(snapshot)) {
+                return yield* finishPaused({
+                    actor,
+                    invokes,
+                    persistence,
+                    durable,
+                    detach,
+                    journal,
+                    queue,
+                    runId
+                });
+            }
+            const observedWakeVersion = wakeVersion;
+            if (queue.length) {
+                continue;
+            }
+            await new Promise((resolve) => {
+                wake = resolve;
+                if (wakeVersion !== observedWakeVersion) {
+                    resolve();
+                }
+            });
+        }
+        await persistence.flush();
+        yield* drainQueue(queue);
+        return { status: 'done', snapshot: actor.getSnapshot() };
+    }
+    finally {
+        signal?.removeEventListener('abort', pause);
+        try {
+            if (actor.getSnapshot().status === 'active') {
+                pause();
+            }
+            if (!detached && (await invokes.drainTimedOut())) {
+                detach();
+            }
+            if (await shouldDetachWhileClosing({ detached, journal, actor })) {
+                detach();
+            }
+            await persistence.checkpoint();
+            await persistence.flush();
+            run.settled?.(actor.getSnapshot());
+        }
+        finally {
+            await persistence.dispose();
+        }
+    }
+}
+async function drainResume({ actor, invokes, persistence, durable, detach }) {
+    if (await invokes.drainTimedOut()) {
+        return interruptedResult({ actor, persistence, durable, detach });
+    }
+    if (!settling(actor.getSnapshot())) {
+        return undefined;
+    }
+    const timedOut = didDrainTimeOut(await raceDrain(waitFor(actor, (state) => !settling(state))));
+    return timedOut ? interruptedResult({ actor, persistence, durable, detach }) : undefined;
+}
+async function drainSettling(options) {
+    const { actor, persistence, durable, detach } = options;
+    const timedOut = didDrainTimeOut(await raceDrain(waitFor(actor, (state) => !settling(state))));
+    return timedOut ? interruptedResult({ actor, persistence, durable, detach }) : undefined;
+}
+async function interruptedResult({ actor, persistence, durable, detach }) {
+    detach();
+    await persistence.flush();
+    const snapshot = actor.getSnapshot();
+    return durable
+        ? { status: 'paused', snapshot }
+        : { status: 'error', snapshot, error: tagError(new AbortedError()) };
+}
+async function detachIfDrained(invokes, detach) {
+    if (await invokes.drainTimedOut()) {
+        detach();
+    }
+}
+function pendingRecovery(snapshot, journal) {
+    const recovery = snapshot.context.recovery;
+    return recovery && journal.getReceipt(recovery.id)?.status !== 'result' ? recovery : undefined;
+}
+async function* finishPaused({ actor, invokes, persistence, durable, detach, journal, queue, runId }) {
+    await detachIfDrained(invokes, detach);
+    await persistence.flush();
+    const snapshot = actor.getSnapshot();
+    const recovery = pendingRecovery(snapshot, journal);
+    if (recovery) {
+        const event = { type: 'recovery-required', runId, operation: copy(recovery) };
+        yield event;
+    }
+    yield* drainQueue(queue);
+    return durable
+        ? { status: 'paused', snapshot }
+        : { status: 'error', snapshot, error: tagError(new AbortedError()) };
+}
+async function shouldDetachWhileClosing({ detached, journal, actor }) {
+    if (detached || journal.failure || !settling(actor.getSnapshot())) {
+        return false;
+    }
+    return didDrainTimeOut(await raceDrain(waitFor(actor, (snapshot) => !settling(snapshot))));
+}
+function* replayInterruptedGeneration(runId, snapshot) {
+    const { round, lastRound } = snapshot.context;
+    const finishingRound = lastRound &&
+        (wasInActiveState(snapshot, 'answering') ||
+            wasInActiveState(snapshot, 'finishing') ||
+            (wasInActiveState(snapshot, 'resolving') && !lastRound.toolCalls.length));
+    const shouldReplay = wasInActiveState(snapshot, 'generating') ||
+        wasInActiveState(snapshot, 'wrapping') ||
+        finishingRound;
+    if (!shouldReplay) {
+        return;
+    }
+    yield { type: 'round-reset', runId, round };
+    if (finishingRound && lastRound) {
+        yield* replayFinalRound({ runId, round, last: lastRound, response: snapshot.context.response });
+    }
+}
+function settling(snapshot) {
+    const value = snapshot.value;
+    return value === 'pausing';
+}
+function* drainQueue(queue) {
+    while (queue.length) {
+        const event = queue.shift();
+        if (event) {
+            yield event;
+        }
+    }
+}
+function createEventMailbox() {
+    const mailbox = {
+        queue: [],
+        notify() { },
+        emit(event) {
+            mailbox.queue.push(event);
+            mailbox.notify();
+        }
+    };
+    return mailbox;
+}
+function boundaryLog(options, runId) {
+    let boot = prepareEngineBoot(options.boot);
+    const log = {
+        [ENGINE_BOUNDARY_BOOT]: () => boot,
+        get length() {
+            return boot.length;
+        },
+        ready: async () => { },
+        get: async () => {
+            return null;
+        },
+        async append(value) {
+            const step = createEngineStep(boot.nextSeq, boot.length, [captureRunRecord(value)]);
+            const prepared = options.redactStep ? captureEngineStep(options.redactStep(step)) : step;
+            const next = applyEngineStep(boot, prepared);
+            const append = options.onStep?.(prepared);
+            if (append) {
+                const configured = options.hostAppendTimeoutMs;
+                const timeoutMs = typeof configured === 'number' && Number.isFinite(configured) && configured > 0
+                    ? Math.min(configured, 2 ** 31 - 1)
+                    : HOST_APPEND_TIMEOUT_MS;
+                const acknowledged = await raceDrain(append, timeoutMs);
+                if (didDrainTimeOut(acknowledged)) {
+                    try {
+                        options.onDetached?.(append, runId);
+                    }
+                    catch {
+                    }
+                    throw new Error(`Host run-log append did not acknowledge within ${timeoutMs}ms`);
+                }
+            }
+            boot = next;
+            return boot;
+        }
+    };
+    return log;
+}
+function getCurrentBoot(log, journal) {
+    const readBoot = Reflect.get(log, ENGINE_BOUNDARY_BOOT);
+    return typeof readBoot === 'function' ? readBoot.call(log) : journal.exportBoot();
+}
+function createOperationPort(journal) {
+    return Object.freeze({
+        flush: () => journal.flush(),
+        operation: operationPort(journal),
+        getReceipt: (id) => journal.getReceipt(id),
+        commitResultFacts: (id, facts) => journal.commitResultFacts(id, facts),
+        uncertainWrite: () => journal.uncertainWrite(),
+        settledCall: (receiptId) => journal.settledCall(receiptId),
+        pendingArgumentsWereRedacted: (snapshot) => journal.pendingArgumentsWereRedacted(snapshot)
+    });
+}
+function operationPort(journal) {
+    return journal.operation.bind(journal);
+}
+function immutableBoot(boot) {
+    return prepareEngineBoot(boot);
+}

package/dist-lib/lib/machine/errors.d.ts

@@ -0,0 +1,44 @@
+import type { Operation } from '../run-log/protocol-types.ts';
+export type { Operation } from '../run-log/protocol-types.ts';
+export declare class RecoveryRequired extends Error {
+    readonly operation: Operation;
+    constructor(operation: Operation);
+}
+export declare class AbortedError extends Error {
+    constructor();
+}
+export declare class ToolPolicyChanged extends Error {
+    constructor(tool: string);
+}
+export declare const EFFECT_DRAIN_TIMEOUT_MS = 5000;
+declare const TIMED_OUT: unique symbol;
+export declare function raceDrain<T>(promise: Promise<T>, timeoutMs?: number): Promise<T | typeof TIMED_OUT>;
+export declare function didDrainTimeOut(value: unknown): value is typeof TIMED_OUT;
+export interface ErrorCause {
+    message: string;
+    name: string;
+    code?: string;
+    stack?: string;
+    cause?: ErrorCause;
+}
+export type EngineError = {
+    tag: 'recovery';
+    message: string;
+    name: string;
+    operation: Operation;
+    stack?: string;
+} | {
+    tag: 'aborted';
+    message: string;
+    name: string;
+    stack?: string;
+} | {
+    tag: 'failed';
+    message: string;
+    name: string;
+    code?: string;
+    stack?: string;
+    cause?: ErrorCause;
+};
+export declare function tagError(error: unknown): EngineError;
+export declare function rehydrateError(value: EngineError | ErrorCause | null | undefined, fallbackMessage?: string): Error;