@qvac/harness 0.7.1 → 1.0.0-rc.1

package/README.md

@@ -308,6 +308,16 @@ bun run build:schema  # regenerates spec/ (wire-compat rewind once v* tags exist
 bun run build:bundle  # stows the sidecar entry into dist/, to prove it still stows
 ```
 
+## Engine layout
+
+`lib/machine/` contains `completion/**`, `engine.ts`, `errors.ts`, `index.ts`, `replay-output.ts`,
+`stable-stringify.ts`, and `guards.ts`. The run-log layer is in `lib/run-log/`:
+`protocol.ts`, `protocol-types.ts`, `history-codec.ts`, `run-journal.ts`,
+`persisted-actor.ts`, `replay.ts`, and `patch.ts`.
+
+The completion graph has 24 states and 61 transitions. Reproduce these
+counts with `examples/bench/graph-metrics.ts`.
+
 The schema baseline rewinds `spec/` against this repo's `v*` release tags and
 skips loudly while none exist. Dependency `overrides` pin the resolutions the
 engine is validated against — keep them aligned with direct specs, npm refuses
@@ -316,3 +326,217 @@ a mismatch on publish.
 ## License
 
 See [LICENSE](./LICENSE.md).
+
+## Resuming a completion
+
+The harness emits resumable steps to a host-owned log. Direct `Harness.completion()` runs without
+that host log work normally, but cannot recover after process death. For remote runs, pass a
+`HostRunLog` to `connectHarness(...).completion()` and reuse the same `runId` when reconnecting.
+The peer must advertise `hostRunLogVersion: 1`.
+
+```ts
+const request = { runId: 'draft-42', model, messages, agent, runLog }
+for await (const event of remote.completion(request)) {
+  // Save the run ID and render completion events.
+}
+
+// begin() returns the saved boot for this run. connectHarness resumes from it and sends the
+// same request identity; supply current callbacks, tool configs, and filesystem grants again.
+for await (const event of remote.completion({ runId: 'draft-42', resume: true, runLog, approve })) {
+  // Handle run-resumed, round-reset, content, tool activity, and paused.
+}
+```
+
+`HostRunLog.begin(runId)` acquires one exclusive generation and returns its compact boot, if the
+run already exists, plus a 32-byte `digestKey`. Each `append(runId, generation, step)` must durably
+commit the whole `EngineStep`—its `seq`, expected logical record length, and records—in one
+transaction. The client waits for that commit before continuing. `release(runId, generation)` fences
+late appends from the ended connection. A host may export its authenticated records for inspection;
+the harness does not fetch storage or decrypt a host's log itself.
+
+`hostAppendTimeoutMs` (default 5000) bounds that wait; it applies in full even to a resume issued
+after the run's own deadline has passed. A spawned host raises it via the `runLog` start payload
+when its appends (e.g. a large first append on resume) can take longer than the default.
+
+Cancellation, iterator disposal, suspension, shutdown, and remote disconnect pause a run. A
+`paused` event ends that stream without making the run terminal. Resume retains completed tool
+results; time spent paused does not count against the run's deadline. A `round-reset` event identifies
+the unfinished round. Ordinary resets carry no `replacedTypes`; only the terminal replay reset
+(when resuming) lists the output kinds to replace. Both replace content and thinking. Text and
+tool events carry run/round identities; tool events also carry their call identity. Completed runs
+replay their final answer and metrics without loading a model.
+
+An unfinished write produces `recovery-required` with an `operation` containing its ID, name,
+resolved arguments, and effect. A write that the pause stopped before it started runs again. Reconcile the external result, then supply exactly that operation's
+resolution on resume:
+
+```ts
+remote.completion({
+  runId: 'draft-42',
+  resume: true,
+  model,
+  messages,
+  agent,
+  recovery: [{ operationId: operation.id, decision: 'result', result: confirmedResult }],
+  runLog,
+  approve
+})
+
+// If re-execution is appropriate, explicitly authorize it instead:
+// recovery: [{ operationId: operation.id, decision: 'retry' }]
+```
+
+A retry can repeat an external effect. Receipts prevent replay of completed operations; they do
+not provide transactions or exactly-once effects in external systems. Missing runtime callbacks or
+a model that cannot load leave the run paused with a recovery reason. For these runtime/read
+reasons, restore the dependencies and resume without a `recovery` resolution. A tool that is gone
+or whose schema changed since the pause ends the run with an error, because no resolution can
+bring the old tool back. Stale resolutions and attempts to change the original conversation or
+model are rejected.
+
+A round can pause with one call's arguments redacted, whether or not a sibling call in the same
+round already landed or was already dispatched. Resuming refuses with `PENDING_ARGS_REDACTED`
+instead of replaying the redacted call. Its `recoveryOperationIds` lists every call the round still
+needs settled before it can proceed: a dispatched write not yet resolved (redacted or not), or a
+call whose real arguments exist only in the redacted snapshot, whether dispatched as a read or
+never dispatched at all. A plain call that was never redacted and never dispatched is not listed;
+it simply runs when the round resumes. Settle any listed operation with `decision: 'result'` and
+the known outcome. `decision: 'skip'` is also available when the host knows a redacted operation
+did not land, or for a read; it records a readable placeholder body,
+`{ skipped: true, reason: 'not executed; arguments unavailable after resume' }`, never the literal
+`undefined`. For a dispatched redacted write, `skip` asserts the call never landed; if the host
+skips a write that did land, the model sees "not executed" and may issue it again. It remains
+refused for a dispatched write whose arguments are available. The `retry` decision re-dispatches a
+call still pending; it is refused once that call already has a settled result, for a call never
+dispatched at all, and for a dispatched call whose arguments were redacted, which re-raises
+`PENDING_ARGS_REDACTED` instead of retrying blind. Settling every listed operation finishes the
+round; the landed write is never repeated, though a redacted read may run again. When no operation
+can be reconciled this way, the error's `forkAt` gives the last log length that never held redacted
+arguments as real ones, and `forkSafe` says whether forking there is actually safe: `false` means a
+write whose intent starts at or after `forkAt` has an unknown outcome — its result already landed, or it was dispatched
+and never resolved — so a fork could repeat it and the host should resolve `recoveryOperationIds`
+instead. `forkRun(sourceCore, targetCore, { at: forkAt, runId })` copies the log up to that point
+under a new `runId`, and the round reruns from there.
+
+Tool definitions are checked again on resume, including their function source. A tool whose
+behavior depends on values captured in a closure must supply and update `policyVersion` when
+those values change; function source cannot reveal them. The `exec` and `http_request` factories
+accept this value as an option; `exec` also fingerprints its static path and Python settings.
+
+Tool configuration values are not stored in the log; a digest verifies that the caller supplies
+the same configuration on resume. Credentials embedded in those values must currently remain
+unchanged. Credentials managed through `setCredentials()` or runtime callbacks can be refreshed
+without changing the request.
+
+For host-owned logs, resume uses the boot returned by `begin()` and does not require the peer's
+local `completionRecovery` capability. A transport `done` frame only means the stream ended; it
+does not mean the run succeeded. Run-log and machine versions must match. XState restore accepts
+older stable patches in the installed minor; future patches, other minors, and prereleases are
+rejected. The first checkpoint and child actor snapshots are trusted inputs; diagnostic replay does
+not verify actor effects or the initial context.
+
+`connectHarness(transport, { runLogReleaseTimeoutMs })` bounds host lease cleanup to 5 seconds
+by default. A rejected, throwing, or timed-out release is logged without replacing the completion
+outcome. The host must keep generation fencing active until release actually settles; a cleanup
+timeout does not authorize another writer.
+
+Each run's metadata contains a JSON-only definition of the machine, including state hierarchy,
+initial states, tags, invoked actors, and named transition guards/actions. Checkpoints and deltas
+also carry the exact event, source/target state IDs, fired guard names, and action names observed in
+XState's transition trace. The inspector displays these recorded traces and can replay state/context
+at the same machine and XState versions; event-triggered guard/action traces are re-evaluated and
+must match the recorded trace exactly, or replay reports a divergence. Eventless traces (no
+triggering event, e.g. `always` transitions) are not replay-verified; the inspector marks each such
+line `[eventless; not replay-verified]`. A version mismatch, legacy log, or prefix selection skips
+replay and leaves the stored snapshot and trace authoritative. The checked-in [completion
+graph](./docs/statecharts/completion-machine.mmd) is generated from the same machine definition and
+CI rejects a stale artifact.
+
+Export records from the host, then inspect a selected logical record or physical core length:
+
+```sh
+bare examples/run-log/inspect.ts <records.json> --context <recordIndex>
+bare examples/run-log/view.ts <records.json> report.html --limit <recordIndex>
+```
+
+Logical record index, engine step sequence, and physical storage length are different positions;
+host exports can include the mapping. The CLI reads exported records only. The host remains
+responsible for remote lookup, authentication, decryption, and sharing access.
+
+`@qvac/harness/run-log-inspect` is the library behind that CLI: `inspectRun(core, opts)` yields the
+same lines the `inspect.ts` example prints, and `renderRunLogHtml(core, opts)` renders the HTML
+report `view.ts` writes. A host imports either directly instead of shelling out to the examples,
+passing a `MachineLog` adapter or an envelope whose `records()` method returns an async iterable
+of `{ recordIndex, stepSeq, physicalLength, record }` entries. A raw records array or parsed JSON
+envelope must first be adapted; the CLI's `recordsCore` helper shows this conversion. Options are
+`recordLimit` (an exclusive logical index), `contextRecord`, `physicalLength`, and
+`contextPhysicalLength`.
+
+The completion machine is internal: its state names are part of the run-log format, not the API.
+Hook consumers observe `runPause` and `runResume` for a run that can resume. A run with no run-log
+store cannot resume, so a pause ends it and `runEnd` reports `aborted` (the caller aborted) or
+`abandoned` (the consumer stopped iterating).
+
+## Tool-call markup grammar
+
+A tool-call block opens with `<function`, up to 64 ASCII whitespace characters, `=`, up to 64
+more ASCII whitespace characters, a name, up to 64 ASCII whitespace characters, and `>`. Tags are
+case-insensitive, except in the legacy nested reading below. An executable name is 1–64
+non-whitespace characters other than `<` or `>`; persistence also recognizes longer names so an
+oversized call cannot bypass redaction. A parameter uses the same form with `parameter` in place
+of `function`, and ends at the first case-insensitive
+`</parameter>`. A function block ends at the first case-insensitive `</function>`. Anything after
+that closer is prose and cannot extend the call, including more parameters or closers.
+
+Text that does not form a complete opener and closed block is prose. In particular, empty names,
+overlong executable names, bare parameter tags, and function-tag examples without a matching
+closer are neither executed nor hidden from the displayed answer. Stray function closers, including
+`</FUNCTION>`, also stay visible. A function opener inside a function block or a parameter opener
+inside a parameter value makes that block ambiguous, and persistence redacts it conservatively.
+A block with a nested function opener keeps the legacy recovery reading: the first lowercase
+`<function=` with no whitespace before `=` names the call. The text between `=` and `>` is limited
+to 64 characters before the name is trimmed. It runs with complete lowercase
+parameters up to the first lowercase `</function>` within that block. That reading ignores other
+function openers, so `<function=a><function=login>…` runs `a`,
+`<FUNCTION=a><function=login>…` and `<function =a><function=login>…` run `login`, and
+`<FUNCTION=a><FUNCTION=login>…` runs nothing. Likewise,
+`<function=x><function= login ><parameter=path>p</parameter></function>` runs `x` with `path: "p"`
+when `x` is a known tool. An unclosed parameter, such as `<parameter=key>S</function>`, contributes
+no argument and does not itself make the function block ambiguous. Persistence still redacts its
+value when the name is a credential field or a registered sensitive parameter, including in prose.
+
+## Run-log data and privacy
+
+The harness redacts structured values before it sends a durable step to the host. String or object
+values under exact credential field names are redacted. Numeric values under known credential
+names are also redacted; generic `token` counts and `secret` flags are preserved. Registered tool
+parameters marked `sensitive: true` are redacted regardless of name or value type. Known names include `db_password`, `GITHUB_TOKEN`,
+`openai_api_key`, `client_password`, and `client_passphrase` (case, underscores, and hyphens
+are normalized).
+Name-based redaction is best effort: provider key names such as `ANTHROPIC_API_KEY`, `GH_TOKEN`,
+and `PGPASSWORD` are not covered, and a bare `token` argument (e.g. a pagination cursor) is
+redacted and lost on resume just like a real secret. The durable way to mark a value as a secret,
+regardless of its name, is the schema's `sensitive: true` annotation.
+Encoded assistant tool calls are redacted when history crosses persistence. Within one live process,
+a model's next request still sees its own prior call's real arguments, because that history lives
+only in the actor's memory. A **resumed** run has no such memory: its context is rebuilt from the
+persisted (redacted) log, so the model's first request after resume sees `[redacted]` for any tool
+call it made before the run was interrupted, not the original argument; that argument cannot be
+recovered. The model sees that prior call as well-formed markup with the literal `"[redacted]"`
+value, not an error; a model that re-issues such a call runs it with that literal placeholder.
+A persisted `raw` or `rawCall` of 4,096 characters or more is rebuilt from its (redacted) arguments
+even when nothing is sensitive, so it may differ byte-for-byte from the live call's `rawCall`.
+Shorter unchanged text stays identical to the original.
+Other arbitrary strings, tool results, and prose are preserved. Persisted machine
+history can reference matching original request messages by index and materializes to the same
+messages on resume and inspection — this applies to the caller-supplied conversation the run
+started from, which is never redacted, not to arguments the model generated mid-run. Hosts must
+treat free text and results as confidential and
+protect their log with encryption and access control. The host's `digestKey` verifies request
+identity and never enters a run record.
+
+A pending tool call whose arguments were redacted cannot be executed on resume and is rejected.
+A literal `[redacted]` supplied by the user is not enough to trigger that check. Tool configuration
+values are not stored in the run log; a keyed digest verifies that the caller supplies the same
+configuration on resume. Credentials managed through `setCredentials()` or runtime callbacks can
+be refreshed without changing the request identity.

package/dist-lib/index.d.ts

@@ -1,5 +1,6 @@
 import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
 import ReadyResource from 'ready-resource';
+import type { EngineBoot, EngineStep, RecoveryResolution } from './lib/machine/index.ts';
 import { type Agent, type ApproveFn, type AskUserFn, type AttachFn, type CompletionEvent, type FetchAttachmentFn, type Message, type ProgressFn, type RagSearchFn, type ResponseFormat, type ToolDefinition, type ToolExecuteContext } from './lib/capabilities/completion.ts';
 import { type DetectLanguageContext, type DetectedLanguage } from './lib/capabilities/detect-language.ts';
 import { type Capability } from './lib/capabilities/invoke.ts';
@@ -40,6 +41,7 @@ export * from './lib/models/model-catalog.ts';
 export * from './lib/models/model-list.ts';
 export type { ModelTaskType } from './lib/models/model-task.ts';
 export type { DeleteModelResult, DeleteModelSummary } from './lib/models/model-delete.ts';
+export type { RecoveryResolution, RecoveryOperation } from './lib/machine/index.ts';
 export { harnessChild, spawnHarness, type HarnessChildOptions, type SpawnHarnessOptions } from './lib/spawn.ts';
 export { settingsDimension, settingsSteps } from './lib/tools/generate-image.ts';
 export * from './lib/tools/skills/catalog.ts';
@@ -52,6 +54,7 @@ export { readSetup as normalizeSkillSetup } from './lib/tools/skills/manifest-pa
 export type { CredentialRecord, CredentialStoreValue } from './lib/tools/skills/auth-resolver.ts';
 export { isAvailableState, SKILL_STATE, type SkillState } from './lib/tools/skills/skill-state.ts';
 export { connectHarness, HARNESS_DIED, RemoteHarness } from './lib/connect.ts';
+export type { ConnectOptions, HostRunLog, HostRunLogLease } from './lib/connect.ts';
 export type { KvCacheDeletionResult } from './lib/capabilities/kv-cache.ts';
 export type { AgentKnobs, ApprovalCall, ChatMessage, CredentialEntry, DownloadContext, HarnessContract, HarnessDiagnostics, HarnessEvent, HarnessInfo, HarnessStatus, InfraModel, ModelState, SkillsOptions, SkillWriteResult } from './lib/contract-types.ts';
 import type { HarnessInfo } from './lib/contract-types.ts';
@@ -96,6 +99,9 @@ export interface HarnessOptions {
     completionModelCap?: number;
 }
 export interface CompletionContext {
+    runId?: string;
+    resume?: boolean;
+    recovery?: RecoveryResolution[];
     model: string;
     agent?: Agent;
     messages: Message[];
@@ -115,8 +121,16 @@ export interface CompletionContext {
     ragSearch?: RagSearchFn;
     progress?: ProgressFn;
     askUser?: AskUserFn;
+    boot?: EngineBoot;
+    onStep?: (step: EngineStep) => Promise<void>;
+    hostAppendTimeoutMs?: number;
+    requestDigestKey?: Uint8Array;
     signal?: BareAbortSignal;
 }
+export type ResumeCompletionContext = Omit<Partial<CompletionContext>, 'runId' | 'resume'> & {
+    runId: string;
+    resume: true;
+};
 export interface InvokeContext {
     capability: Capability;
     args: Record<string, unknown>;
@@ -145,6 +159,7 @@ export declare class Harness extends ReadyResource {
     private _models;
     private _embedModels;
     private readonly _retiredEmbedModels;
+    private readonly _faultedEmbedKeys;
     private _embedLane;
     private _embedFaults;
     private readonly _embedGpuLayers?;
@@ -153,7 +168,20 @@ export declare class Harness extends ReadyResource {
     private _ttsModels;
     private _transcriptionModels;
     private _voiceLane;
-    private _state;
+    private readonly _lifecycle;
+    private readonly _detachedRuns;
+    private readonly _effectDrainWaiters;
+    private _activeWork;
+    private readonly _workDrainWaiters;
+    private _runtimeDraining;
+    private _runtimeUnloadFailed;
+    private _poolUnloadFailed;
+    private _poolCleanupFailed;
+    private readonly _failedModelUnloads;
+    private _sdkClosed;
+    private _runtimesReleased;
+    private _closeRetryNeeded;
+    private _closeRequested;
     private _skillsDir?;
     private readonly _modelsDir?;
     private _overlayWatcher;
@@ -178,6 +206,7 @@ export declare class Harness extends ReadyResource {
     private _musicSdk;
     private _translationSdk;
     private _musicRuntime;
+    private readonly _completionRuns;
     private _translationRuntime;
     private _capabilityTools;
     private _capability;
@@ -228,26 +257,46 @@ export declare class Harness extends ReadyResource {
     private _loadMusicSdk;
     private _deviceConfig;
     private _createPool;
+    private _loadModelLifecycleSdk;
     private _createEmbedPool;
     private _createTtsPool;
     private _createTranscriptionPool;
+    private _recordModelUnloadFailure;
     private _notifyModel;
     private _collectRetiredEmbedPool;
     private _releaseEmbedResource;
+    private _releaseEmbedResourceBeforeWork;
+    private _markEmbedFault;
     private _releaseEmbedResourceAfter;
     private _destroyEmbedPools;
     private _enqueueEmbedControl;
+    private _runLifecycle;
+    private _abortCompletions;
+    private _assertNotClosing;
+    private _enterWork;
+    private _waitForWorkDrain;
+    private _effectsSettled;
+    private _notifyEffectDrain;
+    private _waitForEffectDrain;
+    private _trackDetachedRun;
+    private _closeCompletions;
     getEmbeddingDevice(): EmbedDevice;
     setEmbeddingDevice(device: EmbedDevice): Promise<void>;
     private _setEmbeddingDevice;
     presuspend(): Promise<void>;
     suspend(): Promise<void>;
+    private _suspendRuntimes;
     private _sdkLifecycle;
     private _releaseRuntimes;
     private _unloadIdleModels;
     resume(): Promise<void>;
+    private _resumeRuntimes;
     wakeup(): Promise<void>;
+    ready(): Promise<void>;
     _close(): Promise<void>;
+    close(): Promise<void>;
+    private _closeWhileOpening;
+    private _finishClose;
     download(args: DownloadArgs): DownloadEventStream;
     embed(context: EmbedContext): Promise<number[][]>;
     private _embed;
@@ -278,10 +327,13 @@ export declare class Harness extends ReadyResource {
     private _yieldPins;
     private _residency;
     private _completionLoadOptions;
+    private _acquireModelLease;
     invoke(context: InvokeContext): AsyncGenerator<CompletionEvent>;
     private _invoke;
-    completion(context: CompletionContext): AsyncGenerator<CompletionEvent>;
-    private _completion;
+    completion(context: CompletionContext | ResumeCompletionContext): AsyncGenerator<CompletionEvent>;
+    private _runCompletion;
+    private _completionFailed;
+    private _completionRuntime;
     private _residentEnhancer;
     private _makeRoomForCompletion;
     private _evictWedgedModel;