@ricsam/r5d-worker 0.0.122 → 0.0.124

package/dist/types/recovery-journal.d.ts

@@ -0,0 +1,246 @@
+import type { WorkerOperationRecovery, WorkerOutputOffsets } from "./recovery-protocol";
+/**
+ * The control-thread side of the worker's durable recovery journal.
+ *
+ * The SQLite store (WAL, `synchronous=FULL`) lives on the journal thread;
+ * this facade owns one ordered channel of store operations and the ordered
+ * delivery of the messages those operations make durable. Its invariants:
+ *
+ * - order: every entry is applied in append order, one durable commit per
+ *   batch, so an `unknown` appended after a `record` applies after it, and an
+ *   `admit` of a later request commits after the `record` of an earlier one;
+ * - commit before send: a keyed message is written to the socket only after
+ *   the batch holding its `record` committed, in append order, so a terminal
+ *   never leaves before the output recorded ahead of it and a response is
+ *   never acknowledged before it is durable;
+ * - commit before effects: `admit` resolves after its commit; handlers run
+ *   after it;
+ * - bounded: everything the facade retains (entries waiting for the thread,
+ *   entries in the batch being committed, and committed messages not yet
+ *   drained to the socket) is charged to one general budget of entries and
+ *   bytes, reserved atomically before anything is queued and released only
+ *   when the item is actually done; unknown marks, acknowledgements and
+ *   handshake reads use a separate reserved control budget. Output producers
+ *   and admissions wait for budget in bounded waiting rooms; a result that
+ *   cannot be buffered is never sent early: its operation is marked unknown,
+ *   which is exactly what a crash would have left;
+ * - fail closed: a thread exit, a failed batch, or an exhausted control budget
+ *   fails every pending and future entry and reports once; the store is never
+ *   re-opened in-process.
+ */
+type Message = {
+    type: string;
+    [key: string]: unknown;
+};
+export declare const WORKER_RECOVERY_JOURNAL_MAX_BATCH_ENTRIES = 256;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_PENDING_ENTRIES = 4096;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_PENDING_BYTES: number;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_CONTROL_ENTRIES = 2048;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_CONTROL_BYTES: number;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_ENTRY_BYTES: number;
+export declare const WORKER_RECOVERY_JOURNAL_OUTPUT_CHUNK_CHARACTERS: number;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_WAITING_ADMISSIONS = 512;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_WAITING_OUTPUT = 1024;
+export declare const WORKER_RECOVERY_JOURNAL_MAX_WAITING_BYTES: number;
+export declare const WORKER_RECOVERY_JOURNAL_REPLAY_PAGE_FRAMES = 32;
+export declare class WorkerRecoveryJournalError extends Error {
+    readonly name: string;
+}
+/** The journal is intact but its bounded waiting room is full; the caller must not queue more. */
+export declare class WorkerRecoveryJournalBusyError extends WorkerRecoveryJournalError {
+    readonly name = "WorkerRecoveryJournalBusyError";
+}
+export type WorkerRecoveryJournalAdmission = {
+    admission: "new" | "duplicate" | "unknown" | "conflict";
+    response: Message | undefined;
+    cancelled: boolean;
+};
+export type WorkerRecoveryJournalRecoveryState = {
+    operations: WorkerOperationRecovery[];
+    cursors: WorkerOutputOffsets[];
+    responses: Message[];
+    terminals: Message[];
+};
+export type WorkerRecoveryJournalStatus = {
+    /** Entries waiting for the thread plus entries in the batch being committed. */
+    pendingEntries: number;
+    /** General budget in use: pending entries and undelivered messages, in entries and bytes. */
+    retainedEntries: number;
+    retainedBytes: number;
+    controlEntries: number;
+    controlBytes: number;
+    waitingAdmissions: number;
+    waitingOutput: number;
+    /** Payload bytes held by requests and output chunks waiting for budget. */
+    waitingBytes: number;
+    /** Bytes of store replies (handshake state, output pages) the caller has not consumed yet. */
+    retainedReplyBytes: number;
+    committedSeq: number;
+    appendedSeq: number;
+    undelivered: number;
+    failed: string | null;
+};
+export type WorkerRecoveryJournalOptions = {
+    filename: string;
+    threadModulePath?: string;
+    /** Extra `workerData` for the thread module (test fixtures only). */
+    threadWorkerData?: Record<string, unknown>;
+    log?: (line: string) => void;
+    maxBatchEntries?: number;
+    maxPendingEntries?: number;
+    maxPendingBytes?: number;
+    maxControlEntries?: number;
+    maxControlBytes?: number;
+    maxEntryBytes?: number;
+    maxWaitingAdmissions?: number;
+    maxWaitingOutput?: number;
+    maxWaitingBytes?: number;
+};
+export declare class WorkerRecoveryJournal {
+    readonly ledgerId: string;
+    private readonly worker;
+    private readonly log;
+    private readonly maxBatchEntries;
+    private readonly maxEntryBytes;
+    private readonly maxWaitingAdmissions;
+    private readonly maxWaitingOutput;
+    private readonly maxWaitingBytes;
+    private waitingBytes;
+    private retainedReplyBytes;
+    private readonly general;
+    private readonly control;
+    private nextSeq;
+    private committedSeq;
+    private readonly pending;
+    private inFlight;
+    private nextBatchId;
+    private readonly waiters;
+    private readonly outbound;
+    private deliveryPaused;
+    private replayCoveredSeq;
+    private failure;
+    private closed;
+    private readonly pendingAdmissions;
+    private readonly activeOperations;
+    private readonly cancelledRequestIds;
+    private readonly unknownRequestIds;
+    private readonly unknownRunIds;
+    private readonly coalescedControl;
+    private readonly flushWaiters;
+    /** Delivers one committed message in order; returns false when the transport declined it (the pumps re-send). */
+    deliver: ((message: Message) => boolean) | undefined;
+    /** Invoked once when the journal fails closed. */
+    onFailure: ((error: Error) => void) | undefined;
+    private constructor();
+    /** Start the journal thread, open the store there (its crash recovery runs there) and return once it is ready. */
+    static open(options: WorkerRecoveryJournalOptions): Promise<WorkerRecoveryJournal>;
+    status(): WorkerRecoveryJournalStatus;
+    /**
+     * Durably admit an inbound operation; resolves after the commit. Waits for
+     * general budget in a bounded waiting room; when that room is full the
+     * admission is refused at once (WorkerRecoveryJournalBusyError) so the
+     * message is neither acknowledged nor retained.
+     */
+    admit(message: Message & {
+        requestId: string;
+    }): Promise<WorkerRecoveryJournalAdmission>;
+    /**
+     * Re-admit an unknown branch deletion. Like `admit`, the request is a
+     * pending admission from before the await, so a session cancellation that
+     * lands while the disk is slow snapshots it and the handler's recheck after
+     * the await refuses to rerun the deletion.
+     */
+    readmitUnknownBranchDeletion(requestId: string, sessionId: string | undefined): Promise<boolean>;
+    /**
+     * Queue an outbound message. Keyed messages are recorded (when the store
+     * records their type) and delivered in order after their commit; other
+     * messages are delivered at once. Returns false when the message was not
+     * queued: its operation is already unknown, or the budget cannot hold it,
+     * in which case the operation is marked unknown so it is neither
+     * acknowledged early nor sent later.
+     */
+    send(message: Message): boolean;
+    /**
+     * Queue an output frame, split into bounded records, waiting for budget so
+     * a stalled journal throttles the producer (and through the pipe, the
+     * child) instead of growing memory.
+     */
+    sendOutput(frame: Message & {
+        type: "exec_output";
+        runId: string;
+        stream: "stdout" | "stderr";
+        data: string;
+    }): Promise<void>;
+    /**
+     * Reserve general budget atomically, waiting in the bounded room of `kind`
+     * when it does not fit now. The waiting room is bounded in count and in
+     * payload bytes (the message a waiting admission holds is charged before it
+     * waits), and an entry that could never fit the general budget is refused
+     * at once rather than parked at the head of the queue.
+     */
+    private reserveGeneral;
+    /** Reserve for the head waiter inside this synchronous call, so no continuation can outrun the accounting. */
+    private wakeWaiters;
+    private enqueueRecord;
+    private refuse;
+    private trackRecorded;
+    /**
+     * Mark an operation unknown: never sent again (undelivered messages for it
+     * are dropped now, and the durable mark keeps replay from resurrecting it).
+     * Coalesced per id; uses the reserved control budget.
+     */
+    unknown(requestId: string): void;
+    unknownRun(runId: string): void;
+    acknowledgeResult(requestId: string): void;
+    /** Monotonic per stream: a later acknowledgement replaces a pending earlier one. */
+    acknowledgeOutput(runId: string, stream: "stdout" | "stderr", offset: number): void;
+    acknowledgeTerminal(runId: string): void;
+    /**
+     * Snapshot every admitted and pending operation of the session as cancelled
+     * (the handlers check `isRequestCancelled` at their fences) and append the
+     * durable cancellation, which the thread applies after every admission
+     * appended before it. Resolves after the commit.
+     */
+    cancelSession(sessionId: string): Promise<void>;
+    isRequestCancelled(requestId: string): boolean;
+    /** Every operation admitted or awaiting admission in this process that has not completed, for the runtime fence and lease expiry. */
+    activeAndPendingOperations(): WorkerOperationRecovery[];
+    /**
+     * The handshake read. One entry, so it sees exactly the entries appended
+     * before it and none after: the replay boundary. Callers pause delivery
+     * before appending it and resume after the handshake, so every keyed
+     * message committed before the mark is replayed once (from the store) and
+     * nothing committed after it can enter the replay of responses, operations
+     * or terminals; output pages that follow may re-send frames the pump would
+     * re-send anyway, which the server's offset cursors absorb.
+     */
+    recoveryState(requests: string[]): Promise<WorkerRecoveryJournalRecoveryState>;
+    private retainedStateBytes;
+    /** Output replay, one bounded page at a time; a page is accounted until the consumer asks for the next one. */
+    outputReplay(offsets: WorkerOutputOffsets[] | undefined, pageFrames?: number): AsyncGenerator<Message[]>;
+    terminals(): Promise<Message[]>;
+    /** Delivery of committed keyed messages is held (in order) while a handshake replays from the store. */
+    pauseDelivery(): void;
+    resumeDelivery(): void;
+    /** Resolves once every entry appended so far is committed (or the journal failed). */
+    flush(): Promise<void>;
+    close(): void;
+    private assertOpen;
+    /** Control entries are tiny and idempotent per key: a pending one is updated in place instead of queued again. */
+    private appendControl;
+    /** A control-budget entry whose result the caller awaits; rejects when the budget is exhausted. */
+    private appendControlOrThrow;
+    private lastAppended;
+    /** Queue an entry whose budget was already reserved from `budget`. */
+    private append;
+    private dispatch;
+    private onReply;
+    private dropOutbound;
+    private releaseOutbound;
+    private deliverReady;
+    private fail;
+    private failEntries;
+}
+/** The packaged thread module lives next to this module with the same extension (.ts in source, .cjs/.mjs in the package). */
+export declare function workerRecoveryJournalThreadModulePath(): string;
+export {};

package/dist/types/recovery-store.d.ts

@@ -11,6 +11,12 @@ export declare class WorkerRecoveryStore {
     readonly ledgerId: string;
     constructor(filename: string);
     close(): void;
+    /**
+     * Run several store operations as one durable commit (one FULL fsync).
+     * Operations that open their own transaction nest as savepoints. A throw
+     * rolls the whole group back; callers treat that as a journal failure.
+     */
+    batch<T>(operations: () => T): T;
     private row;
     admit(message: Message & {
         requestId: string;

package/dist/types/session-file-mutations.d.ts

@@ -0,0 +1,22 @@
+/**
+ * File mutations (write, edit, plan updates) run on the asynchronous
+ * filesystem API, so a write can still be in flight when a session is
+ * stopped. A Stop acknowledges cleanup only after every mutation that had
+ * already started for the session has actually settled; mutations that were
+ * still queued re-check the session's cancellation immediately before their
+ * first write and never start. The tracker records the former; the admission
+ * fence inside each mutation's queue callback provides the latter.
+ */
+export declare class SessionFileMutationTracker {
+    private readonly inFlight;
+    /** Register a started mutation; it is forgotten when it settles, however it settles. */
+    track<T>(sessionId: string, mutation: Promise<T>): Promise<T>;
+    count(sessionId: string): number;
+    /**
+     * Resolves once every mutation tracked for the session at the time of the
+     * call has settled. Mutations started later are not joined: the session's
+     * cancellation marker, recorded before this is awaited, makes their
+     * admission recheck refuse them.
+     */
+    settled(sessionId: string): Promise<void>;
+}

package/dist/types/workspace-command-sync-policy.d.ts

@@ -1,5 +1,7 @@
-type WorkspaceCommandTarget = {
+export type WorkspaceCommandTarget = {
     type: "project";
+    projectId: string;
+    branchName: string;
 } | {
     type: "workspace";
     rootProfile: "visible_projects" | "canonical_sync";
@@ -8,16 +10,29 @@ type WorkspaceCommandMutationCoordinator = {
     runMutation<T>(operation: () => Promise<T> | T): Promise<T>;
     acquireMutation(): Promise<() => void>;
 };
-type WorkspaceSyncCompletionBarrier = {
+export type WorkspaceSyncCompletionBarrier = {
     afterCurrent(): Promise<void>;
+    /**
+     * Null when no workspace filesystem job holds the target's mount right now;
+     * otherwise a promise that settles once the current holders released it
+     * (rejecting with WorkspaceMountHoldAbortedError when `signal` aborts
+     * first). Workspace targets have no mount and always get null.
+     */
+    mountHold(target: WorkspaceCommandTarget, signal?: AbortSignal): Promise<void> | null;
 };
 /**
- * A project command may reserve immediately: per-mount hydration rechecks its
- * busy predicate immediately before touching visible bytes. A visible-projects
- * workspace command spans every project, so it keeps the whole-cycle barrier.
- * Canonical remediation is serialized separately by the mutation gate.
+ * A project command may reserve as soon as no workspace filesystem job holds
+ * its mount: per-mount hydration and projection observe busy-ness before they
+ * dispatch a job and hold the mount until the job ended, so a reservation
+ * taken here is never interleaved with a job reading or writing the mount. A
+ * visible-projects workspace command spans every project, so it keeps the
+ * whole-cycle barrier. Canonical remediation is serialized separately by the
+ * mutation gate. Commands without a workspace effect (incident-9 control
+ * commands among them) never come here and never wait.
  */
-export declare function reserveWorkspaceCommandAfterCurrentSync(target: WorkspaceCommandTarget, coordinator: WorkspaceSyncCompletionBarrier, reserve: () => void): Promise<void>;
+export declare function reserveWorkspaceCommandAfterCurrentSync(target: WorkspaceCommandTarget, coordinator: WorkspaceSyncCompletionBarrier, reserve: () => void, options?: {
+    signal?: AbortSignal;
+}): Promise<void>;
 /**
  * Commands in visible checkouts behave like independent Git clients and may
  * overlap periodic synchronization. Canonical remediation commands mutate the

package/dist/types/workspace-filesystem-executor-thread.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/workspace-filesystem-executor.d.ts

@@ -0,0 +1,123 @@
+import { type WorkspaceFilesystemJobInput, type WorkspaceFilesystemJobKind, type WorkspaceFilesystemJobProgress, type WorkspaceFilesystemJobResult } from "./workspace-filesystem-job-types";
+/**
+ * Bounded executor for the worker's workspace filesystem transactions.
+ *
+ * One thread runs one job at a time; the control thread only ever awaits.
+ * The point is the incident this was written for: a single `pread64` or
+ * `pwrite64` blocked in the kernel on a stalled network disk used to freeze
+ * the worker's only event loop for minutes, so no heartbeat, cancel or Stop
+ * could run. On the executor thread that same blocked syscall stalls exactly
+ * one job while the control thread keeps serving the socket.
+ *
+ * A thread rather than a helper process because a thread cannot outlive the
+ * process: when the worker dies, its in-flight syscall completes and nothing
+ * else is written afterwards, which is the crash model every recovery path
+ * already handles. Nothing is ever abandoned: a running job's promise settles
+ * only with the thread's reply or the thread's exit, so the gate its caller
+ * holds is held for the job's true duration. Only jobs that have not started
+ * can fail on the queue-wait bound.
+ *
+ * A thread that exits while a job is running leaves an unknown partial
+ * mutation behind. The executor then fails closed: the job and everything
+ * queued fail, no further job is accepted, and `onFailure` lets the worker
+ * take its established restart path, whose startup recovery (manifests,
+ * receipts, snapshot recovery) runs before any ordinary mutation. No
+ * replacement thread is started over that state in this process.
+ */
+export declare const WORKSPACE_FILESYSTEM_EXECUTOR_MAX_QUEUED = 32;
+export declare const WORKSPACE_FILESYSTEM_EXECUTOR_QUEUE_TIMEOUT_MS: number;
+export declare const WORKSPACE_FILESYSTEM_EXECUTOR_STALL_LOG_INTERVAL_MS = 30000;
+export declare class WorkspaceFilesystemExecutorError extends Error {
+    readonly name = "WorkspaceFilesystemExecutorError";
+}
+/** The executor thread exited (crashed or was terminated) while a job was running. */
+export declare class WorkspaceFilesystemExecutorThreadExitedError extends WorkspaceFilesystemExecutorError {
+    readonly exitCode: number | null;
+    readonly description: string;
+    constructor(exitCode: number | null, description: string, cause?: unknown);
+}
+export type WorkspaceFilesystemExecutorStatus = {
+    running: {
+        kind: WorkspaceFilesystemJobKind;
+        description: string;
+        startedAtMs: number;
+        ageMs: number;
+    } | null;
+    queued: number;
+    /** Set once the thread exited under a running job; every job is then rejected until the worker restarts. */
+    failed: string | null;
+};
+export type WorkspaceFilesystemExecutorOptions = {
+    /** Module the thread executes; must resolve next to the packaged executor module. */
+    threadModulePath: string;
+    log?: (line: string) => void;
+    now?: () => number;
+    maxQueued?: number;
+    queueTimeoutMs?: number;
+    stallLogIntervalMs?: number;
+};
+export declare class WorkspaceFilesystemExecutor {
+    private readonly threadModulePath;
+    private readonly log;
+    private readonly now;
+    private readonly maxQueued;
+    private readonly queueTimeoutMs;
+    private readonly stallLogIntervalMs;
+    private worker;
+    private readonly queue;
+    private running;
+    private nextJobId;
+    private failed;
+    private closed;
+    /** Invoked once when the executor fails closed (thread exit under a running job). */
+    onFailure: ((error: WorkspaceFilesystemExecutorThreadExitedError) => void) | undefined;
+    constructor(options: WorkspaceFilesystemExecutorOptions);
+    /**
+     * Run one job to completion on the executor thread. Resolves with the
+     * job's result; rejects with the job's own error (properties preserved),
+     * with a queue-wait or capacity error when the job never started, or with
+     * WorkspaceFilesystemExecutorThreadExitedError when the thread died under it.
+     */
+    run<K extends WorkspaceFilesystemJobKind>(kind: K, input: WorkspaceFilesystemJobInput<K>, options?: {
+        onProgress?: (progress: WorkspaceFilesystemJobProgress) => void;
+    }): Promise<WorkspaceFilesystemJobResult<K>>;
+    status(): WorkspaceFilesystemExecutorStatus;
+    /**
+     * Stop accepting jobs, fail everything still queued and ask the thread to
+     * stop. A running job settles when the thread actually exits; a thread
+     * blocked in the kernel only exits with the process, which is the point.
+     */
+    close(): void;
+    private describeRunning;
+    private expireQueued;
+    private dispatch;
+    private settleRunning;
+    private ensureWorker;
+    private lastThreadError;
+    private onReply;
+    private onExit;
+}
+/** The packaged thread module lives next to this module with the same extension (.ts in source, .cjs/.mjs in the package). */
+export declare function workspaceFilesystemExecutorThreadModulePath(): string;
+/** What the engine modules dispatch through; the thread executor in production. */
+export type WorkspaceFilesystemJobRunner = Pick<WorkspaceFilesystemExecutor, "run">;
+/**
+ * The worker process's executor. Like the workspace mutation gate it is
+ * process-global: in-process reconnects share it, so a job started by a
+ * previous connection generation is still the one running when the next
+ * generation's configuration queues behind it.
+ */
+export declare function workspaceFilesystemExecutor(): WorkspaceFilesystemJobRunner;
+/** The worker runtime's fail-closed hook: installed once on the process executor (created if needed). */
+export declare function installWorkspaceFilesystemExecutorFailureHandler(handler: (error: WorkspaceFilesystemExecutorThreadExitedError) => void): void;
+/**
+ * Test seam. Engine tests that inject faults or count syscalls by replacing
+ * `node:fs` functions in their own thread cannot observe a body running on
+ * the executor thread; inside `runJobsInline` the same operation table runs
+ * on the calling thread instead. The boundary itself is tested with the real
+ * thread in workspace-filesystem-executor.test.ts. Never used in production.
+ */
+export declare const workspaceFilesystemExecutorTestHarness: {
+    runJobsInline<T>(body: () => Promise<T>): Promise<T>;
+    processExecutor(): WorkspaceFilesystemExecutor | null;
+};