adispatchd 0.0.0-stage → 0.1.0

package/dist/runs/controller.d.ts

@@ -0,0 +1,307 @@
+/**
+ * Run controller (Spec §8, §11, §13, §14): drives a reserved run through
+ * `starting → running → finalizing → <terminal>`, classifies the outcome,
+ * applies technical-failure counting and retry blocks, runs the completion
+ * hook, cleans up the workspace, supervises cancel, recovers after a
+ * controller crash and implements pause / drain / cancel-all shutdown.
+ *
+ * Invariants:
+ * - Every status transition is persisted before work continues; terminal
+ *   statuses are reached only via `finalizing` with a persisted
+ *   {@link FinalizationTarget} (WI-01/WI-03).
+ * - The retry block is created in the SAME transaction as `→ finalizing`,
+ *   i.e. before the completion hook runs (Spec §11 "important order").
+ * - Hooks always use the run's configuration snapshot, never the live config.
+ * - Bus publication and notifications happen only after commit.
+ * - A process is only ever signalled after verifying its start identity
+ *   (recovery); on doubt the run becomes `recovery_pending` and keeps its lock.
+ */
+import { type Clock, type CredentialProvider, type Executor, type Logger, type Notifier, type SourceItem, type TechnicalStatus, type Timers } from '../core/index.js';
+import { type KillFn, type ProcessInspector } from '../executors/index.js';
+import { type LifecycleEngine, type SourceResolver } from '../lifecycle/index.js';
+import { type BlockRecord, type Repositories, type RunRecord } from '../persistence/index.js';
+import { PersistedEventPublisher, RunEventBus, type ControllerStatus } from './event-bus.js';
+/** Orchestrator-side event types persisted by the controller (besides `run.status`). */
+export declare const RUN_EVENTS: {
+    /** Worker spawn is about to happen (`{command, args, workingDirectory}`); recovery uses it. */
+    readonly spawn: 'run.spawn';
+    /** Item re-read before `on_start` (Spec §7): `{found, matches?}`. */
+    readonly itemReread: 'run.item_reread';
+    readonly cancelRequested: 'run.cancel_requested';
+    /** Process group could not be terminated; run stays `cancelling` (Spec §13). */
+    readonly cancelFailed: 'run.cancel_failed';
+    /** A completion hook threw unexpectedly. */
+    readonly hookError: 'run.hook_error';
+    /** Unexpected internal error while supervising a run (run left as is for recovery). */
+    readonly controllerError: 'run.controller_error';
+    /** Recovery could not identify / stop the process → `recovery_pending`. */
+    readonly recoveryFailed: 'recovery.failed';
+    /** Recovery decided the run's fate (`{action, detail}`). */
+    readonly recovered: 'recovery.recovered';
+    /** Workspace cleanup result. */
+    readonly workspaceCleanup: 'workspace.cleanup';
+};
+/** Optional per-start settings. */
+export interface StartRunOptions {
+    /**
+     * Re-check of the freshly re-read item (Spec §7 race), e.g. the scheduler's
+     * cheap filter. `false` → the run is not started and finalized as
+     * `cancelled` (not counted, no hooks). Manual starts omit it.
+     */
+    verifyItem?: (item: SourceItem) => boolean | Promise<boolean>;
+}
+/** Handle returned by {@link RunController.startRun}. */
+export interface RunHandle {
+    runId: string;
+    /**
+     * Resolves with the run record once supervision ended (terminal, or left
+     * in its state for recovery). Rejects only if the database itself fails.
+     */
+    done: Promise<RunRecord>;
+}
+/** Shutdown mode. */
+export type ShutdownMode = 'drain' | 'cancel';
+/** Options of {@link RunController.shutdown}. */
+export interface ShutdownOptions {
+    /** Default `drain`. */
+    mode?: ShutdownMode;
+    /** Max wait for active runs while draining; then cancel-all. `undefined` = wait forever. */
+    drainTimeoutMs?: number;
+    /**
+     * Max wait for cancelled runs (incl. `on_cancel`); then running hooks are
+     * interrupted (resumed by recovery at next start). `undefined` = wait forever.
+     */
+    cancelTimeoutMs?: number;
+    /** Reason recorded on cancelled runs. */
+    reason?: string;
+}
+/** Result of a shutdown. */
+export interface ShutdownResult {
+    /** `drained`: all runs finished normally; `cancelled`: cancel-all was needed. */
+    outcome: 'drained' | 'cancelled';
+    /** Runs still supervised when the stop callback ran (left for recovery). */
+    leftActive: string[];
+}
+/** Report of {@link RunController.recover}. */
+export interface RecoveryReport {
+    /** Runs finalized by recovery, with their final status. */
+    finalized: Array<{
+        runId: string;
+        status: TechnicalStatus;
+        detail: string;
+    }>;
+    /** Runs that are `recovery_pending` after recovery (lock kept). */
+    pending: Array<{
+        runId: string;
+        reason: string;
+    }>;
+    /** Terminal runs whose incomplete action chains were resumed. */
+    resumedActions: string[];
+    /** Unexpected per-run errors (logged; the run is left as is). */
+    errors: Array<{
+        runId: string;
+        message: string;
+    }>;
+}
+/** Dependencies and tuning of the {@link RunController}. */
+export interface RunControllerOptions {
+    repositories: Repositories;
+    /** Lifecycle engine (WI-07); pass `publisher.actionListener()` as its `onActionChange`. */
+    lifecycle: LifecycleEngine;
+    /** Source adapters by configured source ID. */
+    sources: SourceResolver;
+    dataDir: string;
+    /** Shared bus; default: a new one (see {@link RunController.bus}). */
+    bus?: RunEventBus;
+    /** Shared publisher of persisted events; default: a new one over `bus`. */
+    publisher?: PersistedEventPublisher;
+    /** Needed for `execute.secrets`. */
+    credentials?: CredentialProvider;
+    /** Receives `retry_block_created` and `recovery_failed` after commit. Must not block/throw. */
+    notifier?: Notifier;
+    /** Executor per run; default `new ShellExecutor({clock, logger, workspaceBaseDir})`. */
+    createExecutor?: (options: {
+        workspaceBaseDir: string | undefined;
+    }) => Executor;
+    /** `retention.max_log_size_per_run` in bytes (value or getter for the live config). */
+    maxLogBytesPerRun?: number | (() => number | undefined);
+    clock?: Clock;
+    /** Timers for shutdown timeouts and recovery polling. */
+    timers?: Timers;
+    logger?: Logger;
+    /** Recovery: process inspector / signal sender (tests). */
+    processInspector?: ProcessInspector;
+    kill?: KillFn;
+    /** Recovery: "does a process group with this ID exist?" (default `processGroupExists`; tests). */
+    groupExists?: (pgid: number) => boolean;
+    /** Recovery: wait after SIGKILL before giving up (default 5 s). */
+    recoveryKillConfirmMs?: number;
+    /** Recovery: SIGTERM→SIGKILL grace; default the run snapshot's `execute.cancel_grace`. */
+    recoveryGraceMs?: number;
+    /** Max wait for interrupted hooks after a forced stop (default 10 s). */
+    abortSettleMs?: number;
+    /** Called once when a shutdown completed (after drain or cancel-all). */
+    onStop?: (result: ShutdownResult) => void | Promise<void>;
+}
+/** See module docs. One instance per process. */
+export declare class RunController {
+    private readonly options;
+    /** In-process event bus (SSE source for WI-10). */
+    readonly bus: RunEventBus;
+    /** Publisher of persisted events (share with the lifecycle engine). */
+    readonly publisher: PersistedEventPublisher;
+    private readonly repos;
+    private readonly lifecycle;
+    private readonly clock;
+    private readonly timers;
+    private readonly logger;
+    private readonly active;
+    private readonly idleWaiters;
+    /** Woken when a running shutdown is escalated (cancel mode / stricter timeouts). */
+    private readonly shutdownWakeups;
+    private shutdownState;
+    /** Aborted on a forced stop: interrupts completion hooks (resumed by recovery). */
+    private readonly hookAbort;
+    private ready;
+    private paused;
+    private draining;
+    private stopping;
+    private recovering;
+    private shuttingDown;
+    constructor(options: RunControllerOptions);
+    /** Current controller status. */
+    get status(): ControllerStatus;
+    /** Recovery finished (`/ready`). */
+    isReady(): boolean;
+    /** Scheduler gate for automatic starts: ready, not paused, not draining/stopping. */
+    canStartAutomatically(): boolean;
+    /** Gate for manual starts: pause does not apply (Spec §14), drain/stop does. */
+    canStartManually(): boolean;
+    /** Run IDs currently supervised by this process. */
+    activeRunIds(): string[];
+    /** Pause automatic starts (running runs are not touched). */
+    pause(): void;
+    /** Resume automatic starts. */
+    resume(): void;
+    /** True if an active retry block exists for the workflow/item (`itemKey = null` → workflow scope). */
+    isBlocked(workflowId: string, itemKey: string | null): boolean;
+    /** Remove a block: starts a new counting period; runs/history stay (idempotent). */
+    clearBlock(blockId: number, clearedBy?: string | null): BlockRecord;
+    /**
+     * Start supervising a reserved run (from the scheduler or a manual start).
+     * Returns immediately; the run proceeds asynchronously (`handle.done`).
+     *
+     * If the controller does not accept runs (not ready, draining, stopping),
+     * the reserved run is finalized as `cancelled` (never started, not counted,
+     * no hooks) so its lock is released, and a {@link RunControllerError} is thrown.
+     */
+    startRun(run: RunRecord | string, options?: StartRunOptions): RunHandle;
+    /**
+     * Cancel a run: persist `cancelling`, stop the process group (SIGTERM →
+     * grace → SIGKILL → confirmed), run `on_cancel`, end as `cancelled`.
+     * Resolves with the final run. If the group cannot be terminated the run
+     * stays `cancelling`, the failure is persisted and `kill_failed` is thrown
+     * (calling again retries).
+     */
+    cancelRun(runId: string, options?: {
+        reason?: string;
+    }): Promise<RunRecord>;
+    /**
+     * Synchronous variant for the API (HTTP 202/409): validates and persists the
+     * cancel request (`cancelling`) and stops the process in the background.
+     * Throws the same errors as {@link cancelRun}; a later kill failure is only
+     * visible on the run (`run.cancel_failed`, `error.code = kill_failed`).
+     */
+    requestCancel(runId: string, options?: {
+        reason?: string;
+    }): {
+        runId: string;
+        status: TechnicalStatus;
+    };
+    /** Validate + persist a cancel request; `null` if the run was finalized directly (never started). */
+    private beginCancel;
+    /** Drain: no new runs, wait for all active runs, then stop. */
+    drain(): Promise<ShutdownResult>;
+    /** Cancel all runs (with `on_cancel`), then stop. */
+    cancelAllAndShutdown(reason?: string): Promise<ShutdownResult>;
+    /**
+     * Shutdown. The first call starts it; later calls return the same promise
+     * but ESCALATE a running shutdown: `mode: 'cancel'` turns a drain into
+     * cancel-all, and stricter `drainTimeoutMs` / `cancelTimeoutMs` (counted
+     * from the later call) replace looser ones – e.g. API drain followed by
+     * SIGTERM with timeouts, or a second SIGTERM → `mode: 'cancel'`.
+     *
+     * SIGTERM recommendation (WI-10): `shutdown({mode: 'drain', drainTimeoutMs, cancelTimeoutMs})`
+     * – drain with timeout, then cancel-all, then interrupt remaining hooks.
+     * A running recovery is awaited before anything stops.
+     */
+    shutdown(options?: ShutdownOptions): Promise<ShutdownResult>;
+    private doShutdown;
+    private remaining;
+    /**
+     * Recover after a (crashed) previous controller. MUST run before the
+     * scheduler starts; sets `ready` only at the end. Idempotent.
+     */
+    recover(): Promise<RecoveryReport>;
+    /**
+     * Re-attempt the recovery of a `recovery_pending` run (e.g. after the
+     * operator stopped the process). With `assumeGone: true` the operator
+     * confirms the process is gone; nothing is signalled and the run is
+     * finalized (`crashed`, or `cancelled` if it was being cancelled).
+     * An unsupervised `finalizing` run (e.g. after an internal error) resumes
+     * its completion hook and finishes.
+     *
+     * @throws RunControllerError `run_controller_not_ready` during recovery,
+     *   `run_controller_not_accepting` while draining/stopping,
+     *   `run_already_started` if the run is supervised, `run_not_recovery_pending`
+     */
+    retryRecovery(runId: string, options?: {
+        assumeGone?: boolean;
+    }): Promise<RunRecord>;
+    /** Register a recovery job in `active`: drain waits for it, cancel refuses it, no double processing. */
+    private trackRecovery;
+    private doRecover;
+    private recoverRun;
+    /** Runs that may have a worker process: verify/terminate it, then finalize or mark `recovery_pending`. */
+    private recoverProcessRun;
+    private checkProcess;
+    private markRecoveryPending;
+    private resumeFinalizing;
+    private resumeTerminalRunActions;
+    /** Actions left `running` → safe state (WI-07); optionally drop the rest of an interrupted `on_start`. */
+    private makeActionsSafe;
+    private execute;
+    private cancelProcess;
+    /**
+     * `→ finalizing` (target, counter and block in ONE transaction), then the
+     * completion hook, workspace cleanup and `→ <terminal>` (lock released).
+     */
+    private finalize;
+    private beginFinalization;
+    private completeFinalization;
+    /** Finalize a run that never reached `starting`: `cancelled`, not counted, no hooks (synchronous). */
+    private finalizeUnstarted;
+    private cleanupWorkspace;
+    private track;
+    /**
+     * `idle` once no run is supervised, `timeout` after `timeoutMs`, `woken`
+     * (only with `wakeable`) when a running shutdown was escalated.
+     */
+    private waitIdle;
+    /**
+     * Unexpected internal error while supervising a run: never finalize blindly
+     * (the worker may still run). Where the state machine allows it the run
+     * becomes `recovery_pending` (lock kept, error, `recovery_failed`
+     * notification); a `finalizing` run stays as is. The entry stays supervised
+     * until a started worker process ended; then `retryRecovery` resolves the run.
+     */
+    private onUnexpected;
+    private appendEvent;
+    private recordRecovered;
+    private reachedStarting;
+    private spawnAttempted;
+    private groupExists;
+    private maxLogBytes;
+    private notify;
+    private publishStatus;
+}