@inflexa-ai/harness 0.45.1 → 0.46.0

package/dist/workflows/pause.d.ts

@@ -0,0 +1,71 @@
+/**
+ * The pause of durable work (work-pause spec).
+ *
+ * A failure that a person or time can repair does not fail the work. The workflow that meets it waits in its own
+ * body with `DBOS.recv`, on the topic of its pause, and the work goes on when the wait ends. The workflow stays
+ * active, thus a recovery after a crash replays it to the same wait: the failed call, the row, and the part of the
+ * pause come back from the checkpoints. A `recv` that completed replays its message, thus a resume never runs
+ * twice.
+ *
+ * A wait in `retrying` ends at its deadline, and the work sends the request again. A wait in `paused` ends only
+ * with the message of a resume, or with a stop. The server never sends that message by itself.
+ */
+import type { RunSession } from "../auth/types.js";
+import type { PauseCause, WorkKind, PausePart } from "../contracts/chat-parts.js";
+import type { RunAuthorizer } from "../execution/run-authorizer.js";
+import type { Logger } from "../lib/logger.js";
+import type { Querier } from "../state/db.js";
+/** The delays of the timer of an outage: 1, 5, 15, and 30 minutes. After the last one, the work waits in `paused`. */
+export declare const DEFAULT_PAUSE_RETRY_DELAYS_MS: readonly number[];
+/** The message that ends a wait. */
+export declare const RESUME_MESSAGE: {
+    readonly kind: "resume";
+};
+/** The message that moves a wait in `retrying` to `paused`: the work waits for a resume, and the timer stops. */
+export declare const HOLD_MESSAGE: {
+    readonly kind: "hold";
+};
+/** The topic of the resume message of one pause. Each pause has its own topic, thus a late message never ends a later pause. */
+export declare function pauseTopic(pauseId: string): string;
+/** The cause of a pause in a failure, or `undefined` for a failure that no person or time can repair. */
+export declare function pauseCauseOf(failure: unknown): PauseCause | undefined;
+/** How a wait ended: the deadline of `retrying`, the message of a resume, or a stop of the work. */
+export type PauseEnd = "retry" | "resume" | "abort";
+/** The work that a pause belongs to, and where the part of the pause goes. */
+export type PauseScope = {
+    readonly pool: Querier;
+    /** The turn id, the run id, or the id of the profile workflow. A sub-agent names its turn, and a step its run. */
+    readonly workId: string;
+    readonly workKind: WorkKind;
+    readonly analysisId: string;
+    readonly threadId?: string;
+    /** Writes the part of the pause to the stream of the work. */
+    readonly emit: (part: PausePart) => Promise<void>;
+    /** The delays of the timer of an outage. Absent: {@link DEFAULT_PAUSE_RETRY_DELAYS_MS}. */
+    readonly retryDelaysMs?: readonly number[];
+    /**
+     * The run credential of a run or a profile. A wait for a person suspends it, and the row keeps the run session
+     * for the renewal at the resume. Absent for a turn, whose credential comes with the person who resumes it.
+     */
+    readonly run?: {
+        readonly session: RunSession;
+        readonly authorizer: RunAuthorizer;
+    };
+    readonly logger?: Logger;
+};
+export type WorkPause = {
+    /**
+     * Wait out a failure. `attempt` is the count of timer retries of this pause so far. `undefined`: the failure has
+     * no cause, or the code runs where no wait is possible, outside a workflow or inside a step.
+     */
+    readonly afterFailure: (failure: unknown, attempt: number, signal?: AbortSignal) => Promise<PauseEnd | undefined>;
+    /** Wait out a cause, for example a gate refusal with the suspend flag. `undefined` where no wait is possible. */
+    readonly forCause: (cause: PauseCause, attempt: number, signal?: AbortSignal) => Promise<PauseEnd | undefined>;
+};
+/** The pause of the work of one workflow body. */
+export declare function workPause(scope: PauseScope): WorkPause;
+/**
+ * Wait out a gate refusal with the suspend flag, in the body of a workflow. The body calls the gate again after the
+ * wait. A gate inside a workflow body can always wait, thus no wait is a defect of the call site.
+ */
+export declare function pauseAtGate(pause: WorkPause, reason: string, signal?: AbortSignal): Promise<PauseEnd>;

package/dist/workflows/pause.js

@@ -0,0 +1,173 @@
+/**
+ * The pause of durable work (work-pause spec).
+ *
+ * A failure that a person or time can repair does not fail the work. The workflow that meets it waits in its own
+ * body with `DBOS.recv`, on the topic of its pause, and the work goes on when the wait ends. The workflow stays
+ * active, thus a recovery after a crash replays it to the same wait: the failed call, the row, and the part of the
+ * pause come back from the checkpoints. A `recv` that completed replays its message, thus a resume never runs
+ * twice.
+ *
+ * A wait in `retrying` ends at its deadline, and the work sends the request again. A wait in `paused` ends only
+ * with the message of a resume, or with a stop. The server never sends that message by itself.
+ */
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { z } from "zod";
+import { createNoopLogger } from "../lib/console-logger.js";
+import { deliverNotice } from "../lib/hooks.js";
+import { unwrapOrThrow } from "../lib/result.js";
+import { isTurnCredentialMissing } from "../loop/turn-credentials.js";
+import { modelFailureOf } from "../providers/errors.js";
+import { clearPause, recordPause } from "../state/paused-work.js";
+/** The delays of the timer of an outage: 1, 5, 15, and 30 minutes. After the last one, the work waits in `paused`. */
+export const DEFAULT_PAUSE_RETRY_DELAYS_MS = [60_000, 300_000, 900_000, 1_800_000];
+/**
+ * One wait of the state `paused`, in seconds. The wait loops on it. A stop in the process of the turn leaves its
+ * `recv` behind, and that `recv` polls no longer than this.
+ */
+const PAUSED_WAIT_SECONDS = 60 * 60;
+/** The message that ends a wait. */
+export const RESUME_MESSAGE = { kind: "resume" };
+/** The message that moves a wait in `retrying` to `paused`: the work waits for a resume, and the timer stops. */
+export const HOLD_MESSAGE = { kind: "hold" };
+const ResumeMessageSchema = z.object({ kind: z.literal("resume") });
+const HoldMessageSchema = z.object({ kind: z.literal("hold") });
+/** The topic of the resume message of one pause. Each pause has its own topic, thus a late message never ends a later pause. */
+export function pauseTopic(pauseId) {
+    return `pause:${pauseId}`;
+}
+/** The cause of a pause in a failure, or `undefined` for a failure that no person or time can repair. */
+export function pauseCauseOf(failure) {
+    if (isTurnCredentialMissing(failure))
+        return { kind: "credential" };
+    const model = modelFailureOf(failure);
+    switch (model.kind) {
+        case "billing":
+            return { kind: "host", reason: model.reason };
+        case "credential":
+            return { kind: "credential" };
+        case "outage":
+            return { kind: "outage" };
+        case "other":
+            return undefined;
+    }
+}
+const STOPPED = Symbol("stopped");
+/** Settles when the signal aborts, and never when there is no signal. */
+function stopOf(signal) {
+    return new Promise((resolve) => {
+        if (signal === undefined)
+            return;
+        if (signal.aborted)
+            resolve(STOPPED);
+        else
+            signal.addEventListener("abort", () => resolve(STOPPED), { once: true });
+    });
+}
+async function waitForMessage(topic, delayMs, signal) {
+    const stopped = stopOf(signal);
+    if (delayMs !== undefined) {
+        // DBOS records the deadline of the timeout at the first call, thus a recovery waits until the same time.
+        const message = await Promise.race([DBOS.recv(topic, Math.ceil(delayMs / 1000)), stopped]);
+        if (message === STOPPED)
+            return "abort";
+        if (HoldMessageSchema.safeParse(message).success)
+            return "hold";
+        return ResumeMessageSchema.safeParse(message).success ? "resume" : "retry";
+    }
+    for (;;) {
+        const message = await Promise.race([DBOS.recv(topic, PAUSED_WAIT_SECONDS), stopped]);
+        if (message === STOPPED)
+            return "abort";
+        if (ResumeMessageSchema.safeParse(message).success)
+            return "resume";
+    }
+}
+/** The pause of the work of one workflow body. */
+export function workPause(scope) {
+    const log = (scope.logger ?? createNoopLogger()).named("pause");
+    const delays = scope.retryDelaysMs ?? DEFAULT_PAUSE_RETRY_DELAYS_MS;
+    const forCause = async (cause, attempt, signal) => {
+        const workflowId = DBOS.workflowID;
+        if (!DBOS.isInWorkflow() || DBOS.isInStep() || workflowId === undefined)
+            return undefined;
+        if (signal?.aborted === true)
+            return "abort";
+        const delayMs = cause.kind === "outage" ? delays[attempt] : undefined;
+        const state = delayMs === undefined ? "paused" : "retrying";
+        const pauseId = await DBOS.randomUUID();
+        const pausedAt = await DBOS.now();
+        const retryAt = delayMs === undefined ? undefined : new Date(pausedAt + delayMs).toISOString();
+        const topic = pauseTopic(pauseId);
+        const work = { workId: scope.workId, workKind: scope.workKind };
+        await DBOS.runStep(async () => unwrapOrThrow(await recordPause(scope.pool, {
+            workflowId,
+            ...work,
+            analysisId: scope.analysisId,
+            threadId: scope.threadId,
+            state,
+            cause,
+            attempt,
+            retryAt,
+            topic,
+            pausedAt: new Date(pausedAt).toISOString(),
+            runSession: scope.run?.session,
+        })), { name: "pause.record" });
+        // Only a person ends a wait in `paused`, thus the credential stops until the renewal of the resume.
+        const suspendCredential = async (stepName) => {
+            const run = scope.run;
+            if (run === undefined)
+                return;
+            await DBOS.runStep(() => deliverNotice(log.with({ workflowId }), "RunAuthorizer.suspend", run.authorizer.suspend(run.session, `paused:${cause.kind}`)), { name: stepName });
+        };
+        if (state === "paused")
+            await suspendCredential("pause.suspend");
+        await scope.emit({ type: "data-pause", id: workflowId, ...work, state, cause, attempt, retryAt });
+        log.warn("work paused", { workflowId, ...work, state, cause: cause.kind, attempt });
+        const waited = await waitForMessage(topic, delayMs, signal);
+        let end;
+        if (waited === "hold") {
+            // A person held the work: the timer stops, and the work waits for a resume.
+            await DBOS.runStep(async () => unwrapOrThrow(await recordPause(scope.pool, {
+                workflowId,
+                ...work,
+                analysisId: scope.analysisId,
+                threadId: scope.threadId,
+                state: "paused",
+                cause,
+                attempt,
+                topic,
+                pausedAt: new Date(pausedAt).toISOString(),
+                runSession: scope.run?.session,
+            })), { name: "pause.hold" });
+            await suspendCredential("pause.hold.suspend");
+            await scope.emit({ type: "data-pause", id: workflowId, ...work, state: "paused", cause, attempt });
+            log.info("work held", { workflowId, ...work, attempt });
+            const held = await waitForMessage(topic, undefined, signal);
+            end = held === "hold" ? "resume" : held;
+        }
+        else {
+            end = waited;
+        }
+        await DBOS.runStep(async () => unwrapOrThrow(await clearPause(scope.pool, workflowId)), { name: "pause.clear" });
+        await scope.emit({ type: "data-pause", id: workflowId, ...work, state: "running" });
+        log.info("pause ended", { workflowId, ...work, end });
+        return end;
+    };
+    return {
+        afterFailure: async (failure, attempt, signal) => {
+            const cause = pauseCauseOf(failure);
+            return cause === undefined ? undefined : forCause(cause, attempt, signal);
+        },
+        forCause,
+    };
+}
+/**
+ * Wait out a gate refusal with the suspend flag, in the body of a workflow. The body calls the gate again after the
+ * wait. A gate inside a workflow body can always wait, thus no wait is a defect of the call site.
+ */
+export async function pauseAtGate(pause, reason, signal) {
+    const end = await pause.forCause({ kind: "host", reason }, 0, signal);
+    if (end === undefined)
+        throw new Error("A gate refusal with the suspend flag cannot wait outside the body of a workflow.");
+    return end;
+}

package/dist/workflows/sandbox-step.d.ts

@@ -7,10 +7,10 @@
  *
  *  1. `mark-running`        — update `cortex_step_executions` to running.
  *  2. `createSandbox`       — provision the sandbox; persist its handle.
- *  3. `runAgent`            — the harness agent loop. Every LLM and tool
- *                             call inside is its own durableStep.
- *  4. `generateFileMetadata`— the file-metadata continuation of the step agent.
- *  5. `generateStepSummary` — the summary continuation of the step agent,
+ *  3. `runLoop`             — the harness agent loop. Every model call and
+ *                             tool call inside is its own durable step.
+ *  4. `generateFileMetadata`— the file-metadata follow-up request of the step agent.
+ *  5. `generateStepSummary` — the summary follow-up request of the step agent,
  *                             producing `output/summary.md`.
  *  6. `reconcile + register`— write artifact manifest entries to cortex_artifacts.
  *  7. `syncArtifacts`       — register provenance + upload missing file_ids
@@ -20,22 +20,22 @@
  * 10. `mark-terminal`       — write the final status, emit `data-step-activity`
  *                             onto the parent's DBOS stream.
  *
- * The parent owns sibling lifecycle: the halt cascade of a suspension lives
- * on the parent (`executeAnalysis`). This child owns its own teardown and its
- * own suspension (workflow-suspension spec): a `suspend` error of the agent
- * loop, or a refusal with the suspend flag of the label gate or the register
- * gate. The child records the reason on its row, sends the typed suspension on
- * `CHILD_SUSPENDED_TOPIC`, and self-cancels to CANCELLED (resumable, not ERROR).
+ * The parent owns sibling lifecycle. This child owns its own teardown and its
+ * own pause (work-pause spec): a failed model request of the agent loop with a
+ * cause of a pause, or a refusal with the suspend flag of the label gate or the
+ * register gate. The child waits in place, and the run row stays `running`. The
+ * other steps go on, and each one pauses at its own failure.
  */
 import type { TokenUsageRollup } from "@inflexa-ai/harness/contracts/usage.js";
-import type { Pool } from "pg";
+import type { Querier } from "../state/db.js";
 import type { Logger } from "../lib/logger.js";
 import type { UsageRecorder } from "../billing/usage-recorder.js";
 import type { CitationResolver } from "../citations/types.js";
 import type { AgentDefinition, LoopMessage } from "../loop/types.js";
 import { type ToolOutputStore } from "../loop/tool-output.js";
-import type { AgentChat, EmbeddingProvider } from "../providers/types.js";
+import type { ChatProvider, EmbeddingProvider } from "../providers/types.js";
 import { type RunSession, type SpawnSession } from "../auth/types.js";
+import type { RunAuthorizer } from "../execution/run-authorizer.js";
 import type { FileMetadataEntry } from "../execution/artifact-metadata.js";
 import type { ArtifactManifestEntry } from "../schemas/artifact-manifest.js";
 import type { StepSummary } from "../schemas/step-summary.js";
@@ -47,14 +47,13 @@ import { type StepFileMetadata } from "../execution/post-step-pipeline.js";
 import type { ResourceSpec } from "../config/resource-limits.js";
 import type { SandboxRef } from "../sandbox/types.js";
 import { ProvenanceCollector } from "../provenance/collector.js";
-import { type BlockerHolder } from "../tools/sandbox/report-blocker.js";
 import { type FileMetadataCell } from "../tools/sandbox/submit-file-metadata.js";
 import type { TemplateBinding } from "../tools/knowledge/template.js";
 /**
  * Input the parent hands the child. Must be JSON-serialisable — DBOS
  * persists this as the workflow's input row.
  */
-export interface SandboxStepInput {
+export type SandboxStepInput = {
     readonly analysisId: string;
     readonly runId: string;
     readonly stepId: string;
@@ -86,15 +85,6 @@ export interface SandboxStepInput {
     readonly level: number;
     /** User-content prompt the agent receives as its initial message. */
     readonly prompt: string;
-    /**
-     * DBOS workflow id of the parent (`executeAnalysis`). The child uses
-     * this to address the parent for its typed suspension
-     * (`DBOS.send(parent, …, CHILD_SUSPENDED_TOPIC)`) — the parent's
-     * scheduler-loop classifier relies on it to disambiguate a suspension
-     * self-cancel from an operator-driven sibling cancel (CANCELLED is the
-     * same terminal state for both).
-     */
-    readonly parentWorkflowId: string;
     /** Sandbox image override; falls back to the child's configured default. */
     readonly image?: string;
     /** Per-step env overrides forwarded to sandbox-server. */
@@ -115,7 +105,7 @@ export interface SandboxStepInput {
      * from `cortex_runs`.
      */
     readonly runSession: SpawnSession;
-}
+};
 /**
  * Fallback when the plan step omits `timeout`. Plans that need 3-hour runs
  * declare it explicitly.
@@ -128,21 +118,7 @@ export declare const DEFAULT_STEP_TIMEOUT_SECONDS = 3600;
  */
 export declare const CAPPED_OUT_EMPTY_REASON = "The step hit its iteration cap with no output files persisted.";
 export type SandboxStepStatus = "complete" | "failed" | "canceled" | "blocked";
-/**
- * DBOS message topic the child uses to tell the parent that it suspends. The
- * message goes BEFORE `DBOS.cancelWorkflow(self)`, thus the parent's recv
- * accumulator records the child before the child's terminal state lands.
- * Without it the parent observes only `DBOSWorkflowCancelledError` from
- * `getResult`, and it cannot tell a suspension from a cancel.
- */
-export declare const CHILD_SUSPENDED_TOPIC = "child-suspended";
-export interface ChildSuspended {
-    readonly kind: "suspended";
-    readonly childWorkflowId: string;
-    readonly stepId: string;
-    readonly reason: string;
-}
-export interface SandboxStepResult {
+export type SandboxStepResult = {
     readonly status: SandboxStepStatus;
     readonly durationMs: number;
     readonly finishReason: string | null;
@@ -158,27 +134,27 @@ export interface SandboxStepResult {
      * result shape recovers without it.
      */
     readonly usage?: TokenUsageRollup;
-}
+};
 /** One file in the step's terminal file tree / output listing. */
-export interface StepFileEntry {
+export type StepFileEntry = {
     readonly path: string;
     readonly size: number;
     /** Schema-conformant `StepOutputFile.fileType`. */
     readonly fileType: "script" | "output" | "figure" | "log" | "notebook" | "summary";
     readonly description: string;
-}
+};
 /**
  * Post-step detail the body emits to the run stream once the bookkeeping
  * helpers have run. Derived from the metadata entries, the generated step
  * summary, and the reconciled artifact manifest. `collectStepOutputs`
  * assembles it from the same stash the vector-index step consumes.
  */
-export interface StepOutputs {
+export type StepOutputs = {
     /** Reconciled artifact files with category + description + size. */
     readonly files: readonly StepFileEntry[];
     /** Generated step summary markdown (empty when none produced). */
     readonly summaryMarkdown: string;
-}
+};
 /**
  * The typed post-step products threaded through the workflow body: file
  * descriptions, the interpretive summary, and the reconciled (phantom-dropped)
@@ -187,17 +163,17 @@ export interface StepOutputs {
  * composition, so the producer→consumer ordering is a type constraint rather
  * than an implicit contract over shared mutable state.
  */
-export interface PostStepArtifacts {
+export type PostStepArtifacts = {
     readonly metadataEntries: readonly FileMetadataEntry[];
     readonly summary: StepSummary | undefined;
     readonly reconciledManifest: readonly ArtifactManifestEntry[];
-}
+};
 /**
  * Per-step coordinates the workflow body computes once and threads to the
  * deps factory. The parent supplies the durable inputs; the factory
  * supplies the live sandbox ref and the deadline.
  */
-export interface SandboxAgentBuildContext {
+export type SandboxAgentBuildContext = {
     readonly input: SandboxStepInput;
     readonly session: RunSession;
     readonly sandbox: SandboxRef;
@@ -210,11 +186,8 @@ export interface SandboxAgentBuildContext {
      * input/script edges). Same instance on both paths.
      */
     readonly lineageCollector: ProvenanceCollector;
-    /**
-     * Per-run blocker cell that `report_blocker` writes. The body prefers the
-     * blocker read from the transcript and falls back to this cell.
-     */
-    readonly blockerHolder: BlockerHolder;
+    /** The step agent declares `report_blocker`, and the body selects the blocker from the recorded calls. */
+    readonly reportsBlocker: true;
     /**
      * Cell for the file-metadata continuation. `submit_file_metadata` is
      * declared up front for prefix stability but masked until then.
@@ -224,15 +197,15 @@ export interface SandboxAgentBuildContext {
     readonly citationResolver: CitationResolver;
     /** Give it to `SandboxAgentDeps.toolOutputStore`: each loop of the step keeps its texts in it. */
     readonly toolOutputStore?: ToolOutputStore;
-}
+};
 /**
  * Inversion of control: the parent supplies the live agent, the parent's
  * stream emitter, and the post-step helpers. Keeping the workflow body
  * free of per-agent imports means it can drive every agent in the catalog
  * without becoming a switch on agentId.
  */
-export interface SandboxStepDeps {
-    readonly pool: Pool;
+export type SandboxStepDeps = {
+    readonly pool: Querier;
     /**
      * Operational logging seam. Omit it and the body falls back to a no-op —
      * but a failed step's cause is written here and NOWHERE else (`failStep`
@@ -250,8 +223,8 @@ export interface SandboxStepDeps {
     readonly toolOutputStore?: ToolOutputStore;
     /** Shared resolver threaded from `assembleCoreRuntime`. */
     readonly citationResolver: CitationResolver;
-    /** Non-streaming chat — drives the agent loop + the post-step continuations. */
-    readonly provider: AgentChat;
+    /** Drives the agent loop and the post-step follow-up requests. */
+    readonly provider: ChatProvider;
     /** Write-side embedder for the post-step vector index. */
     readonly embedding: EmbeddingProvider;
     readonly sandboxClient: SandboxClient;
@@ -261,6 +234,10 @@ export interface SandboxStepDeps {
      * cloud-free build, external provenance ledger in the managed build).
      */
     readonly artifactRegistry: ArtifactRegistry;
+    /** The delays of the timer of an outage. Absent: the default of the pause. */
+    readonly pauseRetryDelaysMs?: readonly number[];
+    /** The run authorizer of the runtime. A pause of the step for a person suspends the run credential through it. */
+    readonly runAuthorizer: RunAuthorizer;
     /**
      * No longer read by the step body; kept because `src/index.ts` exports
      * this type and an embedder builds it as an object literal.
@@ -273,7 +250,7 @@ export interface SandboxStepDeps {
     /**
      * Build the agent definition for this step. The parent resolves the
      * agent from the catalog, wires its workspace tools, and returns the
-     * configured `AgentDefinition` runAgent will drive.
+     * configured `AgentDefinition` that `runLoop` will drive.
      */
     buildAgent(ctx: SandboxAgentBuildContext): Promise<AgentDefinition>;
     /**
@@ -281,7 +258,7 @@ export interface SandboxStepDeps {
      * workspace mutate tools enforce write confinement against this path.
      */
     resolveWritePrefix(input: SandboxStepInput): Promise<string>;
-}
+};
 /**
  * Per-step context passed to every post-step dep. Carries the durable input
  * (analysisId/runId/stepId/agentId), the durable `RunSession`, the in-memory
@@ -290,7 +267,7 @@ export interface SandboxStepDeps {
  * `lineageCollector` carries the runtime-observed input/script edges the
  * step loop accumulated, which registration translates into managed-root parents.
  */
-export interface PostStepContext {
+export type PostStepContext = {
     readonly input: SandboxStepInput;
     readonly session: RunSession;
     readonly transcript: readonly LoopMessage[];
@@ -304,7 +281,7 @@ export interface PostStepContext {
     readonly sandboxId: string;
     /** Same step-scoped collector the agent loop fed each exec frame into. */
     readonly lineageCollector: ProvenanceCollector;
-}
+};
 /** Normalizes a pre-`messages` checkpoint (a bare entries array) for replay. */
 export declare function readStepFileMetadata(checkpoint: StepFileMetadata | readonly FileMetadataEntry[]): StepFileMetadata;
 /**