@inflexa-ai/harness 0.37.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/auth/local-run-authorizer.js +5 -4
- package/dist/auth/types.d.ts +28 -9
- package/dist/auth/types.js +5 -3
- package/dist/billing/cached-resolver.d.ts +11 -7
- package/dist/billing/cached-resolver.js +17 -11
- package/dist/billing/noop-run-charge.js +3 -2
- package/dist/billing/noop-usage-recorder.js +2 -1
- package/dist/billing/run-charge.d.ts +18 -5
- package/dist/billing/run-charge.js +5 -1
- package/dist/billing/usage-recorder.d.ts +11 -9
- package/dist/billing/usage-recorder.js +2 -3
- package/dist/contracts/chat-events.d.ts +4 -1
- package/dist/contracts/chat-parts.d.ts +1 -1
- package/dist/contracts/schemas/chat-parts.d.ts +4 -4
- package/dist/execution/artifact-registration.d.ts +6 -1
- package/dist/execution/artifact-registration.js +12 -4
- package/dist/execution/artifact-registry.d.ts +10 -3
- package/dist/execution/noop-artifact-registry.js +3 -4
- package/dist/execution/post-step-pipeline.d.ts +14 -3
- package/dist/execution/post-step-pipeline.js +21 -8
- package/dist/execution/run-authorizer.d.ts +5 -3
- package/dist/execution/run-canceler.js +4 -20
- package/dist/index.d.ts +14 -8
- package/dist/index.js +12 -6
- package/dist/lib/hooks.d.ts +49 -0
- package/dist/lib/hooks.js +39 -0
- package/dist/lib/result.d.ts +22 -11
- package/dist/lib/result.js +24 -14
- package/dist/loop/run-agent.d.ts +2 -2
- package/dist/loop/run-agent.js +5 -4
- package/dist/provenance/seam.d.ts +1 -1
- package/dist/providers/ai-sdk.d.ts +43 -5
- package/dist/providers/ai-sdk.js +55 -52
- package/dist/providers/anthropic.d.ts +2 -3
- package/dist/providers/anthropic.js +2 -2
- package/dist/providers/embedding.d.ts +13 -7
- package/dist/providers/embedding.js +26 -6
- package/dist/providers/errors.d.ts +67 -37
- package/dist/providers/errors.js +57 -58
- package/dist/providers/request-headers.d.ts +18 -0
- package/dist/providers/request-headers.js +22 -0
- package/dist/report-model/production-resolver.d.ts +5 -2
- package/dist/report-model/production-resolver.js +12 -2
- package/dist/runtime/assemble.d.ts +3 -2
- package/dist/runtime/dbos.d.ts +4 -3
- package/dist/runtime/dbos.js +18 -3
- package/dist/runtime/result-serialization.d.ts +29 -0
- package/dist/runtime/result-serialization.js +45 -0
- package/dist/sandbox/client.d.ts +11 -3
- package/dist/sandbox/create-sandbox.d.ts +5 -2
- package/dist/sandbox/create-sandbox.js +51 -25
- package/dist/sandbox/docker-client.d.ts +4 -3
- package/dist/sandbox/docker-client.js +23 -31
- package/dist/sandbox/k8s-client.d.ts +4 -13
- package/dist/sandbox/k8s-client.js +28 -61
- package/dist/sandbox/labels.d.ts +34 -0
- package/dist/sandbox/labels.js +47 -0
- package/dist/sandbox/mount-plan.d.ts +3 -1
- package/dist/sandbox/mount-plan.js +9 -0
- package/dist/sandbox/sandbox-error.d.ts +36 -13
- package/dist/sandbox/sandbox-error.js +32 -17
- package/dist/sandbox/types.d.ts +18 -10
- package/dist/schemas/validate-plan.js +13 -0
- package/dist/schemas/workflow-state.d.ts +4 -4
- package/dist/state/analyses.d.ts +4 -4
- package/dist/state/analyses.js +4 -4
- package/dist/state/schema.d.ts +5 -5
- package/dist/tasks/data-profile.d.ts +5 -8
- package/dist/tasks/data-profile.js +88 -43
- package/dist/tasks/derive-table-exec.d.ts +10 -6
- package/dist/tasks/derive-table-exec.js +17 -12
- package/dist/tasks/extract-values.d.ts +16 -10
- package/dist/tasks/extract-values.js +34 -18
- package/dist/tools/execute-analysis.js +29 -24
- package/dist/tools/report-session/derive-table.d.ts +9 -1
- package/dist/tools/report-session/derive-table.js +25 -22
- package/dist/tools/report-session/examine-page.d.ts +6 -3
- package/dist/tools/report-session/examine-page.js +16 -8
- package/dist/tools/report-session/index.d.ts +1 -1
- package/dist/tools/report-session/preview-report.js +11 -9
- package/dist/tools/report-session/session-page-publisher.d.ts +17 -21
- package/dist/tools/report-session/session-page-publisher.js +5 -7
- package/dist/tools/research/inspect-run.d.ts +1 -1
- package/dist/tools/sandbox/link-packages.d.ts +8 -0
- package/dist/tools/sandbox/link-packages.js +19 -9
- package/dist/workflows/execute-analysis.d.ts +25 -16
- package/dist/workflows/execute-analysis.js +223 -216
- package/dist/workflows/metrics.d.ts +4 -3
- package/dist/workflows/metrics.js +4 -3
- package/dist/workflows/sandbox-step.d.ts +20 -29
- package/dist/workflows/sandbox-step.js +71 -127
- package/dist/workflows/suspension.d.ts +32 -0
- package/dist/workflows/suspension.js +38 -0
- package/package.json +1 -1
- package/dist/billing/noop-resolver.d.ts +0 -6
- package/dist/billing/noop-resolver.js +0 -7
- package/dist/billing/resolver.d.ts +0 -64
- package/dist/billing/resolver.js +0 -24
- package/dist/loop/budget-exceeded.d.ts +0 -38
- package/dist/loop/budget-exceeded.js +0 -88
|
@@ -12,11 +12,12 @@
|
|
|
12
12
|
/**
|
|
13
13
|
* Record one child cancellation. The parent calls this once per
|
|
14
14
|
* `DBOS.cancelWorkflow(childWorkflowId)` in the cascade (fail-fast OR
|
|
15
|
-
*
|
|
16
|
-
*
|
|
15
|
+
* suspension). `cause` is `fail_fast`, `external_cancel`, or the reason of
|
|
16
|
+
* the host for a suspension, carried unread, so dashboards can split the
|
|
17
|
+
* counter by the reason that the host gave.
|
|
17
18
|
*/
|
|
18
19
|
export declare function recordCancelledChild(args: {
|
|
19
|
-
readonly cause:
|
|
20
|
+
readonly cause: string;
|
|
20
21
|
}): void;
|
|
21
22
|
/** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
|
|
22
23
|
export declare function __resetWorkflowMetricsForTest(): void;
|
|
@@ -16,7 +16,7 @@ function getInstruments() {
|
|
|
16
16
|
const meter = metrics.getMeter("cortex.workflow");
|
|
17
17
|
instruments = {
|
|
18
18
|
cancelledChildren: meter.createCounter("cortex.workflow.parent.cancelled_children", {
|
|
19
|
-
description: "Children cancelled by the parent's fail-fast or
|
|
19
|
+
description: "Children cancelled by the parent's fail-fast or suspension cascade",
|
|
20
20
|
}),
|
|
21
21
|
};
|
|
22
22
|
}
|
|
@@ -25,8 +25,9 @@ function getInstruments() {
|
|
|
25
25
|
/**
|
|
26
26
|
* Record one child cancellation. The parent calls this once per
|
|
27
27
|
* `DBOS.cancelWorkflow(childWorkflowId)` in the cascade (fail-fast OR
|
|
28
|
-
*
|
|
29
|
-
*
|
|
28
|
+
* suspension). `cause` is `fail_fast`, `external_cancel`, or the reason of
|
|
29
|
+
* the host for a suspension, carried unread, so dashboards can split the
|
|
30
|
+
* counter by the reason that the host gave.
|
|
30
31
|
*/
|
|
31
32
|
export function recordCancelledChild(args) {
|
|
32
33
|
getInstruments().cancelledChildren.add(1, { cause: args.cause });
|
|
@@ -9,9 +9,6 @@
|
|
|
9
9
|
* 2. `createSandbox` — provision the sandbox; persist its handle.
|
|
10
10
|
* 3. `runAgent` — the harness agent loop. Every LLM and tool
|
|
11
11
|
* call inside is its own durableStep.
|
|
12
|
-
* Catches `isBudgetExceeded` and self-cancels
|
|
13
|
-
* via `DBOS.cancelWorkflow(self)` to CANCELLED
|
|
14
|
-
* (resumable, not ERROR).
|
|
15
12
|
* 4. `generateFileMetadata`— post-step LLM call (chunked, slice-to-length).
|
|
16
13
|
* 5. `generateStepSummary` — post-step LLM call producing `output/summary.md`.
|
|
17
14
|
* 6. `reconcile + register`— write artifact manifest entries to cortex_artifacts.
|
|
@@ -22,9 +19,12 @@
|
|
|
22
19
|
* 10. `mark-terminal` — write the final status, emit `data-step-activity`
|
|
23
20
|
* onto the parent's DBOS stream.
|
|
24
21
|
*
|
|
25
|
-
* The parent owns sibling lifecycle:
|
|
26
|
-
* parent (`executeAnalysis`). This child
|
|
27
|
-
*
|
|
22
|
+
* The parent owns sibling lifecycle: the halt cascade of a suspension lives
|
|
23
|
+
* on the parent (`executeAnalysis`). This child owns its own teardown and its
|
|
24
|
+
* own suspension (workflow-suspension spec): a `suspend` error of the agent
|
|
25
|
+
* loop, or a refusal with the suspend flag of the label gate or the register
|
|
26
|
+
* gate. The child records the reason on its row, sends the typed suspension on
|
|
27
|
+
* `CHILD_SUSPENDED_TOPIC`, and self-cancels to CANCELLED (resumable, not ERROR).
|
|
28
28
|
*/
|
|
29
29
|
import type { TokenUsageRollup } from "@inflexa-ai/harness/contracts/usage.js";
|
|
30
30
|
import type { Pool } from "pg";
|
|
@@ -33,7 +33,7 @@ import type { UsageRecorder } from "../billing/usage-recorder.js";
|
|
|
33
33
|
import type { CitationResolver } from "../citations/types.js";
|
|
34
34
|
import type { AgentDefinition, LoopMessage } from "../loop/types.js";
|
|
35
35
|
import type { AgentChat, EmbeddingProvider } from "../providers/types.js";
|
|
36
|
-
import { type RunSession } from "../auth/types.js";
|
|
36
|
+
import { type RunSession, type SpawnSession } from "../auth/types.js";
|
|
37
37
|
import type { FileMetadataEntry } from "../execution/artifact-metadata.js";
|
|
38
38
|
import type { ArtifactManifestEntry } from "../schemas/artifact-manifest.js";
|
|
39
39
|
import type { StepSummary } from "../schemas/step-summary.js";
|
|
@@ -72,9 +72,9 @@ export interface SandboxStepInput {
|
|
|
72
72
|
readonly prompt: string;
|
|
73
73
|
/**
|
|
74
74
|
* DBOS workflow id of the parent (`executeAnalysis`). The child uses
|
|
75
|
-
* this to address the parent for
|
|
76
|
-
* (`DBOS.send(parent, …,
|
|
77
|
-
* scheduler-loop classifier relies on it to disambiguate a
|
|
75
|
+
* this to address the parent for its typed suspension
|
|
76
|
+
* (`DBOS.send(parent, …, CHILD_SUSPENDED_TOPIC)`) — the parent's
|
|
77
|
+
* scheduler-loop classifier relies on it to disambiguate a suspension
|
|
78
78
|
* self-cancel from an operator-driven sibling cancel (CANCELLED is the
|
|
79
79
|
* same terminal state for both).
|
|
80
80
|
*/
|
|
@@ -99,7 +99,7 @@ export interface SandboxStepInput {
|
|
|
99
99
|
* DBOS replay reconstructs it on resume — the body never reads the JWT
|
|
100
100
|
* from `cortex_runs`.
|
|
101
101
|
*/
|
|
102
|
-
readonly runSession:
|
|
102
|
+
readonly runSession: SpawnSession;
|
|
103
103
|
}
|
|
104
104
|
/**
|
|
105
105
|
* Fallback when the plan step omits `timeout`. Plans that need 3-hour runs
|
|
@@ -114,20 +114,18 @@ export declare const DEFAULT_STEP_TIMEOUT_SECONDS = 3600;
|
|
|
114
114
|
export declare const CAPPED_OUT_EMPTY_REASON = "The step hit its iteration cap with no output files persisted.";
|
|
115
115
|
export type SandboxStepStatus = "complete" | "failed" | "canceled" | "blocked";
|
|
116
116
|
/**
|
|
117
|
-
* DBOS message topic the child uses to
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
* `DBOSWorkflowCancelledError` from `getResult` — a generic message that
|
|
123
|
-
* `isBudgetExceeded` does NOT match — and misclassifies the failure as
|
|
124
|
-
* fail-fast, skipping the 402 pause cascade.
|
|
117
|
+
* DBOS message topic the child uses to tell the parent that it suspends. The
|
|
118
|
+
* message goes BEFORE `DBOS.cancelWorkflow(self)`, thus the parent's recv
|
|
119
|
+
* accumulator records the child before the child's terminal state lands.
|
|
120
|
+
* Without it the parent observes only `DBOSWorkflowCancelledError` from
|
|
121
|
+
* `getResult`, and it cannot tell a suspension from a cancel.
|
|
125
122
|
*/
|
|
126
|
-
export declare const
|
|
127
|
-
export interface
|
|
123
|
+
export declare const CHILD_SUSPENDED_TOPIC = "child-suspended";
|
|
124
|
+
export interface ChildSuspended {
|
|
125
|
+
readonly kind: "suspended";
|
|
128
126
|
readonly childWorkflowId: string;
|
|
129
127
|
readonly stepId: string;
|
|
130
|
-
readonly
|
|
128
|
+
readonly reason: string;
|
|
131
129
|
}
|
|
132
130
|
export interface SandboxStepResult {
|
|
133
131
|
readonly status: SandboxStepStatus;
|
|
@@ -238,13 +236,6 @@ export interface SandboxStepDeps {
|
|
|
238
236
|
/** Write-side embedder for the post-step vector index. */
|
|
239
237
|
readonly embedding: EmbeddingProvider;
|
|
240
238
|
readonly sandboxClient: SandboxClient;
|
|
241
|
-
/**
|
|
242
|
-
* Host-supplied labels for this step's sandbox pod, resolved under the
|
|
243
|
-
* step's session. The map is opaque to the harness: it stamps each entry
|
|
244
|
-
* and interprets none of it. Absent in a wiring that attributes nothing —
|
|
245
|
-
* the pod then carries the harness's own labels only.
|
|
246
|
-
*/
|
|
247
|
-
readonly resolvePodLabels?: (session: RunSession) => Promise<Record<string, string>>;
|
|
248
239
|
/**
|
|
249
240
|
* External artifact registration + sync seam. The harness's post-step pipeline
|
|
250
241
|
* registers each step's outputs through it (filesystem index in the
|
|
@@ -9,9 +9,6 @@
|
|
|
9
9
|
* 2. `createSandbox` — provision the sandbox; persist its handle.
|
|
10
10
|
* 3. `runAgent` — the harness agent loop. Every LLM and tool
|
|
11
11
|
* call inside is its own durableStep.
|
|
12
|
-
* Catches `isBudgetExceeded` and self-cancels
|
|
13
|
-
* via `DBOS.cancelWorkflow(self)` to CANCELLED
|
|
14
|
-
* (resumable, not ERROR).
|
|
15
12
|
* 4. `generateFileMetadata`— post-step LLM call (chunked, slice-to-length).
|
|
16
13
|
* 5. `generateStepSummary` — post-step LLM call producing `output/summary.md`.
|
|
17
14
|
* 6. `reconcile + register`— write artifact manifest entries to cortex_artifacts.
|
|
@@ -22,16 +19,19 @@
|
|
|
22
19
|
* 10. `mark-terminal` — write the final status, emit `data-step-activity`
|
|
23
20
|
* onto the parent's DBOS stream.
|
|
24
21
|
*
|
|
25
|
-
* The parent owns sibling lifecycle:
|
|
26
|
-
* parent (`executeAnalysis`). This child
|
|
27
|
-
*
|
|
22
|
+
* The parent owns sibling lifecycle: the halt cascade of a suspension lives
|
|
23
|
+
* on the parent (`executeAnalysis`). This child owns its own teardown and its
|
|
24
|
+
* own suspension (workflow-suspension spec): a `suspend` error of the agent
|
|
25
|
+
* loop, or a refusal with the suspend flag of the label gate or the register
|
|
26
|
+
* gate. The child records the reason on its row, sends the typed suspension on
|
|
27
|
+
* `CHILD_SUSPENDED_TOPIC`, and self-cancels to CANCELLED (resumable, not ERROR).
|
|
28
28
|
*/
|
|
29
29
|
import { DBOS, Error as DBOSErrors } from "@dbos-inc/dbos-sdk";
|
|
30
30
|
import { insertStepExecution, updateStepExecution } from "../state/index.js";
|
|
31
31
|
import { recordStepCompleted } from "../lib/metrics.js";
|
|
32
32
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
33
|
+
import { deliverNotice } from "../lib/hooks.js";
|
|
33
34
|
import { unwrapOrThrow } from "../lib/result.js";
|
|
34
|
-
import { isBudgetExceeded } from "../loop/budget-exceeded.js";
|
|
35
35
|
import { runAgent } from "../loop/run-agent.js";
|
|
36
36
|
import { durableStep } from "../loop/run-step.js";
|
|
37
37
|
import { lastExecOutcome } from "../sandbox/exec-outcome.js";
|
|
@@ -39,10 +39,12 @@ import { activityForTool, applyTreeDelta, isChatDataPart, sandboxTreeDelta, step
|
|
|
39
39
|
import { createDetailResolver } from "../tools/detail-resolver.js";
|
|
40
40
|
import { forSubAgent } from "../auth/types.js";
|
|
41
41
|
import { walkStepArtifacts } from "../execution/post-step.js";
|
|
42
|
-
import { collectStepOutputs, generateStepFileMetadata, generateStepSummaryAndWrite, reconcileAndRegisterStepArtifacts, vectorIndexStepOutputs, } from "../execution/post-step-pipeline.js";
|
|
42
|
+
import { collectStepOutputs, describeStepRegistrationFailure, generateStepFileMetadata, generateStepSummaryAndWrite, reconcileAndRegisterStepArtifacts, vectorIndexStepOutputs, } from "../execution/post-step-pipeline.js";
|
|
43
43
|
import { mintSandboxIdentity } from "../sandbox/identity.js";
|
|
44
|
+
import { keepSuspendingRefusal } from "../sandbox/sandbox-error.js";
|
|
44
45
|
import { ProvenanceCollector } from "../provenance/collector.js";
|
|
45
46
|
import { createBlockerHolder } from "../tools/sandbox/report-blocker.js";
|
|
47
|
+
import { cancelSelf, suspensionOfFailure, suspensionOfRefusal, suspensionOfSpawnRefusal } from "./suspension.js";
|
|
46
48
|
/**
|
|
47
49
|
* Fallback when the plan step omits `timeout`. Plans that need 3-hour runs
|
|
48
50
|
* declare it explicitly.
|
|
@@ -55,16 +57,13 @@ export const DEFAULT_STEP_TIMEOUT_SECONDS = 3600;
|
|
|
55
57
|
*/
|
|
56
58
|
export const CAPPED_OUT_EMPTY_REASON = "The step hit its iteration cap with no output files persisted.";
|
|
57
59
|
/**
|
|
58
|
-
* DBOS message topic the child uses to
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* `DBOSWorkflowCancelledError` from `getResult` — a generic message that
|
|
64
|
-
* `isBudgetExceeded` does NOT match — and misclassifies the failure as
|
|
65
|
-
* fail-fast, skipping the 402 pause cascade.
|
|
60
|
+
* DBOS message topic the child uses to tell the parent that it suspends. The
|
|
61
|
+
* message goes BEFORE `DBOS.cancelWorkflow(self)`, thus the parent's recv
|
|
62
|
+
* accumulator records the child before the child's terminal state lands.
|
|
63
|
+
* Without it the parent observes only `DBOSWorkflowCancelledError` from
|
|
64
|
+
* `getResult`, and it cannot tell a suspension from a cancel.
|
|
66
65
|
*/
|
|
67
|
-
export const
|
|
66
|
+
export const CHILD_SUSPENDED_TOPIC = "child-suspended";
|
|
68
67
|
// ── Helpers ───────────────────────────────────────────────────────────
|
|
69
68
|
function nextFunctionIdFactory() {
|
|
70
69
|
let n = 0;
|
|
@@ -144,6 +143,24 @@ export async function runSandboxStepBody(input, deps) {
|
|
|
144
143
|
childWorkflowId,
|
|
145
144
|
}));
|
|
146
145
|
}, { name: "mark-running" });
|
|
146
|
+
// The sandbox, if one stands, stays up for the resume. The steps run only on
|
|
147
|
+
// this path, and a replay takes the same path from the same checkpoints.
|
|
148
|
+
const suspendStep = async (suspension) => {
|
|
149
|
+
const durationMs = (await DBOS.now()) - startedAt;
|
|
150
|
+
await DBOS.runStep(async () => {
|
|
151
|
+
unwrapOrThrow(await updateStepExecution(deps.pool, input.runId, input.stepId, {
|
|
152
|
+
status: "canceled",
|
|
153
|
+
durationMs,
|
|
154
|
+
error: suspension.reason,
|
|
155
|
+
attempts: 1,
|
|
156
|
+
lastErrorClass: suspension.reason,
|
|
157
|
+
}));
|
|
158
|
+
recordStepCompleted({ agentId: input.agentId, status: "canceled", durationMs });
|
|
159
|
+
}, { name: "mark-suspended" });
|
|
160
|
+
const message = { kind: "suspended", childWorkflowId, stepId: input.stepId, reason: suspension.reason };
|
|
161
|
+
await DBOS.runStep(() => DBOS.send(input.parentWorkflowId, message, CHILD_SUSPENDED_TOPIC), { name: "notify-parent-suspended" });
|
|
162
|
+
return cancelSelf("self-cancel-suspended");
|
|
163
|
+
};
|
|
147
164
|
// Stamp the sandbox agent's id into provenance so loop events carry
|
|
148
165
|
// `source.agentId === input.agentId` (the parent's RunSession still reads
|
|
149
166
|
// "conversation-agent"). `run-agent.ts` builds source from
|
|
@@ -159,34 +176,19 @@ export async function runSandboxStepBody(input, deps) {
|
|
|
159
176
|
const identity = await DBOS.runStep(() => Promise.resolve(mintSandboxIdentity(input.runId)), { name: "sandbox.mint" });
|
|
160
177
|
// (2b) sandbox.create — spawn (or adopt) the machine under the minted
|
|
161
178
|
// identity. The handle (secret included) is cached so recovery picks the
|
|
162
|
-
// same machine back up without re-provisioning.
|
|
163
|
-
|
|
164
|
-
// unchanged — in-flight workflows resumed across a deploy replay cleanly.
|
|
165
|
-
const sandboxMeta = {
|
|
166
|
-
runId: input.runId,
|
|
167
|
-
stepId: input.stepId,
|
|
168
|
-
analysisId: input.analysisId,
|
|
179
|
+
// same machine back up without re-provisioning.
|
|
180
|
+
const sandboxSpec = {
|
|
169
181
|
childWorkflowId,
|
|
170
182
|
image: input.image,
|
|
171
183
|
extraEnv: input.extraEnv,
|
|
172
184
|
resources: input.resources,
|
|
173
185
|
};
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
catch (err) {
|
|
181
|
-
logger.warn("pod-label resolution failed", { analysisId: input.analysisId, ...logger.errorFields(err) });
|
|
182
|
-
}
|
|
183
|
-
// A wired resolver that yields nothing is the one loud case: the
|
|
184
|
-
// host asked for attribution and the pod spawns without it.
|
|
185
|
-
if (!podLabels || Object.keys(podLabels).length === 0)
|
|
186
|
-
logger.warn("sandbox spawned with no pod labels", { analysisId: input.analysisId });
|
|
187
|
-
}
|
|
188
|
-
return deps.sandboxClient.createSandbox({ ...sandboxMeta, podLabels }, identity);
|
|
189
|
-
}, { name: "sandbox.create" });
|
|
186
|
+
const spawned = await DBOS.runStep(async () => keepSuspendingRefusal(await deps.sandboxClient.createSandbox(session, sandboxSpec, identity)), {
|
|
187
|
+
name: "sandbox.create",
|
|
188
|
+
});
|
|
189
|
+
if (spawned.isErr())
|
|
190
|
+
return suspendStep(suspensionOfSpawnRefusal(spawned.error));
|
|
191
|
+
const sandbox = spawned.value;
|
|
190
192
|
// Recovery path: re-check `isAlive` on the persisted ref before continuing.
|
|
191
193
|
// A classified-dead sandbox triggers a fresh `createSandbox` (task 4.6).
|
|
192
194
|
// For first execution this is a cheap no-op; on replay it's the seam
|
|
@@ -371,52 +373,9 @@ export async function runSandboxStepBody(input, deps) {
|
|
|
371
373
|
stepUsage = agentResult.finish.usage;
|
|
372
374
|
}
|
|
373
375
|
catch (err) {
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
// `isBudgetExceeded` cannot match, so without this side-channel the
|
|
378
|
-
// parent classifier falls through to fail-fast and the 402 pause
|
|
379
|
-
// cascade never fires. Wrapped in `DBOS.runStep` for replay
|
|
380
|
-
// idempotency (DBOS persists the send under (workflowID, function_id),
|
|
381
|
-
// a recovered child does not re-send). Failure to send is logged but
|
|
382
|
-
// swallowed: the worst case is the parent misclassifies the cancel
|
|
383
|
-
// as fail-fast, which is the pre-fix behaviour and still safe.
|
|
384
|
-
const errMsg = err instanceof Error ? err.message : String(err);
|
|
385
|
-
const notification = {
|
|
386
|
-
childWorkflowId,
|
|
387
|
-
stepId: input.stepId,
|
|
388
|
-
error: errMsg,
|
|
389
|
-
};
|
|
390
|
-
try {
|
|
391
|
-
await DBOS.runStep(() => DBOS.send(input.parentWorkflowId, notification, BUDGET_EXCEEDED_TOPIC), { name: "notify-parent-budget-exceeded" });
|
|
392
|
-
}
|
|
393
|
-
catch (sendErr) {
|
|
394
|
-
logger.warn("notify-parent-budget-exceeded failed (non-fatal)", {
|
|
395
|
-
...logger.errorFields(sendErr),
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
|
-
// Self-cancel to CANCELLED so the parent's `getResult` observes the
|
|
399
|
-
// cancellation and triggers the 402 pause cascade. ERROR is NOT
|
|
400
|
-
// resumable (NOTES #3); CANCELLED is what `resumeWorkflow` reads.
|
|
401
|
-
// The subsequent runStep flips the cortex_step_executions row to
|
|
402
|
-
// canceled and raises `DBOSWorkflowCancelledError` — nothing past
|
|
403
|
-
// that line executes. The parent's classifier reads the budget
|
|
404
|
-
// notification from the side-channel set populated above and
|
|
405
|
-
// observes the cancel terminal state via `getResult`.
|
|
406
|
-
await DBOS.cancelWorkflow(childWorkflowId);
|
|
407
|
-
const durationMs = (await DBOS.now()) - startedAt;
|
|
408
|
-
await DBOS.runStep(async () => {
|
|
409
|
-
unwrapOrThrow(await updateStepExecution(deps.pool, input.runId, input.stepId, {
|
|
410
|
-
status: "canceled",
|
|
411
|
-
durationMs,
|
|
412
|
-
error: "budget_exceeded",
|
|
413
|
-
attempts: 1,
|
|
414
|
-
lastErrorClass: "budget_exceeded",
|
|
415
|
-
}));
|
|
416
|
-
}, { name: "mark-canceled" });
|
|
417
|
-
// Unreachable — the runStep above raises DBOSWorkflowCancelledError.
|
|
418
|
-
throw new Error("unreachable: mark-canceled did not raise", { cause: err });
|
|
419
|
-
}
|
|
376
|
+
const suspension = suspensionOfFailure(err);
|
|
377
|
+
if (suspension !== undefined)
|
|
378
|
+
return suspendStep(suspension);
|
|
420
379
|
// Non-resumable failure — tear down, mark failed (scrubbed), re-raise so
|
|
421
380
|
// the parent's `getResult` sees ERROR and the fail-fast cascade fires.
|
|
422
381
|
// `failStep` always throws; the outer `throw` is for control-flow analysis.
|
|
@@ -524,67 +483,52 @@ export async function runSandboxStepBody(input, deps) {
|
|
|
524
483
|
const summary = await safeRunValue(logger, () => DBOS.runStep(() => generateStepSummaryAndWrite(deps, postCtx, manifest), {
|
|
525
484
|
name: "post-step.generate-step-summary",
|
|
526
485
|
}), "post-step.summary", undefined);
|
|
527
|
-
// Lineage attestation (registration
|
|
486
|
+
// Lineage attestation (registration) is fail-fast (see the artifact-manifest spec): unlike
|
|
528
487
|
// the best-effort enrichment stages, a failure here orphans the step's real
|
|
529
488
|
// outputs, so it fails the step loudly instead of finishing green. Transient
|
|
530
|
-
// managed-root errors are already retried in the client, so
|
|
489
|
+
// managed-root errors are already retried in the client, so a failure here is
|
|
531
490
|
// persistent — tear down, mark failed, and re-raise so the parent's fail-fast
|
|
532
|
-
// cascade fires (mirrors the agent-loop failure path). The
|
|
533
|
-
//
|
|
534
|
-
//
|
|
491
|
+
// cascade fires (mirrors the agent-loop failure path). The byte-sync is
|
|
492
|
+
// attempted after the registration on every arm, so a rejected registration
|
|
493
|
+
// still uploads the bytes of whatever DID register: the rows that registered
|
|
494
|
+
// carry `artifact_id` with `file_id` still NULL, which is exactly what
|
|
495
|
+
// `queryUnsyncedStepArtifacts` selects, and a row that never registered is not
|
|
496
|
+
// selected, so the sync stays idempotent under re-execution.
|
|
535
497
|
await emitActivity("persisting", "Registering artifacts");
|
|
536
498
|
// A bare call, deliberately not `DBOS.runStep`-wrapped: it is reachable from
|
|
537
499
|
// two arms below, and a checkpointed step reached from a conditional arm
|
|
538
500
|
// would shift the body's function-ID sequence between the two paths (see the
|
|
539
|
-
// harness-durable-runtime spec).
|
|
540
|
-
|
|
501
|
+
// harness-durable-runtime spec). The sync is a notice: it never displaces a
|
|
502
|
+
// registration failure as the cause of one.
|
|
503
|
+
const syncStepArtifacts = () => deliverNotice(logger.named("post-step.sync"), "ArtifactRegistry.sync", deps.artifactRegistry.sync({
|
|
541
504
|
resourceId: input.analysisId,
|
|
542
505
|
runId: input.runId,
|
|
543
506
|
stepId: input.stepId,
|
|
544
|
-
}, session);
|
|
545
|
-
let
|
|
507
|
+
}, session));
|
|
508
|
+
let registered;
|
|
546
509
|
try {
|
|
547
|
-
|
|
548
|
-
}
|
|
549
|
-
catch (registrationErr) {
|
|
550
|
-
// Registration and byte-sync are independent, and a registry rejection is
|
|
551
|
-
// partial: the rows that DID register carry `artifact_id` with `file_id`
|
|
552
|
-
// still NULL, which is exactly what `queryUnsyncedStepArtifacts` selects.
|
|
553
|
-
// Syncing anyway uploads their bytes (rows that never registered are not
|
|
554
|
-
// selected, so this stays idempotent under re-execution); skipping it is
|
|
555
|
-
// what strands them registered-but-byteless.
|
|
556
|
-
try {
|
|
557
|
-
await syncStepArtifacts();
|
|
558
|
-
}
|
|
559
|
-
catch (syncErr) {
|
|
560
|
-
// A cancellation is control-flow, not a sync failure — re-raise it
|
|
561
|
-
// verbatim, as every other stage in this body does. Any real failure
|
|
562
|
-
// is swallowed so it cannot displace the registration rejection as the
|
|
563
|
-
// step's cause; this record is the only account of it.
|
|
564
|
-
if (syncErr instanceof DBOSErrors.DBOSWorkflowCancelledError)
|
|
565
|
-
throw syncErr;
|
|
566
|
-
logger.named("post-step.sync").warn("failed after a registration rejection (non-fatal)", logger.errorFields(syncErr));
|
|
567
|
-
}
|
|
568
|
-
throw await failStep("lineage_attestation", registrationErr);
|
|
510
|
+
registered = await reconcileAndRegisterStepArtifacts(deps, postCtx, manifest);
|
|
569
511
|
}
|
|
570
|
-
|
|
512
|
+
catch (attestationErr) {
|
|
513
|
+
if (attestationErr instanceof DBOSErrors.DBOSWorkflowCancelledError)
|
|
514
|
+
throw attestationErr;
|
|
571
515
|
await syncStepArtifacts();
|
|
516
|
+
throw await failStep("lineage_attestation", attestationErr);
|
|
572
517
|
}
|
|
573
|
-
|
|
574
|
-
|
|
518
|
+
await syncStepArtifacts();
|
|
519
|
+
if (registered.isErr()) {
|
|
520
|
+
const refused = registered.error.kind === "refused" ? suspensionOfRefusal(registered.error.refusal) : undefined;
|
|
521
|
+
if (refused !== undefined)
|
|
522
|
+
return suspendStep(refused);
|
|
523
|
+
throw await failStep("lineage_attestation", new Error(describeStepRegistrationFailure(registered.error)));
|
|
575
524
|
}
|
|
525
|
+
const reconciledManifest = registered.value;
|
|
576
526
|
const postArtifacts = {
|
|
577
527
|
metadataEntries,
|
|
578
528
|
summary,
|
|
579
529
|
reconciledManifest,
|
|
580
530
|
};
|
|
581
|
-
|
|
582
|
-
try {
|
|
583
|
-
stepOutputs = collectStepOutputs(postCtx, postArtifacts);
|
|
584
|
-
}
|
|
585
|
-
catch (err) {
|
|
586
|
-
logger.warn("post-step.collect failed (non-fatal)", logger.errorFields(err));
|
|
587
|
-
}
|
|
531
|
+
const stepOutputs = collectStepOutputs(postCtx, postArtifacts);
|
|
588
532
|
await emitActivity("indexing", "Indexing outputs for search");
|
|
589
533
|
await safeRun(logger, () => vectorIndexStepOutputs(deps, postCtx, postArtifacts), "post-step.vector-index");
|
|
590
534
|
// (9) teardown — destroy sandbox + clear sandbox_ref.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The suspension of work (workflow-suspension spec).
|
|
3
|
+
*
|
|
4
|
+
* A suspension starts only from a model request that fails with a `suspend`
|
|
5
|
+
* error, or from a gate that refuses with the suspend flag. The harness carries
|
|
6
|
+
* the reason of the host as it is and never reads it: no path compares a
|
|
7
|
+
* reason to a value, and no path reads the text of an error message.
|
|
8
|
+
*
|
|
9
|
+
* The owner of the result of a workflow selects the mechanism. A durable owner
|
|
10
|
+
* (`executeAnalysis`, `sandbox-step`, `data-profile`) records the reason in its
|
|
11
|
+
* stored state and ends in the DBOS state `CANCELLED`, which `resumeWorkflow`
|
|
12
|
+
* reads. A workflow with a live caller (`derive-table-exec`, `extract-values`)
|
|
13
|
+
* returns the suspension as the `err` of its result. A suspension of an
|
|
14
|
+
* analysis run or of a data profile marks the analysis through
|
|
15
|
+
* `suspendAnalysis` (`state/analyses.ts`). A workflow with a live caller does
|
|
16
|
+
* not mark it: the conversation agent reads the suspension as a tool result.
|
|
17
|
+
*/
|
|
18
|
+
import type { GateRefusal } from "../lib/hooks.js";
|
|
19
|
+
import type { SuspendingRefusal } from "../sandbox/sandbox-error.js";
|
|
20
|
+
/** Plain data, thus it crosses a DBOS checkpoint as it is. */
|
|
21
|
+
export interface Suspension {
|
|
22
|
+
readonly kind: "suspended";
|
|
23
|
+
readonly reason: string;
|
|
24
|
+
}
|
|
25
|
+
export declare function suspensionOfFailure(err: unknown): Suspension | undefined;
|
|
26
|
+
export declare function suspensionOfRefusal(refusal: GateRefusal): Suspension | undefined;
|
|
27
|
+
export declare function suspensionOfSpawnRefusal(refusal: SuspendingRefusal): Suspension;
|
|
28
|
+
/**
|
|
29
|
+
* The cancel lands at the next DBOS operation, thus the named step raises
|
|
30
|
+
* `DBOSWorkflowCancelledError` and nothing after it runs.
|
|
31
|
+
*/
|
|
32
|
+
export declare function cancelSelf(stepName: string): Promise<never>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The suspension of work (workflow-suspension spec).
|
|
3
|
+
*
|
|
4
|
+
* A suspension starts only from a model request that fails with a `suspend`
|
|
5
|
+
* error, or from a gate that refuses with the suspend flag. The harness carries
|
|
6
|
+
* the reason of the host as it is and never reads it: no path compares a
|
|
7
|
+
* reason to a value, and no path reads the text of an error message.
|
|
8
|
+
*
|
|
9
|
+
* The owner of the result of a workflow selects the mechanism. A durable owner
|
|
10
|
+
* (`executeAnalysis`, `sandbox-step`, `data-profile`) records the reason in its
|
|
11
|
+
* stored state and ends in the DBOS state `CANCELLED`, which `resumeWorkflow`
|
|
12
|
+
* reads. A workflow with a live caller (`derive-table-exec`, `extract-values`)
|
|
13
|
+
* returns the suspension as the `err` of its result. A suspension of an
|
|
14
|
+
* analysis run or of a data profile marks the analysis through
|
|
15
|
+
* `suspendAnalysis` (`state/analyses.ts`). A workflow with a live caller does
|
|
16
|
+
* not mark it: the conversation agent reads the suspension as a tool result.
|
|
17
|
+
*/
|
|
18
|
+
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
19
|
+
import { findSuspendError } from "../providers/errors.js";
|
|
20
|
+
export function suspensionOfFailure(err) {
|
|
21
|
+
const found = findSuspendError(err);
|
|
22
|
+
return found === undefined ? undefined : { kind: "suspended", reason: found.reason };
|
|
23
|
+
}
|
|
24
|
+
export function suspensionOfRefusal(refusal) {
|
|
25
|
+
return refusal.kind === "suspended" ? { kind: "suspended", reason: refusal.reason } : undefined;
|
|
26
|
+
}
|
|
27
|
+
export function suspensionOfSpawnRefusal(refusal) {
|
|
28
|
+
return { kind: "suspended", reason: refusal.reason };
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The cancel lands at the next DBOS operation, thus the named step raises
|
|
32
|
+
* `DBOSWorkflowCancelledError` and nothing after it runs.
|
|
33
|
+
*/
|
|
34
|
+
export async function cancelSelf(stepName) {
|
|
35
|
+
await DBOS.cancelWorkflow(DBOS.workflowID);
|
|
36
|
+
await DBOS.runStep(async () => undefined, { name: stepName });
|
|
37
|
+
throw new Error(`unreachable: ${stepName} did not raise the cancellation`);
|
|
38
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { ResolveBilling } from "./resolver.js";
|
|
2
|
-
/**
|
|
3
|
-
* Local/OSS billing resolver: resolves to an empty header map. No upstream
|
|
4
|
-
* call, no credential reads. Wire calls carry no attribution headers.
|
|
5
|
-
*/
|
|
6
|
-
export declare function createNoopBillingResolver(): ResolveBilling;
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Billing attribution seam.
|
|
3
|
-
*
|
|
4
|
-
* Defines the `ResolveBilling` contract the provider calls at the
|
|
5
|
-
* LLM/embedding call site, plus the session view and fetcher types a
|
|
6
|
-
* concrete resolver implements. The provider calls `resolveBilling(session)`
|
|
7
|
-
* just before the wire call; read-only routes never call it, so they never
|
|
8
|
-
* incur a billing round-trip.
|
|
9
|
-
*
|
|
10
|
-
* This module is the vendor-neutral seam: it declares types only. A concrete
|
|
11
|
-
* resolver (managed/upstream-backed or noop) supplies the fetch + assembly
|
|
12
|
-
* behavior behind the `ResolveBilling` function type.
|
|
13
|
-
*/
|
|
14
|
-
import type { AuthContext, BillingSessionView, Credential, ResourceCoordinates } from "../auth/types.js";
|
|
15
|
-
export type BillingMap = Record<string, string>;
|
|
16
|
-
/** Final wire-format header map sent to the upstream gateway. */
|
|
17
|
-
export type BillingHeaders = Record<string, string>;
|
|
18
|
-
/**
|
|
19
|
-
* Result of a single billing-attribution fetch. The seam's own type — a
|
|
20
|
-
* concrete resolver's richer result (e.g. a generic upstream envelope)
|
|
21
|
-
* structurally satisfies it. The failure branch carries what the resolver
|
|
22
|
-
* needs to raise a `BillingResolutionError`.
|
|
23
|
-
*/
|
|
24
|
-
export type BillingFetchResult = {
|
|
25
|
-
ok: true;
|
|
26
|
-
data: BillingMap;
|
|
27
|
-
} | {
|
|
28
|
-
ok: false;
|
|
29
|
-
status: number;
|
|
30
|
-
error: {
|
|
31
|
-
error: string;
|
|
32
|
-
message: string;
|
|
33
|
-
};
|
|
34
|
-
};
|
|
35
|
-
/**
|
|
36
|
-
* The minimal session view a resolver reads. The cached map is keyed by
|
|
37
|
-
* `(user, resourceType, resourceId)` (from `identity`/`scope` on
|
|
38
|
-
* `BillingSessionView`); header assembly additionally consumes
|
|
39
|
-
* `provenance`/`runFrame` (per-call). The fetch resolves the credential
|
|
40
|
-
* + org via `getAuth(session)`, so only the opaque `auth` is needed here —
|
|
41
|
-
* `AgentSession` structurally satisfies it.
|
|
42
|
-
*/
|
|
43
|
-
export interface ResolvableSession extends BillingSessionView {
|
|
44
|
-
readonly auth: AuthContext;
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Resolve the FINAL wire headers at the call site. The raw attribution map is
|
|
48
|
-
* fetched once per `(user, resource)` and cached; the per-call identity tags
|
|
49
|
-
* (`agentId`/`runId`/`stepId`) vary within that key, so header assembly runs
|
|
50
|
-
* POST-CACHE on every call against the live `session` — assembled headers are
|
|
51
|
-
* never cached.
|
|
52
|
-
*/
|
|
53
|
-
export type ResolveBilling = (session: ResolvableSession) => Promise<BillingHeaders>;
|
|
54
|
-
/**
|
|
55
|
-
* The upstream call seam — overridable in tests with a fake. A concrete
|
|
56
|
-
* resolver resolves the attribution map for the given credential/org/coords.
|
|
57
|
-
*/
|
|
58
|
-
export type BillingFetcher = (credential: Credential, orgId: string, coords: ResourceCoordinates) => Promise<BillingFetchResult>;
|
|
59
|
-
/** A billing-resolution failure (e.g. 403/404). Fails the LLM call. */
|
|
60
|
-
export declare class BillingResolutionError extends Error {
|
|
61
|
-
readonly status: number;
|
|
62
|
-
readonly code: string;
|
|
63
|
-
constructor(status: number, code: string, message: string);
|
|
64
|
-
}
|
package/dist/billing/resolver.js
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Billing attribution seam.
|
|
3
|
-
*
|
|
4
|
-
* Defines the `ResolveBilling` contract the provider calls at the
|
|
5
|
-
* LLM/embedding call site, plus the session view and fetcher types a
|
|
6
|
-
* concrete resolver implements. The provider calls `resolveBilling(session)`
|
|
7
|
-
* just before the wire call; read-only routes never call it, so they never
|
|
8
|
-
* incur a billing round-trip.
|
|
9
|
-
*
|
|
10
|
-
* This module is the vendor-neutral seam: it declares types only. A concrete
|
|
11
|
-
* resolver (managed/upstream-backed or noop) supplies the fetch + assembly
|
|
12
|
-
* behavior behind the `ResolveBilling` function type.
|
|
13
|
-
*/
|
|
14
|
-
/** A billing-resolution failure (e.g. 403/404). Fails the LLM call. */
|
|
15
|
-
export class BillingResolutionError extends Error {
|
|
16
|
-
status;
|
|
17
|
-
code;
|
|
18
|
-
constructor(status, code, message) {
|
|
19
|
-
super(`Billing resolution failed (${status} ${code}): ${message}`);
|
|
20
|
-
this.status = status;
|
|
21
|
-
this.code = code;
|
|
22
|
-
this.name = "BillingResolutionError";
|
|
23
|
-
}
|
|
24
|
-
}
|