@inflexa-ai/harness 0.37.0 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +1 -2
  2. package/dist/auth/local-run-authorizer.js +5 -4
  3. package/dist/auth/types.d.ts +28 -9
  4. package/dist/auth/types.js +5 -3
  5. package/dist/billing/cached-resolver.d.ts +11 -7
  6. package/dist/billing/cached-resolver.js +17 -11
  7. package/dist/billing/noop-run-charge.js +3 -2
  8. package/dist/billing/noop-usage-recorder.js +2 -1
  9. package/dist/billing/run-charge.d.ts +18 -5
  10. package/dist/billing/run-charge.js +5 -1
  11. package/dist/billing/usage-recorder.d.ts +11 -9
  12. package/dist/billing/usage-recorder.js +2 -3
  13. package/dist/contracts/chat-events.d.ts +4 -1
  14. package/dist/contracts/chat-parts.d.ts +1 -1
  15. package/dist/contracts/schemas/chat-parts.d.ts +4 -4
  16. package/dist/execution/artifact-registration.d.ts +6 -1
  17. package/dist/execution/artifact-registration.js +12 -4
  18. package/dist/execution/artifact-registry.d.ts +10 -3
  19. package/dist/execution/noop-artifact-registry.js +3 -4
  20. package/dist/execution/post-step-pipeline.d.ts +14 -3
  21. package/dist/execution/post-step-pipeline.js +21 -8
  22. package/dist/execution/run-authorizer.d.ts +5 -3
  23. package/dist/execution/run-canceler.js +4 -20
  24. package/dist/index.d.ts +14 -8
  25. package/dist/index.js +12 -6
  26. package/dist/lib/hooks.d.ts +49 -0
  27. package/dist/lib/hooks.js +39 -0
  28. package/dist/lib/result.d.ts +22 -11
  29. package/dist/lib/result.js +24 -14
  30. package/dist/loop/run-agent.d.ts +2 -2
  31. package/dist/loop/run-agent.js +5 -4
  32. package/dist/provenance/seam.d.ts +1 -1
  33. package/dist/providers/ai-sdk.d.ts +43 -5
  34. package/dist/providers/ai-sdk.js +55 -52
  35. package/dist/providers/anthropic.d.ts +2 -3
  36. package/dist/providers/anthropic.js +2 -2
  37. package/dist/providers/embedding.d.ts +13 -7
  38. package/dist/providers/embedding.js +26 -6
  39. package/dist/providers/errors.d.ts +67 -37
  40. package/dist/providers/errors.js +57 -58
  41. package/dist/providers/request-headers.d.ts +18 -0
  42. package/dist/providers/request-headers.js +22 -0
  43. package/dist/report-model/production-resolver.d.ts +5 -2
  44. package/dist/report-model/production-resolver.js +12 -2
  45. package/dist/runtime/assemble.d.ts +3 -2
  46. package/dist/runtime/dbos.d.ts +4 -3
  47. package/dist/runtime/dbos.js +18 -3
  48. package/dist/runtime/result-serialization.d.ts +29 -0
  49. package/dist/runtime/result-serialization.js +45 -0
  50. package/dist/sandbox/client.d.ts +11 -3
  51. package/dist/sandbox/create-sandbox.d.ts +5 -2
  52. package/dist/sandbox/create-sandbox.js +51 -25
  53. package/dist/sandbox/docker-client.d.ts +4 -3
  54. package/dist/sandbox/docker-client.js +23 -31
  55. package/dist/sandbox/k8s-client.d.ts +4 -13
  56. package/dist/sandbox/k8s-client.js +28 -61
  57. package/dist/sandbox/labels.d.ts +34 -0
  58. package/dist/sandbox/labels.js +47 -0
  59. package/dist/sandbox/mount-plan.d.ts +3 -1
  60. package/dist/sandbox/mount-plan.js +9 -0
  61. package/dist/sandbox/sandbox-error.d.ts +36 -13
  62. package/dist/sandbox/sandbox-error.js +32 -17
  63. package/dist/sandbox/types.d.ts +18 -10
  64. package/dist/schemas/validate-plan.js +13 -0
  65. package/dist/schemas/workflow-state.d.ts +4 -4
  66. package/dist/state/analyses.d.ts +4 -4
  67. package/dist/state/analyses.js +4 -4
  68. package/dist/state/schema.d.ts +5 -5
  69. package/dist/tasks/data-profile.d.ts +5 -8
  70. package/dist/tasks/data-profile.js +88 -43
  71. package/dist/tasks/derive-table-exec.d.ts +10 -6
  72. package/dist/tasks/derive-table-exec.js +17 -12
  73. package/dist/tasks/extract-values.d.ts +16 -10
  74. package/dist/tasks/extract-values.js +34 -18
  75. package/dist/tools/execute-analysis.js +29 -24
  76. package/dist/tools/report-session/derive-table.d.ts +9 -1
  77. package/dist/tools/report-session/derive-table.js +25 -22
  78. package/dist/tools/report-session/examine-page.d.ts +6 -3
  79. package/dist/tools/report-session/examine-page.js +16 -8
  80. package/dist/tools/report-session/index.d.ts +1 -1
  81. package/dist/tools/report-session/preview-report.js +11 -9
  82. package/dist/tools/report-session/session-page-publisher.d.ts +17 -21
  83. package/dist/tools/report-session/session-page-publisher.js +5 -7
  84. package/dist/tools/research/inspect-run.d.ts +1 -1
  85. package/dist/tools/sandbox/link-packages.d.ts +8 -0
  86. package/dist/tools/sandbox/link-packages.js +19 -9
  87. package/dist/workflows/execute-analysis.d.ts +25 -16
  88. package/dist/workflows/execute-analysis.js +223 -216
  89. package/dist/workflows/metrics.d.ts +4 -3
  90. package/dist/workflows/metrics.js +4 -3
  91. package/dist/workflows/sandbox-step.d.ts +20 -29
  92. package/dist/workflows/sandbox-step.js +71 -127
  93. package/dist/workflows/suspension.d.ts +32 -0
  94. package/dist/workflows/suspension.js +38 -0
  95. package/package.json +1 -1
  96. package/dist/billing/noop-resolver.d.ts +0 -6
  97. package/dist/billing/noop-resolver.js +0 -7
  98. package/dist/billing/resolver.d.ts +0 -64
  99. package/dist/billing/resolver.js +0 -24
  100. package/dist/loop/budget-exceeded.d.ts +0 -38
  101. package/dist/loop/budget-exceeded.js +0 -88
@@ -12,11 +12,12 @@
12
12
  /**
13
13
  * Record one child cancellation. The parent calls this once per
14
14
  * `DBOS.cancelWorkflow(childWorkflowId)` in the cascade (fail-fast OR
15
- * pause). `cause` distinguishes the two so dashboards can split the
16
- * counter.
15
+ * suspension). `cause` is `fail_fast`, `external_cancel`, or the reason of
16
+ * the host for a suspension, carried unread, so dashboards can split the
17
+ * counter by the reason that the host gave.
17
18
  */
18
19
  export declare function recordCancelledChild(args: {
19
- readonly cause: "fail_fast" | "budget_exceeded" | "external_cancel";
20
+ readonly cause: string;
20
21
  }): void;
21
22
  /** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
22
23
  export declare function __resetWorkflowMetricsForTest(): void;
@@ -16,7 +16,7 @@ function getInstruments() {
16
16
  const meter = metrics.getMeter("cortex.workflow");
17
17
  instruments = {
18
18
  cancelledChildren: meter.createCounter("cortex.workflow.parent.cancelled_children", {
19
- description: "Children cancelled by the parent's fail-fast or pause cascade",
19
+ description: "Children cancelled by the parent's fail-fast or suspension cascade",
20
20
  }),
21
21
  };
22
22
  }
@@ -25,8 +25,9 @@ function getInstruments() {
25
25
  /**
26
26
  * Record one child cancellation. The parent calls this once per
27
27
  * `DBOS.cancelWorkflow(childWorkflowId)` in the cascade (fail-fast OR
28
- * pause). `cause` distinguishes the two so dashboards can split the
29
- * counter.
28
+ * suspension). `cause` is `fail_fast`, `external_cancel`, or the reason of
29
+ * the host for a suspension, carried unread, so dashboards can split the
30
+ * counter by the reason that the host gave.
30
31
  */
31
32
  export function recordCancelledChild(args) {
32
33
  getInstruments().cancelledChildren.add(1, { cause: args.cause });
@@ -9,9 +9,6 @@
9
9
  * 2. `createSandbox` — provision the sandbox; persist its handle.
10
10
  * 3. `runAgent` — the harness agent loop. Every LLM and tool
11
11
  * call inside is its own durableStep.
12
- * Catches `isBudgetExceeded` and self-cancels
13
- * via `DBOS.cancelWorkflow(self)` to CANCELLED
14
- * (resumable, not ERROR).
15
12
  * 4. `generateFileMetadata`— post-step LLM call (chunked, slice-to-length).
16
13
  * 5. `generateStepSummary` — post-step LLM call producing `output/summary.md`.
17
14
  * 6. `reconcile + register`— write artifact manifest entries to cortex_artifacts.
@@ -22,9 +19,12 @@
22
19
  * 10. `mark-terminal` — write the final status, emit `data-step-activity`
23
20
  * onto the parent's DBOS stream.
24
21
  *
25
- * The parent owns sibling lifecycle: fail-fast and 402-cascade live on the
26
- * parent (`executeAnalysis`). This child only owns its own teardown and
27
- * its own self-cancel on 402.
22
+ * The parent owns sibling lifecycle: the halt cascade of a suspension lives
23
+ * on the parent (`executeAnalysis`). This child owns its own teardown and its
24
+ * own suspension (workflow-suspension spec): a `suspend` error of the agent
25
+ * loop, or a refusal with the suspend flag of the label gate or the register
26
+ * gate. The child records the reason on its row, sends the typed suspension on
27
+ * `CHILD_SUSPENDED_TOPIC`, and self-cancels to CANCELLED (resumable, not ERROR).
28
28
  */
29
29
  import type { TokenUsageRollup } from "@inflexa-ai/harness/contracts/usage.js";
30
30
  import type { Pool } from "pg";
@@ -33,7 +33,7 @@ import type { UsageRecorder } from "../billing/usage-recorder.js";
33
33
  import type { CitationResolver } from "../citations/types.js";
34
34
  import type { AgentDefinition, LoopMessage } from "../loop/types.js";
35
35
  import type { AgentChat, EmbeddingProvider } from "../providers/types.js";
36
- import { type RunSession } from "../auth/types.js";
36
+ import { type RunSession, type SpawnSession } from "../auth/types.js";
37
37
  import type { FileMetadataEntry } from "../execution/artifact-metadata.js";
38
38
  import type { ArtifactManifestEntry } from "../schemas/artifact-manifest.js";
39
39
  import type { StepSummary } from "../schemas/step-summary.js";
@@ -72,9 +72,9 @@ export interface SandboxStepInput {
72
72
  readonly prompt: string;
73
73
  /**
74
74
  * DBOS workflow id of the parent (`executeAnalysis`). The child uses
75
- * this to address the parent for the budget-exceeded side-channel
76
- * (`DBOS.send(parent, …, "child-budget-exceeded")`) — the parent's
77
- * scheduler-loop classifier relies on it to disambiguate a budget-induced
75
+ * this to address the parent for its typed suspension
76
+ * (`DBOS.send(parent, …, CHILD_SUSPENDED_TOPIC)`) — the parent's
77
+ * scheduler-loop classifier relies on it to disambiguate a suspension
78
78
  * self-cancel from an operator-driven sibling cancel (CANCELLED is the
79
79
  * same terminal state for both).
80
80
  */
@@ -99,7 +99,7 @@ export interface SandboxStepInput {
99
99
  * DBOS replay reconstructs it on resume — the body never reads the JWT
100
100
  * from `cortex_runs`.
101
101
  */
102
- readonly runSession: RunSession;
102
+ readonly runSession: SpawnSession;
103
103
  }
104
104
  /**
105
105
  * Fallback when the plan step omits `timeout`. Plans that need 3-hour runs
@@ -114,20 +114,18 @@ export declare const DEFAULT_STEP_TIMEOUT_SECONDS = 3600;
114
114
  export declare const CAPPED_OUT_EMPTY_REASON = "The step hit its iteration cap with no output files persisted.";
115
115
  export type SandboxStepStatus = "complete" | "failed" | "canceled" | "blocked";
116
116
  /**
117
- * DBOS message topic the child uses to notify the parent that it is
118
- * about to self-cancel because of a 402 budget-exceeded error. The
119
- * notification fires BEFORE `DBOS.cancelWorkflow(self)` so the parent's
120
- * concurrent recv accumulator records the child id before the child's
121
- * terminal state lands. Without it the parent observes
122
- * `DBOSWorkflowCancelledError` from `getResult` — a generic message that
123
- * `isBudgetExceeded` does NOT match — and misclassifies the failure as
124
- * fail-fast, skipping the 402 pause cascade.
117
+ * DBOS message topic the child uses to tell the parent that it suspends. The
118
+ * message goes BEFORE `DBOS.cancelWorkflow(self)`, thus the parent's recv
119
+ * accumulator records the child before the child's terminal state lands.
120
+ * Without it the parent observes only `DBOSWorkflowCancelledError` from
121
+ * `getResult`, and it cannot tell a suspension from a cancel.
125
122
  */
126
- export declare const BUDGET_EXCEEDED_TOPIC = "child-budget-exceeded";
127
- export interface BudgetExceededNotification {
123
+ export declare const CHILD_SUSPENDED_TOPIC = "child-suspended";
124
+ export interface ChildSuspended {
125
+ readonly kind: "suspended";
128
126
  readonly childWorkflowId: string;
129
127
  readonly stepId: string;
130
- readonly error: string;
128
+ readonly reason: string;
131
129
  }
132
130
  export interface SandboxStepResult {
133
131
  readonly status: SandboxStepStatus;
@@ -238,13 +236,6 @@ export interface SandboxStepDeps {
238
236
  /** Write-side embedder for the post-step vector index. */
239
237
  readonly embedding: EmbeddingProvider;
240
238
  readonly sandboxClient: SandboxClient;
241
- /**
242
- * Host-supplied labels for this step's sandbox pod, resolved under the
243
- * step's session. The map is opaque to the harness: it stamps each entry
244
- * and interprets none of it. Absent in a wiring that attributes nothing —
245
- * the pod then carries the harness's own labels only.
246
- */
247
- readonly resolvePodLabels?: (session: RunSession) => Promise<Record<string, string>>;
248
239
  /**
249
240
  * External artifact registration + sync seam. The harness's post-step pipeline
250
241
  * registers each step's outputs through it (filesystem index in the
@@ -9,9 +9,6 @@
9
9
  * 2. `createSandbox` — provision the sandbox; persist its handle.
10
10
  * 3. `runAgent` — the harness agent loop. Every LLM and tool
11
11
  * call inside is its own durableStep.
12
- * Catches `isBudgetExceeded` and self-cancels
13
- * via `DBOS.cancelWorkflow(self)` to CANCELLED
14
- * (resumable, not ERROR).
15
12
  * 4. `generateFileMetadata`— post-step LLM call (chunked, slice-to-length).
16
13
  * 5. `generateStepSummary` — post-step LLM call producing `output/summary.md`.
17
14
  * 6. `reconcile + register`— write artifact manifest entries to cortex_artifacts.
@@ -22,16 +19,19 @@
22
19
  * 10. `mark-terminal` — write the final status, emit `data-step-activity`
23
20
  * onto the parent's DBOS stream.
24
21
  *
25
- * The parent owns sibling lifecycle: fail-fast and 402-cascade live on the
26
- * parent (`executeAnalysis`). This child only owns its own teardown and
27
- * its own self-cancel on 402.
22
+ * The parent owns sibling lifecycle: the halt cascade of a suspension lives
23
+ * on the parent (`executeAnalysis`). This child owns its own teardown and its
24
+ * own suspension (workflow-suspension spec): a `suspend` error of the agent
25
+ * loop, or a refusal with the suspend flag of the label gate or the register
26
+ * gate. The child records the reason on its row, sends the typed suspension on
27
+ * `CHILD_SUSPENDED_TOPIC`, and self-cancels to CANCELLED (resumable, not ERROR).
28
28
  */
29
29
  import { DBOS, Error as DBOSErrors } from "@dbos-inc/dbos-sdk";
30
30
  import { insertStepExecution, updateStepExecution } from "../state/index.js";
31
31
  import { recordStepCompleted } from "../lib/metrics.js";
32
32
  import { createNoopLogger } from "../lib/console-logger.js";
33
+ import { deliverNotice } from "../lib/hooks.js";
33
34
  import { unwrapOrThrow } from "../lib/result.js";
34
- import { isBudgetExceeded } from "../loop/budget-exceeded.js";
35
35
  import { runAgent } from "../loop/run-agent.js";
36
36
  import { durableStep } from "../loop/run-step.js";
37
37
  import { lastExecOutcome } from "../sandbox/exec-outcome.js";
@@ -39,10 +39,12 @@ import { activityForTool, applyTreeDelta, isChatDataPart, sandboxTreeDelta, step
39
39
  import { createDetailResolver } from "../tools/detail-resolver.js";
40
40
  import { forSubAgent } from "../auth/types.js";
41
41
  import { walkStepArtifacts } from "../execution/post-step.js";
42
- import { collectStepOutputs, generateStepFileMetadata, generateStepSummaryAndWrite, reconcileAndRegisterStepArtifacts, vectorIndexStepOutputs, } from "../execution/post-step-pipeline.js";
42
+ import { collectStepOutputs, describeStepRegistrationFailure, generateStepFileMetadata, generateStepSummaryAndWrite, reconcileAndRegisterStepArtifacts, vectorIndexStepOutputs, } from "../execution/post-step-pipeline.js";
43
43
  import { mintSandboxIdentity } from "../sandbox/identity.js";
44
+ import { keepSuspendingRefusal } from "../sandbox/sandbox-error.js";
44
45
  import { ProvenanceCollector } from "../provenance/collector.js";
45
46
  import { createBlockerHolder } from "../tools/sandbox/report-blocker.js";
47
+ import { cancelSelf, suspensionOfFailure, suspensionOfRefusal, suspensionOfSpawnRefusal } from "./suspension.js";
46
48
  /**
47
49
  * Fallback when the plan step omits `timeout`. Plans that need 3-hour runs
48
50
  * declare it explicitly.
@@ -55,16 +57,13 @@ export const DEFAULT_STEP_TIMEOUT_SECONDS = 3600;
55
57
  */
56
58
  export const CAPPED_OUT_EMPTY_REASON = "The step hit its iteration cap with no output files persisted.";
57
59
  /**
58
- * DBOS message topic the child uses to notify the parent that it is
59
- * about to self-cancel because of a 402 budget-exceeded error. The
60
- * notification fires BEFORE `DBOS.cancelWorkflow(self)` so the parent's
61
- * concurrent recv accumulator records the child id before the child's
62
- * terminal state lands. Without it the parent observes
63
- * `DBOSWorkflowCancelledError` from `getResult` — a generic message that
64
- * `isBudgetExceeded` does NOT match — and misclassifies the failure as
65
- * fail-fast, skipping the 402 pause cascade.
60
+ * DBOS message topic the child uses to tell the parent that it suspends. The
61
+ * message goes BEFORE `DBOS.cancelWorkflow(self)`, thus the parent's recv
62
+ * accumulator records the child before the child's terminal state lands.
63
+ * Without it the parent observes only `DBOSWorkflowCancelledError` from
64
+ * `getResult`, and it cannot tell a suspension from a cancel.
66
65
  */
67
- export const BUDGET_EXCEEDED_TOPIC = "child-budget-exceeded";
66
+ export const CHILD_SUSPENDED_TOPIC = "child-suspended";
68
67
  // ── Helpers ───────────────────────────────────────────────────────────
69
68
  function nextFunctionIdFactory() {
70
69
  let n = 0;
@@ -144,6 +143,24 @@ export async function runSandboxStepBody(input, deps) {
144
143
  childWorkflowId,
145
144
  }));
146
145
  }, { name: "mark-running" });
146
+ // The sandbox, if one stands, stays up for the resume. The steps run only on
147
+ // this path, and a replay takes the same path from the same checkpoints.
148
+ const suspendStep = async (suspension) => {
149
+ const durationMs = (await DBOS.now()) - startedAt;
150
+ await DBOS.runStep(async () => {
151
+ unwrapOrThrow(await updateStepExecution(deps.pool, input.runId, input.stepId, {
152
+ status: "canceled",
153
+ durationMs,
154
+ error: suspension.reason,
155
+ attempts: 1,
156
+ lastErrorClass: suspension.reason,
157
+ }));
158
+ recordStepCompleted({ agentId: input.agentId, status: "canceled", durationMs });
159
+ }, { name: "mark-suspended" });
160
+ const message = { kind: "suspended", childWorkflowId, stepId: input.stepId, reason: suspension.reason };
161
+ await DBOS.runStep(() => DBOS.send(input.parentWorkflowId, message, CHILD_SUSPENDED_TOPIC), { name: "notify-parent-suspended" });
162
+ return cancelSelf("self-cancel-suspended");
163
+ };
147
164
  // Stamp the sandbox agent's id into provenance so loop events carry
148
165
  // `source.agentId === input.agentId` (the parent's RunSession still reads
149
166
  // "conversation-agent"). `run-agent.ts` builds source from
@@ -159,34 +176,19 @@ export async function runSandboxStepBody(input, deps) {
159
176
  const identity = await DBOS.runStep(() => Promise.resolve(mintSandboxIdentity(input.runId)), { name: "sandbox.mint" });
160
177
  // (2b) sandbox.create — spawn (or adopt) the machine under the minted
161
178
  // identity. The handle (secret included) is cached so recovery picks the
162
- // same machine back up without re-provisioning. Pod-label resolution lives
163
- // INSIDE this step (not as its own step) so the child's step sequence is
164
- // unchanged — in-flight workflows resumed across a deploy replay cleanly.
165
- const sandboxMeta = {
166
- runId: input.runId,
167
- stepId: input.stepId,
168
- analysisId: input.analysisId,
179
+ // same machine back up without re-provisioning.
180
+ const sandboxSpec = {
169
181
  childWorkflowId,
170
182
  image: input.image,
171
183
  extraEnv: input.extraEnv,
172
184
  resources: input.resources,
173
185
  };
174
- const sandbox = await DBOS.runStep(async () => {
175
- let podLabels;
176
- if (deps.resolvePodLabels) {
177
- try {
178
- podLabels = await deps.resolvePodLabels(session);
179
- }
180
- catch (err) {
181
- logger.warn("pod-label resolution failed", { analysisId: input.analysisId, ...logger.errorFields(err) });
182
- }
183
- // A wired resolver that yields nothing is the one loud case: the
184
- // host asked for attribution and the pod spawns without it.
185
- if (!podLabels || Object.keys(podLabels).length === 0)
186
- logger.warn("sandbox spawned with no pod labels", { analysisId: input.analysisId });
187
- }
188
- return deps.sandboxClient.createSandbox({ ...sandboxMeta, podLabels }, identity);
189
- }, { name: "sandbox.create" });
186
+ const spawned = await DBOS.runStep(async () => keepSuspendingRefusal(await deps.sandboxClient.createSandbox(session, sandboxSpec, identity)), {
187
+ name: "sandbox.create",
188
+ });
189
+ if (spawned.isErr())
190
+ return suspendStep(suspensionOfSpawnRefusal(spawned.error));
191
+ const sandbox = spawned.value;
190
192
  // Recovery path: re-check `isAlive` on the persisted ref before continuing.
191
193
  // A classified-dead sandbox triggers a fresh `createSandbox` (task 4.6).
192
194
  // For first execution this is a cheap no-op; on replay it's the seam
@@ -371,52 +373,9 @@ export async function runSandboxStepBody(input, deps) {
371
373
  stepUsage = agentResult.finish.usage;
372
374
  }
373
375
  catch (err) {
374
- if (isBudgetExceeded(err)) {
375
- // Notify the parent BEFORE self-cancel — DBOSWorkflowCancelledError
376
- // surfaced by `getResult` carries a generic message that
377
- // `isBudgetExceeded` cannot match, so without this side-channel the
378
- // parent classifier falls through to fail-fast and the 402 pause
379
- // cascade never fires. Wrapped in `DBOS.runStep` for replay
380
- // idempotency (DBOS persists the send under (workflowID, function_id),
381
- // a recovered child does not re-send). Failure to send is logged but
382
- // swallowed: the worst case is the parent misclassifies the cancel
383
- // as fail-fast, which is the pre-fix behaviour and still safe.
384
- const errMsg = err instanceof Error ? err.message : String(err);
385
- const notification = {
386
- childWorkflowId,
387
- stepId: input.stepId,
388
- error: errMsg,
389
- };
390
- try {
391
- await DBOS.runStep(() => DBOS.send(input.parentWorkflowId, notification, BUDGET_EXCEEDED_TOPIC), { name: "notify-parent-budget-exceeded" });
392
- }
393
- catch (sendErr) {
394
- logger.warn("notify-parent-budget-exceeded failed (non-fatal)", {
395
- ...logger.errorFields(sendErr),
396
- });
397
- }
398
- // Self-cancel to CANCELLED so the parent's `getResult` observes the
399
- // cancellation and triggers the 402 pause cascade. ERROR is NOT
400
- // resumable (NOTES #3); CANCELLED is what `resumeWorkflow` reads.
401
- // The subsequent runStep flips the cortex_step_executions row to
402
- // canceled and raises `DBOSWorkflowCancelledError` — nothing past
403
- // that line executes. The parent's classifier reads the budget
404
- // notification from the side-channel set populated above and
405
- // observes the cancel terminal state via `getResult`.
406
- await DBOS.cancelWorkflow(childWorkflowId);
407
- const durationMs = (await DBOS.now()) - startedAt;
408
- await DBOS.runStep(async () => {
409
- unwrapOrThrow(await updateStepExecution(deps.pool, input.runId, input.stepId, {
410
- status: "canceled",
411
- durationMs,
412
- error: "budget_exceeded",
413
- attempts: 1,
414
- lastErrorClass: "budget_exceeded",
415
- }));
416
- }, { name: "mark-canceled" });
417
- // Unreachable — the runStep above raises DBOSWorkflowCancelledError.
418
- throw new Error("unreachable: mark-canceled did not raise", { cause: err });
419
- }
376
+ const suspension = suspensionOfFailure(err);
377
+ if (suspension !== undefined)
378
+ return suspendStep(suspension);
420
379
  // Non-resumable failure — tear down, mark failed (scrubbed), re-raise so
421
380
  // the parent's `getResult` sees ERROR and the fail-fast cascade fires.
422
381
  // `failStep` always throws; the outer `throw` is for control-flow analysis.
@@ -524,67 +483,52 @@ export async function runSandboxStepBody(input, deps) {
524
483
  const summary = await safeRunValue(logger, () => DBOS.runStep(() => generateStepSummaryAndWrite(deps, postCtx, manifest), {
525
484
  name: "post-step.generate-step-summary",
526
485
  }), "post-step.summary", undefined);
527
- // Lineage attestation (registration + sync) is fail-fast (see the artifact-manifest spec): unlike
486
+ // Lineage attestation (registration) is fail-fast (see the artifact-manifest spec): unlike
528
487
  // the best-effort enrichment stages, a failure here orphans the step's real
529
488
  // outputs, so it fails the step loudly instead of finishing green. Transient
530
- // managed-root errors are already retried in the client, so anything thrown here is
489
+ // managed-root errors are already retried in the client, so a failure here is
531
490
  // persistent — tear down, mark failed, and re-raise so the parent's fail-fast
532
- // cascade fires (mirrors the agent-loop failure path). The two operations are
533
- // sequenced independently: byte-sync is attempted on both arms, so a rejected
534
- // registration still uploads the bytes of whatever DID register.
491
+ // cascade fires (mirrors the agent-loop failure path). The byte-sync is
492
+ // attempted after the registration on every arm, so a rejected registration
493
+ // still uploads the bytes of whatever DID register: the rows that registered
494
+ // carry `artifact_id` with `file_id` still NULL, which is exactly what
495
+ // `queryUnsyncedStepArtifacts` selects, and a row that never registered is not
496
+ // selected, so the sync stays idempotent under re-execution.
535
497
  await emitActivity("persisting", "Registering artifacts");
536
498
  // A bare call, deliberately not `DBOS.runStep`-wrapped: it is reachable from
537
499
  // two arms below, and a checkpointed step reached from a conditional arm
538
500
  // would shift the body's function-ID sequence between the two paths (see the
539
- // harness-durable-runtime spec).
540
- const syncStepArtifacts = () => deps.artifactRegistry.sync({
501
+ // harness-durable-runtime spec). The sync is a notice: it never displaces a
502
+ // registration failure as the cause of one.
503
+ const syncStepArtifacts = () => deliverNotice(logger.named("post-step.sync"), "ArtifactRegistry.sync", deps.artifactRegistry.sync({
541
504
  resourceId: input.analysisId,
542
505
  runId: input.runId,
543
506
  stepId: input.stepId,
544
- }, session);
545
- let reconciledManifest;
507
+ }, session));
508
+ let registered;
546
509
  try {
547
- reconciledManifest = await reconcileAndRegisterStepArtifacts(deps, postCtx, manifest);
548
- }
549
- catch (registrationErr) {
550
- // Registration and byte-sync are independent, and a registry rejection is
551
- // partial: the rows that DID register carry `artifact_id` with `file_id`
552
- // still NULL, which is exactly what `queryUnsyncedStepArtifacts` selects.
553
- // Syncing anyway uploads their bytes (rows that never registered are not
554
- // selected, so this stays idempotent under re-execution); skipping it is
555
- // what strands them registered-but-byteless.
556
- try {
557
- await syncStepArtifacts();
558
- }
559
- catch (syncErr) {
560
- // A cancellation is control-flow, not a sync failure — re-raise it
561
- // verbatim, as every other stage in this body does. Any real failure
562
- // is swallowed so it cannot displace the registration rejection as the
563
- // step's cause; this record is the only account of it.
564
- if (syncErr instanceof DBOSErrors.DBOSWorkflowCancelledError)
565
- throw syncErr;
566
- logger.named("post-step.sync").warn("failed after a registration rejection (non-fatal)", logger.errorFields(syncErr));
567
- }
568
- throw await failStep("lineage_attestation", registrationErr);
510
+ registered = await reconcileAndRegisterStepArtifacts(deps, postCtx, manifest);
569
511
  }
570
- try {
512
+ catch (attestationErr) {
513
+ if (attestationErr instanceof DBOSErrors.DBOSWorkflowCancelledError)
514
+ throw attestationErr;
571
515
  await syncStepArtifacts();
516
+ throw await failStep("lineage_attestation", attestationErr);
572
517
  }
573
- catch (err) {
574
- throw await failStep("lineage_attestation", err);
518
+ await syncStepArtifacts();
519
+ if (registered.isErr()) {
520
+ const refused = registered.error.kind === "refused" ? suspensionOfRefusal(registered.error.refusal) : undefined;
521
+ if (refused !== undefined)
522
+ return suspendStep(refused);
523
+ throw await failStep("lineage_attestation", new Error(describeStepRegistrationFailure(registered.error)));
575
524
  }
525
+ const reconciledManifest = registered.value;
576
526
  const postArtifacts = {
577
527
  metadataEntries,
578
528
  summary,
579
529
  reconciledManifest,
580
530
  };
581
- let stepOutputs;
582
- try {
583
- stepOutputs = collectStepOutputs(postCtx, postArtifacts);
584
- }
585
- catch (err) {
586
- logger.warn("post-step.collect failed (non-fatal)", logger.errorFields(err));
587
- }
531
+ const stepOutputs = collectStepOutputs(postCtx, postArtifacts);
588
532
  await emitActivity("indexing", "Indexing outputs for search");
589
533
  await safeRun(logger, () => vectorIndexStepOutputs(deps, postCtx, postArtifacts), "post-step.vector-index");
590
534
  // (9) teardown — destroy sandbox + clear sandbox_ref.
@@ -0,0 +1,32 @@
1
+ /**
2
+ * The suspension of work (workflow-suspension spec).
3
+ *
4
+ * A suspension starts only from a model request that fails with a `suspend`
5
+ * error, or from a gate that refuses with the suspend flag. The harness carries
6
+ * the reason of the host as it is and never reads it: no path compares a
7
+ * reason to a value, and no path reads the text of an error message.
8
+ *
9
+ * The owner of the result of a workflow selects the mechanism. A durable owner
10
+ * (`executeAnalysis`, `sandbox-step`, `data-profile`) records the reason in its
11
+ * stored state and ends in the DBOS state `CANCELLED`, which `resumeWorkflow`
12
+ * reads. A workflow with a live caller (`derive-table-exec`, `extract-values`)
13
+ * returns the suspension as the `err` of its result. A suspension of an
14
+ * analysis run or of a data profile marks the analysis through
15
+ * `suspendAnalysis` (`state/analyses.ts`). A workflow with a live caller does
16
+ * not mark it: the conversation agent reads the suspension as a tool result.
17
+ */
18
+ import type { GateRefusal } from "../lib/hooks.js";
19
+ import type { SuspendingRefusal } from "../sandbox/sandbox-error.js";
20
+ /** Plain data, thus it crosses a DBOS checkpoint as it is. */
21
+ export interface Suspension {
22
+ readonly kind: "suspended";
23
+ readonly reason: string;
24
+ }
25
+ export declare function suspensionOfFailure(err: unknown): Suspension | undefined;
26
+ export declare function suspensionOfRefusal(refusal: GateRefusal): Suspension | undefined;
27
+ export declare function suspensionOfSpawnRefusal(refusal: SuspendingRefusal): Suspension;
28
+ /**
29
+ * The cancel lands at the next DBOS operation, thus the named step raises
30
+ * `DBOSWorkflowCancelledError` and nothing after it runs.
31
+ */
32
+ export declare function cancelSelf(stepName: string): Promise<never>;
@@ -0,0 +1,38 @@
1
+ /**
2
+ * The suspension of work (workflow-suspension spec).
3
+ *
4
+ * A suspension starts only from a model request that fails with a `suspend`
5
+ * error, or from a gate that refuses with the suspend flag. The harness carries
6
+ * the reason of the host as it is and never reads it: no path compares a
7
+ * reason to a value, and no path reads the text of an error message.
8
+ *
9
+ * The owner of the result of a workflow selects the mechanism. A durable owner
10
+ * (`executeAnalysis`, `sandbox-step`, `data-profile`) records the reason in its
11
+ * stored state and ends in the DBOS state `CANCELLED`, which `resumeWorkflow`
12
+ * reads. A workflow with a live caller (`derive-table-exec`, `extract-values`)
13
+ * returns the suspension as the `err` of its result. A suspension of an
14
+ * analysis run or of a data profile marks the analysis through
15
+ * `suspendAnalysis` (`state/analyses.ts`). A workflow with a live caller does
16
+ * not mark it: the conversation agent reads the suspension as a tool result.
17
+ */
18
+ import { DBOS } from "@dbos-inc/dbos-sdk";
19
+ import { findSuspendError } from "../providers/errors.js";
20
+ export function suspensionOfFailure(err) {
21
+ const found = findSuspendError(err);
22
+ return found === undefined ? undefined : { kind: "suspended", reason: found.reason };
23
+ }
24
+ export function suspensionOfRefusal(refusal) {
25
+ return refusal.kind === "suspended" ? { kind: "suspended", reason: refusal.reason } : undefined;
26
+ }
27
+ export function suspensionOfSpawnRefusal(refusal) {
28
+ return { kind: "suspended", reason: refusal.reason };
29
+ }
30
+ /**
31
+ * The cancel lands at the next DBOS operation, thus the named step raises
32
+ * `DBOSWorkflowCancelledError` and nothing after it runs.
33
+ */
34
+ export async function cancelSelf(stepName) {
35
+ await DBOS.cancelWorkflow(DBOS.workflowID);
36
+ await DBOS.runStep(async () => undefined, { name: stepName });
37
+ throw new Error(`unreachable: ${stepName} did not raise the cancellation`);
38
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@inflexa-ai/harness",
3
- "version": "0.37.0",
3
+ "version": "0.39.0",
4
4
  "description": "Host-agnostic bioinformatics agent harness — hand-rolled agent loop, DBOS-durable workflows, sandboxed compute.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -1,6 +0,0 @@
1
- import type { ResolveBilling } from "./resolver.js";
2
- /**
3
- * Local/OSS billing resolver: resolves to an empty header map. No upstream
4
- * call, no credential reads. Wire calls carry no attribution headers.
5
- */
6
- export declare function createNoopBillingResolver(): ResolveBilling;
@@ -1,7 +0,0 @@
1
- /**
2
- * Local/OSS billing resolver: resolves to an empty header map. No upstream
3
- * call, no credential reads. Wire calls carry no attribution headers.
4
- */
5
- export function createNoopBillingResolver() {
6
- return async () => ({});
7
- }
@@ -1,64 +0,0 @@
1
- /**
2
- * Billing attribution seam.
3
- *
4
- * Defines the `ResolveBilling` contract the provider calls at the
5
- * LLM/embedding call site, plus the session view and fetcher types a
6
- * concrete resolver implements. The provider calls `resolveBilling(session)`
7
- * just before the wire call; read-only routes never call it, so they never
8
- * incur a billing round-trip.
9
- *
10
- * This module is the vendor-neutral seam: it declares types only. A concrete
11
- * resolver (managed/upstream-backed or noop) supplies the fetch + assembly
12
- * behavior behind the `ResolveBilling` function type.
13
- */
14
- import type { AuthContext, BillingSessionView, Credential, ResourceCoordinates } from "../auth/types.js";
15
- export type BillingMap = Record<string, string>;
16
- /** Final wire-format header map sent to the upstream gateway. */
17
- export type BillingHeaders = Record<string, string>;
18
- /**
19
- * Result of a single billing-attribution fetch. The seam's own type — a
20
- * concrete resolver's richer result (e.g. a generic upstream envelope)
21
- * structurally satisfies it. The failure branch carries what the resolver
22
- * needs to raise a `BillingResolutionError`.
23
- */
24
- export type BillingFetchResult = {
25
- ok: true;
26
- data: BillingMap;
27
- } | {
28
- ok: false;
29
- status: number;
30
- error: {
31
- error: string;
32
- message: string;
33
- };
34
- };
35
- /**
36
- * The minimal session view a resolver reads. The cached map is keyed by
37
- * `(user, resourceType, resourceId)` (from `identity`/`scope` on
38
- * `BillingSessionView`); header assembly additionally consumes
39
- * `provenance`/`runFrame` (per-call). The fetch resolves the credential
40
- * + org via `getAuth(session)`, so only the opaque `auth` is needed here —
41
- * `AgentSession` structurally satisfies it.
42
- */
43
- export interface ResolvableSession extends BillingSessionView {
44
- readonly auth: AuthContext;
45
- }
46
- /**
47
- * Resolve the FINAL wire headers at the call site. The raw attribution map is
48
- * fetched once per `(user, resource)` and cached; the per-call identity tags
49
- * (`agentId`/`runId`/`stepId`) vary within that key, so header assembly runs
50
- * POST-CACHE on every call against the live `session` — assembled headers are
51
- * never cached.
52
- */
53
- export type ResolveBilling = (session: ResolvableSession) => Promise<BillingHeaders>;
54
- /**
55
- * The upstream call seam — overridable in tests with a fake. A concrete
56
- * resolver resolves the attribution map for the given credential/org/coords.
57
- */
58
- export type BillingFetcher = (credential: Credential, orgId: string, coords: ResourceCoordinates) => Promise<BillingFetchResult>;
59
- /** A billing-resolution failure (e.g. 403/404). Fails the LLM call. */
60
- export declare class BillingResolutionError extends Error {
61
- readonly status: number;
62
- readonly code: string;
63
- constructor(status: number, code: string, message: string);
64
- }
@@ -1,24 +0,0 @@
1
- /**
2
- * Billing attribution seam.
3
- *
4
- * Defines the `ResolveBilling` contract the provider calls at the
5
- * LLM/embedding call site, plus the session view and fetcher types a
6
- * concrete resolver implements. The provider calls `resolveBilling(session)`
7
- * just before the wire call; read-only routes never call it, so they never
8
- * incur a billing round-trip.
9
- *
10
- * This module is the vendor-neutral seam: it declares types only. A concrete
11
- * resolver (managed/upstream-backed or noop) supplies the fetch + assembly
12
- * behavior behind the `ResolveBilling` function type.
13
- */
14
- /** A billing-resolution failure (e.g. 403/404). Fails the LLM call. */
15
- export class BillingResolutionError extends Error {
16
- status;
17
- code;
18
- constructor(status, code, message) {
19
- super(`Billing resolution failed (${status} ${code}): ${message}`);
20
- this.status = status;
21
- this.code = code;
22
- this.name = "BillingResolutionError";
23
- }
24
- }