@inflexa-ai/harness 0.37.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/auth/local-run-authorizer.js +5 -4
- package/dist/auth/types.d.ts +28 -9
- package/dist/auth/types.js +5 -3
- package/dist/billing/noop-run-charge.js +3 -2
- package/dist/billing/noop-usage-recorder.js +2 -1
- package/dist/billing/run-charge.d.ts +18 -5
- package/dist/billing/run-charge.js +5 -1
- package/dist/billing/usage-recorder.d.ts +11 -9
- package/dist/billing/usage-recorder.js +2 -3
- package/dist/contracts/chat-events.d.ts +4 -1
- package/dist/contracts/chat-parts.d.ts +1 -1
- package/dist/contracts/schemas/chat-parts.d.ts +4 -4
- package/dist/execution/artifact-registration.d.ts +6 -1
- package/dist/execution/artifact-registration.js +12 -4
- package/dist/execution/artifact-registry.d.ts +10 -3
- package/dist/execution/noop-artifact-registry.js +3 -4
- package/dist/execution/post-step-pipeline.d.ts +14 -3
- package/dist/execution/post-step-pipeline.js +21 -8
- package/dist/execution/run-authorizer.d.ts +5 -3
- package/dist/execution/run-canceler.js +4 -20
- package/dist/index.d.ts +13 -7
- package/dist/index.js +12 -6
- package/dist/lib/hooks.d.ts +49 -0
- package/dist/lib/hooks.js +39 -0
- package/dist/lib/result.d.ts +22 -11
- package/dist/lib/result.js +24 -14
- package/dist/loop/run-agent.d.ts +2 -2
- package/dist/loop/run-agent.js +5 -4
- package/dist/provenance/seam.d.ts +1 -1
- package/dist/providers/ai-sdk.d.ts +43 -5
- package/dist/providers/ai-sdk.js +55 -52
- package/dist/providers/anthropic.d.ts +2 -3
- package/dist/providers/anthropic.js +2 -2
- package/dist/providers/embedding.d.ts +13 -7
- package/dist/providers/embedding.js +26 -6
- package/dist/providers/errors.d.ts +67 -37
- package/dist/providers/errors.js +57 -58
- package/dist/providers/request-headers.d.ts +18 -0
- package/dist/providers/request-headers.js +22 -0
- package/dist/report-model/production-resolver.d.ts +5 -2
- package/dist/report-model/production-resolver.js +12 -2
- package/dist/runtime/assemble.d.ts +3 -2
- package/dist/runtime/dbos.d.ts +4 -3
- package/dist/runtime/dbos.js +18 -3
- package/dist/runtime/result-serialization.d.ts +29 -0
- package/dist/runtime/result-serialization.js +45 -0
- package/dist/sandbox/client.d.ts +11 -3
- package/dist/sandbox/create-sandbox.d.ts +5 -2
- package/dist/sandbox/create-sandbox.js +51 -25
- package/dist/sandbox/docker-client.d.ts +4 -3
- package/dist/sandbox/docker-client.js +23 -31
- package/dist/sandbox/k8s-client.d.ts +4 -13
- package/dist/sandbox/k8s-client.js +28 -61
- package/dist/sandbox/labels.d.ts +34 -0
- package/dist/sandbox/labels.js +47 -0
- package/dist/sandbox/mount-plan.d.ts +3 -1
- package/dist/sandbox/mount-plan.js +9 -0
- package/dist/sandbox/sandbox-error.d.ts +36 -13
- package/dist/sandbox/sandbox-error.js +32 -17
- package/dist/sandbox/types.d.ts +13 -9
- package/dist/schemas/validate-plan.js +13 -0
- package/dist/schemas/workflow-state.d.ts +4 -4
- package/dist/state/analyses.d.ts +4 -4
- package/dist/state/analyses.js +4 -4
- package/dist/state/schema.d.ts +5 -5
- package/dist/tasks/data-profile.d.ts +5 -8
- package/dist/tasks/data-profile.js +88 -43
- package/dist/tasks/derive-table-exec.d.ts +10 -6
- package/dist/tasks/derive-table-exec.js +17 -12
- package/dist/tasks/extract-values.d.ts +16 -10
- package/dist/tasks/extract-values.js +34 -18
- package/dist/tools/execute-analysis.js +22 -15
- package/dist/tools/report-session/derive-table.d.ts +9 -1
- package/dist/tools/report-session/derive-table.js +25 -22
- package/dist/tools/research/inspect-run.d.ts +1 -1
- package/dist/workflows/execute-analysis.d.ts +25 -16
- package/dist/workflows/execute-analysis.js +223 -216
- package/dist/workflows/metrics.d.ts +4 -3
- package/dist/workflows/metrics.js +4 -3
- package/dist/workflows/sandbox-step.d.ts +20 -29
- package/dist/workflows/sandbox-step.js +71 -127
- package/dist/workflows/suspension.d.ts +32 -0
- package/dist/workflows/suspension.js +38 -0
- package/package.json +1 -1
- package/dist/billing/noop-resolver.d.ts +0 -6
- package/dist/billing/noop-resolver.js +0 -7
- package/dist/billing/resolver.d.ts +0 -64
- package/dist/billing/resolver.js +0 -24
- package/dist/loop/budget-exceeded.d.ts +0 -38
- package/dist/loop/budget-exceeded.js +0 -88
package/dist/providers/errors.js
CHANGED
|
@@ -2,18 +2,20 @@
|
|
|
2
2
|
* Provider error taxonomy.
|
|
3
3
|
*
|
|
4
4
|
* Callers (the agent loop, the chat route, DBOS steps) branch on error
|
|
5
|
-
* origin: a
|
|
5
|
+
* origin: a status that the host maps to a suspension stops the work; a
|
|
6
6
|
* transient upstream hiccup is retryable. This module owns the
|
|
7
7
|
* classification so callers do not re-derive it.
|
|
8
8
|
*
|
|
9
|
-
* The
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
9
|
+
* The classification keys on the HTTP status alone. The meaning of a status
|
|
10
|
+
* belongs to the gateway of the host, thus each provider takes a map from a
|
|
11
|
+
* status to a suspend reason (`suspendOn`). A status in the map is a `suspend`
|
|
12
|
+
* error, before each other rule, and it is never retried. The harness carries
|
|
13
|
+
* the reason and never reads it. Provider-originated failures (429, 5xx,
|
|
14
|
+
* connection) surface with their own status or as a network error with no
|
|
15
|
+
* status at all. A `401` is `auth`: the credential the host put behind the call
|
|
16
|
+
* is expired, revoked, or absent, and only a human re-authenticating fixes it —
|
|
17
|
+
* which is a different remedy from every other non-retryable 4xx, so it is a
|
|
18
|
+
* different kind.
|
|
17
19
|
*/
|
|
18
20
|
import { redactSecrets } from "../input-sanitization.js";
|
|
19
21
|
/**
|
|
@@ -34,6 +36,8 @@ export class RequestTimeoutError extends Error {
|
|
|
34
36
|
this.requestTimeoutMs = requestTimeoutMs;
|
|
35
37
|
}
|
|
36
38
|
}
|
|
39
|
+
/** The map of a provider that the host gives no map: `402` names a payment that the gateway requires. */
|
|
40
|
+
export const DEFAULT_SUSPEND_ON = { 402: "payment_required" };
|
|
37
41
|
/**
|
|
38
42
|
* Structural guard for an already-constructed `ProviderError` value. A
|
|
39
43
|
* `ProviderError` is a plain object, not an `Error`, so it cannot be
|
|
@@ -44,16 +48,27 @@ export function isProviderError(value) {
|
|
|
44
48
|
if (typeof value !== "object" || value === null)
|
|
45
49
|
return false;
|
|
46
50
|
const v = value;
|
|
47
|
-
return (
|
|
48
|
-
|
|
49
|
-
|
|
51
|
+
return (v.type === "auth" || v.type === "suspend" || v.type === "provider") && typeof v.retryable === "boolean" && typeof v.message === "string";
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Find a `suspend` error on a failure or on its `cause` chain. A step boundary
|
|
55
|
+
* rethrows a `ProviderError` inside a `ResultError`, with the value on
|
|
56
|
+
* `.cause`, thus the walk reaches it.
|
|
57
|
+
*
|
|
58
|
+
* A failed step that DBOS replays gives the error that DBOS recorded. That
|
|
59
|
+
* record keeps only the enumerable fields of a `ResultError`: `.value` stays,
|
|
60
|
+
* and the non-enumerable `.cause` does not. Thus the walk also reads `.value`.
|
|
61
|
+
*/
|
|
62
|
+
export function findSuspendError(err) {
|
|
63
|
+
const asSuspend = (value) => (isProviderError(value) && value.type === "suspend" ? value : undefined);
|
|
64
|
+
return findInCauseChain(err, (link) => asSuspend(link) ?? (typeof link === "object" && link !== null ? asSuspend(link.value) : undefined));
|
|
50
65
|
}
|
|
51
66
|
/**
|
|
52
67
|
* Turn a caught SDK throwable into a `ProviderError` value. Routes through
|
|
53
|
-
* `classifyProviderError`
|
|
54
|
-
* transient (429 / 5xx / connection)
|
|
55
|
-
*
|
|
56
|
-
*
|
|
68
|
+
* `classifyProviderError` with the `suspendOn` map of the provider, so the
|
|
69
|
+
* `provider` variant's `retryable` keeps the transient (429 / 5xx / connection)
|
|
70
|
+
* classification and a mapped status becomes a `suspend` error. `cause` is the
|
|
71
|
+
* original throwable verbatim.
|
|
57
72
|
*
|
|
58
73
|
* Idempotent: `chatStream` throws a `ProviderError` value, which
|
|
59
74
|
* `streaming-chat`'s `catch` re-wraps by calling this again. Returning an
|
|
@@ -61,7 +76,7 @@ export function isProviderError(value) {
|
|
|
61
76
|
* `String()`-ing the object (a `ProviderError` is not an `Error`) into a
|
|
62
77
|
* `"[object Object]"` message that would bury the real inner one.
|
|
63
78
|
*/
|
|
64
|
-
export function toProviderError(e, workload) {
|
|
79
|
+
export function toProviderError(e, workload, suspendOn = DEFAULT_SUSPEND_ON) {
|
|
65
80
|
if (isProviderError(e))
|
|
66
81
|
return e;
|
|
67
82
|
// A guard abort names the configured value in its message. Compose a message
|
|
@@ -78,9 +93,9 @@ export function toProviderError(e, workload) {
|
|
|
78
93
|
cause: e,
|
|
79
94
|
};
|
|
80
95
|
}
|
|
81
|
-
const
|
|
96
|
+
const classified = classifyProviderError(e, suspendOn);
|
|
82
97
|
const detail = e instanceof Error ? e.message : String(e);
|
|
83
|
-
if (kind === "auth") {
|
|
98
|
+
if (classified.kind === "auth") {
|
|
84
99
|
return {
|
|
85
100
|
type: "auth",
|
|
86
101
|
retryable: false,
|
|
@@ -88,25 +103,11 @@ export function toProviderError(e, workload) {
|
|
|
88
103
|
cause: e,
|
|
89
104
|
};
|
|
90
105
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
cause: e,
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
if (kind === "tenant-blocked") {
|
|
100
|
-
return {
|
|
101
|
-
type: "tenant-blocked",
|
|
102
|
-
retryable: false,
|
|
103
|
-
message: `Billing gateway blocked tenant for ${workload}: ${detail}`,
|
|
104
|
-
cause: e,
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
// The `provider` arm composes like its three siblings rather than forwarding
|
|
108
|
-
// the SDK message verbatim. It has to: when a 4xx body does not parse against
|
|
109
|
-
// the configured provider's error schema, the AI SDK falls back to
|
|
106
|
+
// The `provider` and `suspend` arms compose one generic HTTP message rather
|
|
107
|
+
// than forwarding the SDK message verbatim. A `suspend` message names no
|
|
108
|
+
// cause of its own: the reason of the host rides beside it. The composition
|
|
109
|
+
// has to happen: when a 4xx body does not parse against the configured
|
|
110
|
+
// provider's error schema, the AI SDK falls back to
|
|
110
111
|
// `response.statusText`, so the "detail" is a bare HTTP reason phrase —
|
|
111
112
|
// `"Bad Request"` — that names neither the call nor the cause, while the
|
|
112
113
|
// status, the workload, and the body the SDK captured are all still on the
|
|
@@ -120,18 +121,17 @@ export function toProviderError(e, workload) {
|
|
|
120
121
|
// A transport that carries no reason phrase (an HTTP/2 hop yields
|
|
121
122
|
// `statusText === ""`) leaves `detail` empty; appending it regardless would
|
|
122
123
|
// trail a bare `": "` on an otherwise complete message.
|
|
123
|
-
const
|
|
124
|
-
const diagnosed =
|
|
124
|
+
const phrase = detail.trim();
|
|
125
|
+
const diagnosed = phrase ? `${lead}: ${phrase}` : lead;
|
|
125
126
|
const body = extractResponseBody(e);
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
};
|
|
127
|
+
// The excerpt trails deliberately — workload and status lead, so a
|
|
128
|
+
// downstream truncation (`profileFailureReason` cuts the line at 200)
|
|
129
|
+
// eats the least diagnostic content first.
|
|
130
|
+
const message = body ? `${diagnosed} — response body: ${excerptResponseBody(body)}` : diagnosed;
|
|
131
|
+
if (classified.kind === "suspend") {
|
|
132
|
+
return { type: "suspend", retryable: false, reason: classified.reason, status: classified.status, message, cause: e };
|
|
133
|
+
}
|
|
134
|
+
return { type: "provider", retryable: classified.retryable, message, cause: e };
|
|
135
135
|
}
|
|
136
136
|
/**
|
|
137
137
|
* Max characters of a captured provider response body carried in a message.
|
|
@@ -284,13 +284,13 @@ function looksLikeConnectionError(err) {
|
|
|
284
284
|
/**
|
|
285
285
|
* Classify a provider failure by origin.
|
|
286
286
|
*
|
|
287
|
+
* - A status in `suspendOn` → `suspend` with the reason of the map, not
|
|
288
|
+
* retryable. This rule comes before each other status rule.
|
|
287
289
|
* - Provider `401` → `auth`, not retryable.
|
|
288
|
-
* - Billing-gateway `402` → `budget`, not retryable.
|
|
289
|
-
* - Billing-gateway `403` → `tenant-blocked`, not retryable.
|
|
290
290
|
* - Provider `429` / `5xx` / connection errors → `provider`, retryable.
|
|
291
291
|
* - Other `4xx` and parse / unknown errors → `provider`, not retryable.
|
|
292
292
|
*/
|
|
293
|
-
export function classifyProviderError(e) {
|
|
293
|
+
export function classifyProviderError(e, suspendOn = DEFAULT_SUSPEND_ON) {
|
|
294
294
|
// The request-timeout guard aborts an attempt with a typed sentinel, and the
|
|
295
295
|
// chunk bound of the SDK aborts one with a `TimeoutError`. Each classifies as
|
|
296
296
|
// a retryable provider timeout, thus the envelope retries it under the same
|
|
@@ -300,6 +300,9 @@ export function classifyProviderError(e) {
|
|
|
300
300
|
if (findRequestTimeout(e) !== undefined || findSdkTimeout(e) !== undefined)
|
|
301
301
|
return { kind: "provider", retryable: true };
|
|
302
302
|
const status = extractStatus(e);
|
|
303
|
+
if (status !== undefined && Object.hasOwn(suspendOn, status)) {
|
|
304
|
+
return { kind: "suspend", retryable: false, reason: suspendOn[status], status };
|
|
305
|
+
}
|
|
303
306
|
// `auth` is its own kind rather than a plain non-retryable 4xx because the
|
|
304
307
|
// remedy is categorically different: the request was well-formed and the
|
|
305
308
|
// credential behind it is not, so no amount of re-issuing or rephrasing
|
|
@@ -307,16 +310,12 @@ export function classifyProviderError(e) {
|
|
|
307
310
|
// 4xx branch below would tell them the request was wrong, which is false.
|
|
308
311
|
if (status === 401)
|
|
309
312
|
return { kind: "auth", retryable: false };
|
|
310
|
-
if (status === 402)
|
|
311
|
-
return { kind: "budget", retryable: false };
|
|
312
|
-
if (status === 403)
|
|
313
|
-
return { kind: "tenant-blocked", retryable: false };
|
|
314
313
|
if (status === 429 || (status !== undefined && status >= 500)) {
|
|
315
314
|
return { kind: "provider", retryable: true };
|
|
316
315
|
}
|
|
317
316
|
if (status !== undefined) {
|
|
318
|
-
// A concrete 4xx
|
|
319
|
-
// it unchanged will fail again.
|
|
317
|
+
// A concrete 4xx that the map does not hold — the request is wrong;
|
|
318
|
+
// retrying it unchanged will fail again.
|
|
320
319
|
return { kind: "provider", retryable: false };
|
|
321
320
|
}
|
|
322
321
|
// No status: a transport failure is retryable; anything else (parse
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model request headers hook.
|
|
3
|
+
*
|
|
4
|
+
* The hook is a gate (`lib/hooks.ts`): the provider calls it before each
|
|
5
|
+
* attempt of a model request, and it adds the headers to the request as the
|
|
6
|
+
* hook gives them. The harness reads no header and makes no header of its own.
|
|
7
|
+
*
|
|
8
|
+
* The hook is not the model wire, thus a refusal stops the call at once, with
|
|
9
|
+
* no retry, and the refused attempt is never sent.
|
|
10
|
+
*/
|
|
11
|
+
import { type ResultAsync } from "neverthrow";
|
|
12
|
+
import type { ResolvableSession } from "../auth/types.js";
|
|
13
|
+
import { type GateFailure, type GateRefusal } from "../lib/hooks.js";
|
|
14
|
+
import type { ProviderError } from "./errors.js";
|
|
15
|
+
export type RequestHeaders = Readonly<Record<string, string>>;
|
|
16
|
+
export type ResolveRequestHeaders = (session: ResolvableSession) => ResultAsync<RequestHeaders, GateFailure>;
|
|
17
|
+
export declare function requestHeadersFor(hook: ResolveRequestHeaders | undefined, session: ResolvableSession): ResultAsync<RequestHeaders, GateRefusal>;
|
|
18
|
+
export declare function headersRefusalError(refusal: GateRefusal, workload: string): ProviderError;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model request headers hook.
|
|
3
|
+
*
|
|
4
|
+
* The hook is a gate (`lib/hooks.ts`): the provider calls it before each
|
|
5
|
+
* attempt of a model request, and it adds the headers to the request as the
|
|
6
|
+
* hook gives them. The harness reads no header and makes no header of its own.
|
|
7
|
+
*
|
|
8
|
+
* The hook is not the model wire, thus a refusal stops the call at once, with
|
|
9
|
+
* no retry, and the refused attempt is never sent.
|
|
10
|
+
*/
|
|
11
|
+
import { okAsync } from "neverthrow";
|
|
12
|
+
import { passGate } from "../lib/hooks.js";
|
|
13
|
+
const NO_HEADERS = {};
|
|
14
|
+
export function requestHeadersFor(hook, session) {
|
|
15
|
+
return hook === undefined ? okAsync(NO_HEADERS) : passGate("resolveRequestHeaders", hook(session));
|
|
16
|
+
}
|
|
17
|
+
export function headersRefusalError(refusal, workload) {
|
|
18
|
+
const message = `The request headers hook refused the model call for ${workload}: ${refusal.reason}`;
|
|
19
|
+
return refusal.kind === "suspended"
|
|
20
|
+
? { type: "suspend", retryable: false, reason: refusal.reason, message }
|
|
21
|
+
: { type: "provider", retryable: false, message };
|
|
22
|
+
}
|
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
* and the reason choices of `fixture-resolver.ts`. Thus one semantics exists, and the fixture stays the
|
|
18
18
|
* executable specification of the value tier. The two realizations are substitutable behind the seam.
|
|
19
19
|
*/
|
|
20
|
+
import { type Result } from "neverthrow";
|
|
21
|
+
import type { Suspension } from "../workflows/suspension.js";
|
|
20
22
|
import { type ReferenceResolver } from "./reference-resolver.js";
|
|
21
23
|
/**
|
|
22
24
|
* One artifact that the host fast path could not read, addressed for the extraction arm. The `hash` pins
|
|
@@ -37,13 +39,14 @@ export interface ExtractionArtifact {
|
|
|
37
39
|
/**
|
|
38
40
|
* The out-of-process read arm. One `extract` call covers every fall-through artifact of one document
|
|
39
41
|
* pass. The result maps each requested path to its rows. A path that the arm could not read is absent
|
|
40
|
-
* from the map, and the reference at that path fails as `unreadable-artifact`.
|
|
42
|
+
* from the map, and the reference at that path fails as `unreadable-artifact`. A suspension of the pass is
|
|
43
|
+
* the `err`, and each reference of the batch reports its reason.
|
|
41
44
|
*
|
|
42
45
|
* The arm is a seam, thus a test stubs it. While no realization is wired, a fall-through reference fails
|
|
43
46
|
* with `extraction-unavailable`, and the detail names the absent arm.
|
|
44
47
|
*/
|
|
45
48
|
export interface ExtractionArm {
|
|
46
|
-
extract(requests: readonly ExtractionRequest[]): Promise<ReadonlyMap<string, ExtractionArtifact>>;
|
|
49
|
+
extract(requests: readonly ExtractionRequest[]): Promise<Result<ReadonlyMap<string, ExtractionArtifact>, Suspension>>;
|
|
47
50
|
}
|
|
48
51
|
/**
|
|
49
52
|
* The disk facts of one artifact that outlive one resolver.
|
|
@@ -437,9 +437,9 @@ async function extractFallThrough(arm, requests) {
|
|
|
437
437
|
}
|
|
438
438
|
return out;
|
|
439
439
|
}
|
|
440
|
-
let
|
|
440
|
+
let answered;
|
|
441
441
|
try {
|
|
442
|
-
|
|
442
|
+
answered = await arm.extract(requests.map((request) => ({ path: request.path, hash: request.hash, format: request.format })));
|
|
443
443
|
}
|
|
444
444
|
catch {
|
|
445
445
|
// The arm speaks the throw protocol for a genuine infrastructure fault. This boundary turns the
|
|
@@ -449,6 +449,16 @@ async function extractFallThrough(arm, requests) {
|
|
|
449
449
|
}
|
|
450
450
|
return out;
|
|
451
451
|
}
|
|
452
|
+
if (answered.isErr()) {
|
|
453
|
+
for (const request of requests) {
|
|
454
|
+
out.set(request.path, {
|
|
455
|
+
kind: "unavailable",
|
|
456
|
+
detail: `the analysis is suspended (${answered.error.reason}), thus the file at ${request.path} was not read`,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
return out;
|
|
460
|
+
}
|
|
461
|
+
const answers = answered.value;
|
|
452
462
|
for (const request of requests) {
|
|
453
463
|
const answer = answers.get(request.path);
|
|
454
464
|
out.set(request.path, answer === undefined
|
|
@@ -25,6 +25,7 @@ import { type AcquireEyes } from "../lib/eyes.js";
|
|
|
25
25
|
import type { ThreadType } from "../memory/thread-store.js";
|
|
26
26
|
import { type ExecuteAnalysisDeps, type ExecuteAnalysisInput, type ExecuteAnalysisResult } from "../workflows/execute-analysis.js";
|
|
27
27
|
import { type SandboxStepDeps, type SandboxStepInput, type SandboxStepResult } from "../workflows/sandbox-step.js";
|
|
28
|
+
import type { Suspension } from "../workflows/suspension.js";
|
|
28
29
|
import { type DataProfileDeps, type DataProfileWorkflowInput } from "../tasks/data-profile.js";
|
|
29
30
|
import { type ExtractValuesResult, type ExtractValuesWorkflowInput } from "../tasks/extract-values.js";
|
|
30
31
|
import type { DeriveTableExecInput } from "../tools/report-session/derive-table.js";
|
|
@@ -60,8 +61,8 @@ export interface RegisteredWorkflows {
|
|
|
60
61
|
readonly executeAnalysis: (input: ExecuteAnalysisInput) => Promise<ExecuteAnalysisResult>;
|
|
61
62
|
readonly sandboxStep: SandboxStepCallable;
|
|
62
63
|
readonly dataProfile: (input: DataProfileWorkflowInput) => Promise<void>;
|
|
63
|
-
readonly extractValues: (input: ExtractValuesWorkflowInput) => Promise<ExtractValuesResult
|
|
64
|
-
readonly deriveTableExec: (input: DeriveTableExecInput) => Promise<ExecResult
|
|
64
|
+
readonly extractValues: (input: ExtractValuesWorkflowInput) => Promise<Result<ExtractValuesResult, Suspension>>;
|
|
65
|
+
readonly deriveTableExec: (input: DeriveTableExecInput) => Promise<Result<ExecResult, Suspension>>;
|
|
65
66
|
}
|
|
66
67
|
/**
|
|
67
68
|
* Conversation-agent deps minus the workflow callable, the resource policy, the
|
package/dist/runtime/dbos.d.ts
CHANGED
|
@@ -92,9 +92,10 @@ export declare function dbosSdkConfig(config: DbosConfig, logger: Logger): DBOSC
|
|
|
92
92
|
* Launch DBOS. Idempotent — a second call is a no-op so tests that drive
|
|
93
93
|
* the harness twice (or accidentally double-import) don't re-launch.
|
|
94
94
|
*
|
|
95
|
-
* Order is load-bearing: `setConfig`
|
|
96
|
-
* must resolve before the HTTP listener accepts traffic
|
|
97
|
-
* readiness probe could 200 against a runtime that can't own
|
|
95
|
+
* Order is load-bearing: `setConfig` and the `Result` recipes must precede
|
|
96
|
+
* `launch`, and `launch` must resolve before the HTTP listener accepts traffic
|
|
97
|
+
* (otherwise the readiness probe could 200 against a runtime that can't own
|
|
98
|
+
* workflows).
|
|
98
99
|
*/
|
|
99
100
|
export declare function launchDbos({ config, logger: injected }: {
|
|
100
101
|
config: DbosConfig;
|
package/dist/runtime/dbos.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
16
16
|
import { DBOS_SYSTEM_POOL_SIZE } from "./pools.js";
|
|
17
|
+
import { errRecipe, okRecipe } from "./result-serialization.js";
|
|
17
18
|
const state = {
|
|
18
19
|
launched: false,
|
|
19
20
|
recoveryStarted: false,
|
|
@@ -115,13 +116,26 @@ export function dbosSdkConfig(config, logger) {
|
|
|
115
116
|
otelAttributeFormat: "semconv",
|
|
116
117
|
};
|
|
117
118
|
}
|
|
119
|
+
let resultRecipesRegistered = false;
|
|
120
|
+
/**
|
|
121
|
+
* DBOS refuses a registration after the launch, and the registry of SuperJSON
|
|
122
|
+
* is global to the process, thus a relaunch keeps the recipes.
|
|
123
|
+
*/
|
|
124
|
+
function registerResultSerialization() {
|
|
125
|
+
if (resultRecipesRegistered)
|
|
126
|
+
return;
|
|
127
|
+
DBOS.registerSerialization(okRecipe);
|
|
128
|
+
DBOS.registerSerialization(errRecipe);
|
|
129
|
+
resultRecipesRegistered = true;
|
|
130
|
+
}
|
|
118
131
|
/**
|
|
119
132
|
* Launch DBOS. Idempotent — a second call is a no-op so tests that drive
|
|
120
133
|
* the harness twice (or accidentally double-import) don't re-launch.
|
|
121
134
|
*
|
|
122
|
-
* Order is load-bearing: `setConfig`
|
|
123
|
-
* must resolve before the HTTP listener accepts traffic
|
|
124
|
-
* readiness probe could 200 against a runtime that can't own
|
|
135
|
+
* Order is load-bearing: `setConfig` and the `Result` recipes must precede
|
|
136
|
+
* `launch`, and `launch` must resolve before the HTTP listener accepts traffic
|
|
137
|
+
* (otherwise the readiness probe could 200 against a runtime that can't own
|
|
138
|
+
* workflows).
|
|
125
139
|
*/
|
|
126
140
|
export async function launchDbos({ config, logger: injected }) {
|
|
127
141
|
if (state.launched)
|
|
@@ -129,6 +143,7 @@ export async function launchDbos({ config, logger: injected }) {
|
|
|
129
143
|
const logger = injected.named("dbos");
|
|
130
144
|
const sdkConfig = dbosSdkConfig(config, logger);
|
|
131
145
|
DBOS.setConfig(sdkConfig);
|
|
146
|
+
registerResultSerialization();
|
|
132
147
|
const start = performance.now();
|
|
133
148
|
await DBOS.launch();
|
|
134
149
|
state.launched = true;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The DBOS serialization recipes of the neverthrow `Ok` and `Err` classes.
|
|
3
|
+
*
|
|
4
|
+
* DBOS saves each workflow input, step output, workflow output, and message
|
|
5
|
+
* with SuperJSON, and SuperJSON drops the class of an object that it does not
|
|
6
|
+
* know. Without these recipes a step that returns a `Result` gives back a plain
|
|
7
|
+
* object with no `Result` methods, on the first run and on each replay alike,
|
|
8
|
+
* because DBOS hands the body the deserialized value. `launchDbos` registers
|
|
9
|
+
* the recipes one time, before DBOS launches.
|
|
10
|
+
*
|
|
11
|
+
* SuperJSON does not walk the output of a custom recipe, thus each recipe
|
|
12
|
+
* encodes the value that it holds as JSON itself. An `Error` inside that value
|
|
13
|
+
* keeps its name and its message only, and a `Result` nested inside the value
|
|
14
|
+
* loses its class. A step that must keep more maps its error to plain data
|
|
15
|
+
* inside the step, before the checkpoint.
|
|
16
|
+
*/
|
|
17
|
+
import type { SerializationRecipe } from "@dbos-inc/dbos-sdk";
|
|
18
|
+
import { Err, Ok } from "neverthrow";
|
|
19
|
+
export type Json = string | number | boolean | null | undefined | Json[] | {
|
|
20
|
+
[key: string]: Json;
|
|
21
|
+
};
|
|
22
|
+
export type EncodedOk = {
|
|
23
|
+
value?: Json;
|
|
24
|
+
};
|
|
25
|
+
export type EncodedErr = {
|
|
26
|
+
error?: Json;
|
|
27
|
+
};
|
|
28
|
+
export declare const okRecipe: SerializationRecipe<Ok<unknown, never>, EncodedOk>;
|
|
29
|
+
export declare const errRecipe: SerializationRecipe<Err<never, unknown>, EncodedErr>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The DBOS serialization recipes of the neverthrow `Ok` and `Err` classes.
|
|
3
|
+
*
|
|
4
|
+
* DBOS saves each workflow input, step output, workflow output, and message
|
|
5
|
+
* with SuperJSON, and SuperJSON drops the class of an object that it does not
|
|
6
|
+
* know. Without these recipes a step that returns a `Result` gives back a plain
|
|
7
|
+
* object with no `Result` methods, on the first run and on each replay alike,
|
|
8
|
+
* because DBOS hands the body the deserialized value. `launchDbos` registers
|
|
9
|
+
* the recipes one time, before DBOS launches.
|
|
10
|
+
*
|
|
11
|
+
* SuperJSON does not walk the output of a custom recipe, thus each recipe
|
|
12
|
+
* encodes the value that it holds as JSON itself. An `Error` inside that value
|
|
13
|
+
* keeps its name and its message only, and a `Result` nested inside the value
|
|
14
|
+
* loses its class. A step that must keep more maps its error to plain data
|
|
15
|
+
* inside the step, before the checkpoint.
|
|
16
|
+
*/
|
|
17
|
+
import { Err, Ok, err, ok } from "neverthrow";
|
|
18
|
+
/**
|
|
19
|
+
* The JSON form of the value inside a `Result`, with the rules of
|
|
20
|
+
* `JSON.stringify`. An `Error` becomes `{ name, message }`, because
|
|
21
|
+
* `JSON.stringify` gives `{}` for one. A `bigint` becomes its decimal text,
|
|
22
|
+
* because `JSON.stringify` throws on one.
|
|
23
|
+
*/
|
|
24
|
+
function toJson(value) {
|
|
25
|
+
const text = JSON.stringify(value, (_key, v) => {
|
|
26
|
+
if (v instanceof Error)
|
|
27
|
+
return { name: v.name, message: v.message };
|
|
28
|
+
if (typeof v === "bigint")
|
|
29
|
+
return v.toString();
|
|
30
|
+
return v;
|
|
31
|
+
});
|
|
32
|
+
return text === undefined ? undefined : JSON.parse(text);
|
|
33
|
+
}
|
|
34
|
+
export const okRecipe = {
|
|
35
|
+
name: "neverthrow.Ok",
|
|
36
|
+
isApplicable: (v) => v instanceof Ok,
|
|
37
|
+
serialize: (v) => ({ value: toJson(v.value) }),
|
|
38
|
+
deserialize: (s) => ok(s.value),
|
|
39
|
+
};
|
|
40
|
+
export const errRecipe = {
|
|
41
|
+
name: "neverthrow.Err",
|
|
42
|
+
isApplicable: (v) => v instanceof Err,
|
|
43
|
+
serialize: (v) => ({ error: toJson(v.error) }),
|
|
44
|
+
deserialize: (s) => err(s.error),
|
|
45
|
+
};
|
package/dist/sandbox/client.d.ts
CHANGED
|
@@ -25,7 +25,10 @@
|
|
|
25
25
|
* `tasks/derive-table-exec.ts` are the two worked examples. No type carries this
|
|
26
26
|
* rule yet, thus a new caller must read this paragraph.
|
|
27
27
|
*/
|
|
28
|
-
import type {
|
|
28
|
+
import type { ResultAsync } from "neverthrow";
|
|
29
|
+
import type { SpawnSession } from "../auth/types.js";
|
|
30
|
+
import type { SandboxError } from "./sandbox-error.js";
|
|
31
|
+
import type { ExecEmit, ExecResult, ManagedSandbox, SandboxIdentity, SandboxLiveness, SandboxRef, SandboxSpec, SubmitExecBody, ToolchainSource } from "./types.js";
|
|
29
32
|
export interface SandboxClient {
|
|
30
33
|
/**
|
|
31
34
|
* The declared owner of the sandbox toolchain, as the client was composed
|
|
@@ -40,12 +43,17 @@ export interface SandboxClient {
|
|
|
40
43
|
* DBOS step (`sandbox.create`) — the spawn half of the two-step create
|
|
41
44
|
* (see the harness-sandbox-exec spec). Launches the sandbox-base container/Job under the
|
|
42
45
|
* pre-minted `identity` (name + HMAC secret checkpointed by `sandbox.mint`),
|
|
43
|
-
* stamps
|
|
46
|
+
* stamps the labels, waits for `/health`, records the live handle in
|
|
44
47
|
* the active-sandbox registry, and returns the in-memory `SandboxRef`. A
|
|
45
48
|
* recovery re-run whose machine already exists (the crash window between
|
|
46
49
|
* spawn and checkpoint) **adopts** it rather than leaking a second one.
|
|
50
|
+
*
|
|
51
|
+
* Each spawn failure is an `err` value, the refusal of the label hook
|
|
52
|
+
* (`labels_refused`) included: a spawn path splits the result with
|
|
53
|
+
* `keepSuspendingRefusal`. Only a defect of the deployment config, or a
|
|
54
|
+
* throw of the workspace-root resolver, throws, and it fails the workflow.
|
|
47
55
|
*/
|
|
48
|
-
createSandbox(
|
|
56
|
+
createSandbox(session: SpawnSession, spec: SandboxSpec, identity: SandboxIdentity): ResultAsync<SandboxRef, SandboxError>;
|
|
49
57
|
/**
|
|
50
58
|
* DBOS step (`sandbox.submit-exec.${execId}`). POSTs the command to
|
|
51
59
|
* sandbox-server's `/exec` and returns after the HTTP 202 ack. Does NOT
|
|
@@ -19,8 +19,9 @@ import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
|
19
19
|
import { clearSandboxRef } from "../state/index.js";
|
|
20
20
|
import { type AwaitExecOptions } from "./await-exec.js";
|
|
21
21
|
import type { SandboxClient } from "./client.js";
|
|
22
|
+
import { type MountPlanCoords } from "./mount-plan.js";
|
|
22
23
|
import { type SubmitExecDeps } from "./submit-exec.js";
|
|
23
|
-
import { type
|
|
24
|
+
import { type FarmSource, type ResolveSandboxLabels, type SandboxTransport, type ToolchainSource } from "./types.js";
|
|
24
25
|
/**
|
|
25
26
|
* Narrow config slice the sandbox factory reads off `Env` — the backend
|
|
26
27
|
* default and the K8s namespace fallback. Composition roots map their
|
|
@@ -144,6 +145,8 @@ export interface CreateSandboxClientConfig {
|
|
|
144
145
|
runtimeClassName?: string;
|
|
145
146
|
/** Override the backend selection — defaults to `env.SANDBOX_BACKEND`. */
|
|
146
147
|
backend?: "docker" | "k8s";
|
|
148
|
+
/** Absent, a sandbox carries no host labels. */
|
|
149
|
+
resolveSandboxLabels?: ResolveSandboxLabels;
|
|
147
150
|
/**
|
|
148
151
|
* Optional logger forwarded to the Docker backend so a store degradation
|
|
149
152
|
* (a resolved farm whose `inflexa.lock` vanished or went invalid
|
|
@@ -186,7 +189,7 @@ export declare function composeAwaitOptions(base: AwaitExecOptions | undefined,
|
|
|
186
189
|
export declare function precreateStepTree(deps: {
|
|
187
190
|
resolveWorkspaceRoot: ResolveWorkspaceRoot;
|
|
188
191
|
stepTreeAccess?: "world-writable";
|
|
189
|
-
},
|
|
192
|
+
}, coords: MountPlanCoords): Promise<void>;
|
|
190
193
|
export declare function createSandboxClient(config: CreateSandboxClientConfig): SandboxClient;
|
|
191
194
|
/** Re-exports so consumers import once from this module. */
|
|
192
195
|
export { clearSandboxRef };
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { chmod, mkdir } from "node:fs/promises";
|
|
15
15
|
import { join } from "node:path";
|
|
16
|
+
import { ResultAsync, err, okAsync } from "neverthrow";
|
|
17
|
+
import { passGate } from "../lib/hooks.js";
|
|
16
18
|
import { recordSandboxExec } from "../lib/metrics.js";
|
|
17
19
|
import { clampResources } from "../config/resource-limits.js";
|
|
18
20
|
import { tailWritePrefix } from "../workspace/paths.js";
|
|
@@ -24,10 +26,10 @@ import { awaitExec } from "./await-exec.js";
|
|
|
24
26
|
import { createDockerSandboxOps } from "./docker-client.js";
|
|
25
27
|
import { noteExecOutcome, sandboxExecOutcomeOf, summarizeExec } from "./exec-outcome.js";
|
|
26
28
|
import { createK8sSandboxOps } from "./k8s-client.js";
|
|
27
|
-
import { sandboxWriteTail } from "./mount-plan.js";
|
|
29
|
+
import { mountCoordsOf, sandboxWriteTail } from "./mount-plan.js";
|
|
28
30
|
import { SandboxFailure } from "./sandbox-error.js";
|
|
29
31
|
import { submitExec } from "./submit-exec.js";
|
|
30
|
-
import { toPersistedRef } from "./types.js";
|
|
32
|
+
import { toPersistedRef, } from "./types.js";
|
|
31
33
|
/**
|
|
32
34
|
* Assemble `awaitExec`'s options: the liveness probe self-wires from the
|
|
33
35
|
* backend ops (the poll loop's escalation always has its arbiter under the
|
|
@@ -58,14 +60,14 @@ export function composeAwaitOptions(base, transport, isAlive) {
|
|
|
58
60
|
* would not touch them either way. Scoped to the write tree — the
|
|
59
61
|
* read-only mount sources are never re-moded. Exported for tests.
|
|
60
62
|
*/
|
|
61
|
-
export async function precreateStepTree(deps,
|
|
62
|
-
const write = sandboxWriteTail(
|
|
63
|
+
export async function precreateStepTree(deps, coords) {
|
|
64
|
+
const write = sandboxWriteTail(coords);
|
|
63
65
|
if (write === undefined)
|
|
64
66
|
return;
|
|
65
67
|
// `tailWritePrefix` (not a raw `join`) so every segment runs through the same
|
|
66
68
|
// id validation the mount builders apply — a crafted tail cannot escape the
|
|
67
69
|
// resolved root.
|
|
68
|
-
const writeDir = tailWritePrefix({ workspaceRoot: deps.resolveWorkspaceRoot(
|
|
70
|
+
const writeDir = tailWritePrefix({ workspaceRoot: deps.resolveWorkspaceRoot(coords.analysisId), tail: write.tail });
|
|
69
71
|
await mkdir(writeDir, { recursive: true });
|
|
70
72
|
await Promise.all(write.subdirs.map((sub) => mkdir(join(writeDir, sub), { recursive: true })));
|
|
71
73
|
if (deps.stepTreeAccess === "world-writable") {
|
|
@@ -109,8 +111,8 @@ export function createSandboxClient(config) {
|
|
|
109
111
|
* unwraps below stay on the plain bridge: they carry a `DbError`.
|
|
110
112
|
*/
|
|
111
113
|
const failing = (e) => new SandboxFailure(e);
|
|
112
|
-
const registerSandbox = async (
|
|
113
|
-
unwrapOrThrow(await setSandboxRef(config.pool,
|
|
114
|
+
const registerSandbox = async (session, spec, ref) => {
|
|
115
|
+
unwrapOrThrow(await setSandboxRef(config.pool, session.runFrame.runId, session.runFrame.stepId, toPersistedRef(ref), spec.execId ?? null));
|
|
114
116
|
};
|
|
115
117
|
const ops = backend === "k8s"
|
|
116
118
|
? createK8sSandboxOps({
|
|
@@ -171,26 +173,50 @@ export function createSandboxClient(config) {
|
|
|
171
173
|
// One arbiter for both surfaces: the client's `isAlive` method and the
|
|
172
174
|
// poll loop's escalation probe are the same backend inspect.
|
|
173
175
|
const isAlive = async (ref) => unwrapOrThrow((await ops.isAlive(ref)).mapErr(failing));
|
|
176
|
+
const hostLabelsFor = (session) => {
|
|
177
|
+
const hook = config.resolveSandboxLabels;
|
|
178
|
+
if (hook === undefined)
|
|
179
|
+
return okAsync({});
|
|
180
|
+
return passGate("resolveSandboxLabels", hook(session)).mapErr((refusal) => ({
|
|
181
|
+
type: "labels_refused",
|
|
182
|
+
op: "createSandbox",
|
|
183
|
+
reason: refusal.reason,
|
|
184
|
+
suspend: refusal.kind === "suspended",
|
|
185
|
+
}));
|
|
186
|
+
};
|
|
187
|
+
const createSandbox = async (session, spec, identity) => {
|
|
188
|
+
// The label hook runs first: a refusal makes no step tree and calls no backend.
|
|
189
|
+
const hostLabels = await hostLabelsFor(session);
|
|
190
|
+
if (hostLabels.isErr())
|
|
191
|
+
return err(hostLabels.error);
|
|
192
|
+
const createFailed = (op) => (cause) => ({
|
|
193
|
+
type: "container_create_failed",
|
|
194
|
+
op,
|
|
195
|
+
sandboxId: identity.sandboxId,
|
|
196
|
+
cause,
|
|
197
|
+
});
|
|
198
|
+
// Every caller must declare resources — a sandbox with no cpu/memory
|
|
199
|
+
// request is a semantic error, not something to paper over with a
|
|
200
|
+
// default.
|
|
201
|
+
if (!spec.resources) {
|
|
202
|
+
return err(createFailed("createSandbox")(new Error(`createSandbox: ${session.scope.analysisId}/${session.runFrame.runId}/${session.runFrame.stepId} has no resources — every caller must declare cpu/memoryGb`)));
|
|
203
|
+
}
|
|
204
|
+
// The config states an engine fact (binds preserve host ownership);
|
|
205
|
+
// which remediation that fact demands — today, a world-writable step
|
|
206
|
+
// write tree — is harness-owned and chosen here, not by the embedder.
|
|
207
|
+
const precreated = await ResultAsync.fromPromise(precreateStepTree({
|
|
208
|
+
resolveWorkspaceRoot: config.resolveWorkspaceRoot,
|
|
209
|
+
stepTreeAccess: config.engineBindOwnership === "host-preserved" ? "world-writable" : undefined,
|
|
210
|
+
}, mountCoordsOf(session, spec)), createFailed("createSandbox.precreateStepTree"));
|
|
211
|
+
if (precreated.isErr())
|
|
212
|
+
return err(precreated.error);
|
|
213
|
+
// Clamp to cluster ceilings so the pod is always quota-admissible.
|
|
214
|
+
const resources = clampResources(spec.resources, config.resourceLimits);
|
|
215
|
+
return ops.createSandbox(session, { ...spec, resources }, identity, hostLabels.value);
|
|
216
|
+
};
|
|
174
217
|
return {
|
|
175
218
|
toolchainSource,
|
|
176
|
-
createSandbox:
|
|
177
|
-
// The config states an engine fact (binds preserve host ownership);
|
|
178
|
-
// which remediation that fact demands — today, a world-writable step
|
|
179
|
-
// write tree — is harness-owned and chosen here, not by the embedder.
|
|
180
|
-
await precreateStepTree({
|
|
181
|
-
resolveWorkspaceRoot: config.resolveWorkspaceRoot,
|
|
182
|
-
stepTreeAccess: config.engineBindOwnership === "host-preserved" ? "world-writable" : undefined,
|
|
183
|
-
}, meta);
|
|
184
|
-
// Every caller must declare resources — a sandbox with no cpu/memory
|
|
185
|
-
// request is a semantic error, not something to paper over with a
|
|
186
|
-
// default (a DBOS replay of a pre-resources workflow input lands here).
|
|
187
|
-
if (!meta.resources) {
|
|
188
|
-
throw new Error(`createSandbox: ${meta.analysisId}/${meta.runId}/${meta.stepId} has no resources — every caller must declare cpu/memoryGb`);
|
|
189
|
-
}
|
|
190
|
-
// Clamp to cluster ceilings so the pod is always quota-admissible.
|
|
191
|
-
const resources = clampResources(meta.resources, config.resourceLimits);
|
|
192
|
-
return unwrapOrThrow((await ops.createSandbox({ ...meta, resources }, identity)).mapErr(failing));
|
|
193
|
-
},
|
|
219
|
+
createSandbox: (session, spec, identity) => new ResultAsync(createSandbox(session, spec, identity)),
|
|
194
220
|
submitExec: async (ref, body) => submitExec(ref, body, {
|
|
195
221
|
...config.submitDeps,
|
|
196
222
|
execStreamByteCap: config.submitDeps?.execStreamByteCap ?? config.execStreamByteCap,
|