@inflexa-ai/harness 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -0
- package/dist/agents/conversation-agent.d.ts +3 -0
- package/dist/agents/conversation-agent.js +2 -2
- package/dist/agents/sandbox/shared.d.ts +3 -0
- package/dist/agents/sandbox/shared.js +2 -2
- package/dist/app/chat-turn.d.ts +3 -0
- package/dist/app/chat-turn.js +3 -1
- package/dist/app/synthesize-run.d.ts +3 -0
- package/dist/app/synthesize-run.js +3 -1
- package/dist/execution/artifact-metadata.d.ts +3 -0
- package/dist/execution/artifact-metadata.js +8 -2
- package/dist/execution/artifact-registration.d.ts +2 -1
- package/dist/execution/artifact-registration.js +8 -2
- package/dist/execution/ephemeral-runner.d.ts +3 -0
- package/dist/execution/ephemeral-runner.js +8 -4
- package/dist/execution/post-step-pipeline.d.ts +3 -0
- package/dist/execution/post-step-pipeline.js +24 -4
- package/dist/execution/reconcile-manifest.d.ts +7 -0
- package/dist/execution/reconcile-manifest.js +19 -5
- package/dist/execution/report-runner.d.ts +3 -0
- package/dist/execution/report-runner.js +4 -4
- package/dist/execution/step-summary.d.ts +3 -0
- package/dist/execution/step-summary.js +4 -2
- package/dist/index.d.ts +3 -0
- package/dist/index.js +5 -0
- package/dist/lib/chrome.d.ts +4 -1
- package/dist/lib/chrome.js +10 -7
- package/dist/lib/console-logger.d.ts +19 -0
- package/dist/lib/console-logger.js +62 -0
- package/dist/lib/logger.d.ts +88 -0
- package/dist/lib/logger.js +37 -0
- package/dist/lib/otel.d.ts +2 -1
- package/dist/lib/otel.js +8 -3
- package/dist/runtime/boot.d.ts +2 -2
- package/dist/runtime/boot.js +1 -1
- package/dist/runtime/connection-budget.d.ts +2 -2
- package/dist/runtime/connection-budget.js +5 -3
- package/dist/runtime/dbos.d.ts +7 -7
- package/dist/runtime/dbos.js +14 -11
- package/dist/runtime/shutdown.d.ts +2 -2
- package/dist/runtime/shutdown.js +4 -3
- package/dist/sandbox/create-sandbox.d.ts +41 -3
- package/dist/sandbox/create-sandbox.js +39 -19
- package/dist/sandbox/docker-client.d.ts +17 -2
- package/dist/sandbox/docker-client.js +39 -12
- package/dist/sandbox/k8s-client.d.ts +3 -0
- package/dist/sandbox/k8s-client.js +5 -4
- package/dist/sandbox/notification-sweep.d.ts +3 -3
- package/dist/sandbox/notification-sweep.js +4 -2
- package/dist/sandbox/reaper.d.ts +3 -3
- package/dist/sandbox/reaper.js +7 -4
- package/dist/sandbox/watchdog.d.ts +3 -3
- package/dist/sandbox/watchdog.js +8 -2
- package/dist/state/init.d.ts +2 -1
- package/dist/state/init.js +5 -6
- package/dist/tasks/data-profile.d.ts +5 -0
- package/dist/tasks/data-profile.js +24 -17
- package/dist/tools/bio/search-dgidb.d.ts +11 -1
- package/dist/tools/bio/search-dgidb.js +92 -78
- package/dist/tools/iterate-report.d.ts +3 -0
- package/dist/tools/iterate-report.js +3 -1
- package/dist/tools/research/literature-reviewer.d.ts +3 -0
- package/dist/tools/research/literature-reviewer.js +2 -2
- package/dist/tools/workspace/execute-command.d.ts +3 -0
- package/dist/tools/workspace/execute-command.js +3 -1
- package/dist/workflows/execute-analysis.d.ts +3 -0
- package/dist/workflows/execute-analysis.js +11 -7
- package/dist/workflows/execute-target-assessment.d.ts +3 -0
- package/dist/workflows/execute-target-assessment.js +19 -15
- package/dist/workflows/sandbox-step.d.ts +9 -0
- package/dist/workflows/sandbox-step.js +39 -17
- package/dist/workflows/target-assessment/lib/llm-step.d.ts +3 -0
- package/dist/workflows/target-assessment/lib/llm-step.js +3 -1
- package/dist/workflows/target-assessment/lib/structured-llm.d.ts +3 -0
- package/dist/workflows/target-assessment/lib/structured-llm.js +1 -0
- package/dist/workflows/target-assessment/phase5-persist.d.ts +3 -0
- package/dist/workflows/target-assessment/phase5-persist.js +8 -3
- package/dist/workflows/target-assessment/progress.d.ts +2 -1
- package/dist/workflows/target-assessment/progress.js +4 -3
- package/dist/workspace/filesystem.js +163 -74
- package/package.json +95 -96
|
@@ -13,14 +13,14 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import type { V1Toleration } from "@kubernetes/client-node";
|
|
15
15
|
import type { Pool } from "pg";
|
|
16
|
-
import type
|
|
16
|
+
import type { Logger } from "../lib/logger.js";
|
|
17
17
|
import { type ResourceLimits } from "../config/resource-limits.js";
|
|
18
18
|
import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
19
19
|
import { clearSandboxRef } from "../state/index.js";
|
|
20
20
|
import { type AwaitExecOptions } from "./await-exec.js";
|
|
21
21
|
import type { SandboxClient } from "./client.js";
|
|
22
22
|
import { type SubmitExecDeps } from "./submit-exec.js";
|
|
23
|
-
import { type SandboxTransport } from "./types.js";
|
|
23
|
+
import { type CreateSandboxMeta, type SandboxTransport } from "./types.js";
|
|
24
24
|
/**
|
|
25
25
|
* Narrow config slice the sandbox factory reads off `Env` — the backend
|
|
26
26
|
* default and the K8s namespace fallback. Composition roots map their
|
|
@@ -62,6 +62,25 @@ export interface CreateSandboxClientConfig {
|
|
|
62
62
|
refStorePath?: string;
|
|
63
63
|
/** Docker: force the container platform (e.g. `linux/amd64`) so the sandbox matches the mounted lib store's arch. */
|
|
64
64
|
platform?: string;
|
|
65
|
+
/**
|
|
66
|
+
* Docker: unix socket of the Docker-API engine to dial — a Docker daemon
|
|
67
|
+
* socket or a podman Docker-compat socket. Unset preserves dockerode's
|
|
68
|
+
* default resolution (`DOCKER_HOST`, then the default Docker socket).
|
|
69
|
+
*/
|
|
70
|
+
engineSocketPath?: string;
|
|
71
|
+
/**
|
|
72
|
+
* Docker: the engine's bind mounts expose honest host file ownership to
|
|
73
|
+
* the container — podman machine's virtiofs and native Linux binds present
|
|
74
|
+
* the embedder user's real uid and modes, which the uid-1000 sandbox
|
|
75
|
+
* workload cannot write; Docker Desktop's file-sharing layer masks that
|
|
76
|
+
* mismatch, so it stays unset there. When set, the client loosens each
|
|
77
|
+
* step's pre-created write tree so workload writes land. The field names
|
|
78
|
+
* the engine fact rather than the remediation, so how the harness
|
|
79
|
+
* compensates can evolve without changing the embedder surface; a
|
|
80
|
+
* single-valued union (not a boolean) so future ownership classes extend
|
|
81
|
+
* it without a breaking change.
|
|
82
|
+
*/
|
|
83
|
+
engineBindOwnership?: "host-preserved";
|
|
65
84
|
/** K8s: PVC claim backing the session PVC the workspace roots live under. */
|
|
66
85
|
sessionPvc?: string;
|
|
67
86
|
/**
|
|
@@ -87,7 +106,7 @@ export interface CreateSandboxClientConfig {
|
|
|
87
106
|
* (a configured store whose `current` vanished/went incomplete mid-session) is
|
|
88
107
|
* observable rather than a silent mount drop. No-op when unset.
|
|
89
108
|
*/
|
|
90
|
-
logger?:
|
|
109
|
+
logger?: Logger;
|
|
91
110
|
/** Dependency seams (fetch, durable step/sleep, clock, recv, warn sink) forwarded to submit/await. */
|
|
92
111
|
submitDeps?: SubmitExecDeps;
|
|
93
112
|
awaitOptions?: AwaitExecOptions;
|
|
@@ -100,6 +119,25 @@ export interface CreateSandboxClientConfig {
|
|
|
100
119
|
* Exported for tests.
|
|
101
120
|
*/
|
|
102
121
|
export declare function composeAwaitOptions(base: AwaitExecOptions | undefined, transport: SandboxTransport, isAlive: NonNullable<AwaitExecOptions["isAlive"]>): AwaitExecOptions;
|
|
122
|
+
/**
|
|
123
|
+
* Backend-agnostic pre-creation of the writable step tree under the analysis's
|
|
124
|
+
* resolved workspace root. On Docker that root is the host dir the container
|
|
125
|
+
* binds; on K8s it lives under the session PVC the sandbox pod mounts via
|
|
126
|
+
* subPath. `mkdir(recursive)` is idempotent, so an existing tree (replay, retry)
|
|
127
|
+
* is not an error. A read-only sandbox has no writable step mount, so there is
|
|
128
|
+
* nothing to pre-create.
|
|
129
|
+
*
|
|
130
|
+
* `stepTreeAccess: "world-writable"` chmods the step dir and each subdir so the
|
|
131
|
+
* uid-1000 sandbox workload can write them through engines that honor host bind
|
|
132
|
+
* ownership. The chmod is explicit, not `mkdir`'s `mode` option: the process
|
|
133
|
+
* umask masks `mkdir`'s mode, and on replay the dirs already exist so `mkdir`
|
|
134
|
+
* would not touch them either way. Scoped to the step write tree — the
|
|
135
|
+
* read-only mount sources are never re-moded. Exported for tests.
|
|
136
|
+
*/
|
|
137
|
+
export declare function precreateStepTree(deps: {
|
|
138
|
+
resolveWorkspaceRoot: ResolveWorkspaceRoot;
|
|
139
|
+
stepTreeAccess?: "world-writable";
|
|
140
|
+
}, meta: CreateSandboxMeta): Promise<void>;
|
|
103
141
|
export declare function createSandboxClient(config: CreateSandboxClientConfig): SandboxClient;
|
|
104
142
|
/** Re-exports so consumers import once from this module. */
|
|
105
143
|
export { clearSandboxRef };
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* coupling (they just call the closures we hand them). `submitExec` and
|
|
12
12
|
* `awaitExec` are backend-agnostic — they only need the SandboxRef.
|
|
13
13
|
*/
|
|
14
|
-
import { mkdir } from "node:fs/promises";
|
|
14
|
+
import { chmod, mkdir } from "node:fs/promises";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { clampResources } from "../config/resource-limits.js";
|
|
17
17
|
import { stepWritePrefix } from "../workspace/paths.js";
|
|
@@ -34,6 +34,36 @@ import { toPersistedRef } from "./types.js";
|
|
|
34
34
|
export function composeAwaitOptions(base, transport, isAlive) {
|
|
35
35
|
return { isAlive, ...base, transport };
|
|
36
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Backend-agnostic pre-creation of the writable step tree under the analysis's
|
|
39
|
+
* resolved workspace root. On Docker that root is the host dir the container
|
|
40
|
+
* binds; on K8s it lives under the session PVC the sandbox pod mounts via
|
|
41
|
+
* subPath. `mkdir(recursive)` is idempotent, so an existing tree (replay, retry)
|
|
42
|
+
* is not an error. A read-only sandbox has no writable step mount, so there is
|
|
43
|
+
* nothing to pre-create.
|
|
44
|
+
*
|
|
45
|
+
* `stepTreeAccess: "world-writable"` chmods the step dir and each subdir so the
|
|
46
|
+
* uid-1000 sandbox workload can write them through engines that honor host bind
|
|
47
|
+
* ownership. The chmod is explicit, not `mkdir`'s `mode` option: the process
|
|
48
|
+
* umask masks `mkdir`'s mode, and on replay the dirs already exist so `mkdir`
|
|
49
|
+
* would not touch them either way. Scoped to the step write tree — the
|
|
50
|
+
* read-only mount sources are never re-moded. Exported for tests.
|
|
51
|
+
*/
|
|
52
|
+
export async function precreateStepTree(deps, meta) {
|
|
53
|
+
if (meta.readOnly)
|
|
54
|
+
return;
|
|
55
|
+
const stepDir = stepWritePrefix({
|
|
56
|
+
workspaceRoot: deps.resolveWorkspaceRoot(meta.analysisId),
|
|
57
|
+
runId: meta.runId,
|
|
58
|
+
stepId: meta.stepId,
|
|
59
|
+
});
|
|
60
|
+
await mkdir(stepDir, { recursive: true });
|
|
61
|
+
await Promise.all(STEP_SUBDIRS.map((sub) => mkdir(join(stepDir, sub), { recursive: true })));
|
|
62
|
+
if (deps.stepTreeAccess === "world-writable") {
|
|
63
|
+
await chmod(stepDir, 0o777);
|
|
64
|
+
await Promise.all(STEP_SUBDIRS.map((sub) => chmod(join(stepDir, sub), 0o777)));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
37
67
|
export function createSandboxClient(config) {
|
|
38
68
|
const backend = config.backend ?? config.env.backend;
|
|
39
69
|
const transport = config.transport ?? "poll";
|
|
@@ -64,26 +94,10 @@ export function createSandboxClient(config) {
|
|
|
64
94
|
libStorePath: config.libStorePath,
|
|
65
95
|
refStorePath: config.refStorePath,
|
|
66
96
|
platform: config.platform,
|
|
97
|
+
engineSocketPath: config.engineSocketPath,
|
|
67
98
|
logger: config.logger,
|
|
68
99
|
registerSandbox,
|
|
69
100
|
});
|
|
70
|
-
// Backend-agnostic pre-creation of the writable step tree under the
|
|
71
|
-
// analysis's resolved workspace root. On Docker that root is the host dir
|
|
72
|
-
// the container binds; on K8s it lives under the session PVC the sandbox
|
|
73
|
-
// pod mounts via subPath. mkdir(recursive) is idempotent, so an existing
|
|
74
|
-
// tree (replay, retry) is not an error.
|
|
75
|
-
const precreateStepTree = async (meta) => {
|
|
76
|
-
// A read-only sandbox has no writable step mount — nothing to pre-create.
|
|
77
|
-
if (meta.readOnly)
|
|
78
|
-
return;
|
|
79
|
-
const stepDir = stepWritePrefix({
|
|
80
|
-
workspaceRoot: config.resolveWorkspaceRoot(meta.analysisId),
|
|
81
|
-
runId: meta.runId,
|
|
82
|
-
stepId: meta.stepId,
|
|
83
|
-
});
|
|
84
|
-
await mkdir(stepDir, { recursive: true });
|
|
85
|
-
await Promise.all(STEP_SUBDIRS.map((sub) => mkdir(join(stepDir, sub), { recursive: true })));
|
|
86
|
-
};
|
|
87
101
|
// `teardown` needs the registry-clear closure but the SandboxRef alone
|
|
88
102
|
// doesn't carry runId/stepId. Wrap the backend impl so callers pass the
|
|
89
103
|
// ref + step coordinates and the wrapper does the clear after the
|
|
@@ -109,7 +123,13 @@ export function createSandboxClient(config) {
|
|
|
109
123
|
const isAlive = async (ref) => unwrapOrThrow(await ops.isAlive(ref));
|
|
110
124
|
return {
|
|
111
125
|
createSandbox: async (meta, identity) => {
|
|
112
|
-
|
|
126
|
+
// The config states an engine fact (binds preserve host ownership);
|
|
127
|
+
// which remediation that fact demands — today, a world-writable step
|
|
128
|
+
// write tree — is harness-owned and chosen here, not by the embedder.
|
|
129
|
+
await precreateStepTree({
|
|
130
|
+
resolveWorkspaceRoot: config.resolveWorkspaceRoot,
|
|
131
|
+
stepTreeAccess: config.engineBindOwnership === "host-preserved" ? "world-writable" : undefined,
|
|
132
|
+
}, meta);
|
|
113
133
|
// Every caller must declare resources — a sandbox with no cpu/memory
|
|
114
134
|
// request is a semantic error, not something to paper over with a
|
|
115
135
|
// default (a DBOS replay of a pre-resources workflow input lands here).
|
|
@@ -29,8 +29,8 @@
|
|
|
29
29
|
* effective capabilities.
|
|
30
30
|
*/
|
|
31
31
|
import Docker from "dockerode";
|
|
32
|
-
import type pino from "pino";
|
|
33
32
|
import { ResultAsync } from "neverthrow";
|
|
33
|
+
import type { Logger } from "../lib/logger.js";
|
|
34
34
|
import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
35
35
|
import { type SandboxError } from "./sandbox-error.js";
|
|
36
36
|
import type { CreateSandboxMeta, ManagedSandbox, SandboxIdentity, SandboxLiveness, SandboxRef, SandboxTransport } from "./types.js";
|
|
@@ -53,6 +53,13 @@ export interface DockerClientConfig {
|
|
|
53
53
|
* create rather than crashing inside the sandbox.
|
|
54
54
|
*/
|
|
55
55
|
platform?: string;
|
|
56
|
+
/**
|
|
57
|
+
* Unix socket of the Docker-API engine to dial — a Docker daemon socket or a
|
|
58
|
+
* podman Docker-compat socket (both serve the identical REST API). Unset
|
|
59
|
+
* preserves dockerode's default resolution (`DOCKER_HOST`, then the default
|
|
60
|
+
* Docker socket).
|
|
61
|
+
*/
|
|
62
|
+
engineSocketPath?: string;
|
|
56
63
|
/** Injected for tests. */
|
|
57
64
|
docker?: Docker;
|
|
58
65
|
/** Injected for tests so `/health` polling can be stubbed. */
|
|
@@ -62,10 +69,18 @@ export interface DockerClientConfig {
|
|
|
62
69
|
* vanished or went incomplete by sandbox-create time) is observable instead of a
|
|
63
70
|
* silent libs-mount drop. Matches the `reaper`/`watchdog` logger seam.
|
|
64
71
|
*/
|
|
65
|
-
logger?:
|
|
72
|
+
logger?: Logger;
|
|
66
73
|
/** Hook called after the registry row is written. */
|
|
67
74
|
registerSandbox: (meta: CreateSandboxMeta, ref: SandboxRef) => Promise<void>;
|
|
68
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* dockerode connection options for a configured engine socket, or `undefined`
|
|
78
|
+
* to leave construction as a bare `new Docker()` — which preserves dockerode's
|
|
79
|
+
* default resolution (`DOCKER_HOST`, then the default Docker socket). A set
|
|
80
|
+
* `engineSocketPath` dials that unix socket instead (a Docker daemon or a
|
|
81
|
+
* podman Docker-compat socket).
|
|
82
|
+
*/
|
|
83
|
+
export declare function engineConnectionOptions(engineSocketPath: string | undefined): Docker.DockerOptions | undefined;
|
|
69
84
|
export declare function createDockerSandboxOps(config: DockerClientConfig): {
|
|
70
85
|
createSandbox(meta: CreateSandboxMeta, identity: SandboxIdentity): ResultAsync<SandboxRef, SandboxError>;
|
|
71
86
|
teardown(ref: SandboxRef): ResultAsync<void, SandboxError>;
|
|
@@ -32,6 +32,7 @@ import { existsSync, statSync } from "node:fs";
|
|
|
32
32
|
import { join } from "node:path";
|
|
33
33
|
import Docker from "dockerode";
|
|
34
34
|
import { ResultAsync, err, ok } from "neverthrow";
|
|
35
|
+
import { createNoopLogger } from "../lib/console-logger.js";
|
|
35
36
|
import { stepWritePrefix } from "../workspace/paths.js";
|
|
36
37
|
import { trySandbox } from "./sandbox-error.js";
|
|
37
38
|
import { buildMountPlan } from "./mount-plan.js";
|
|
@@ -96,36 +97,58 @@ async function pollHealth(fetchImpl, url, timeoutMs) {
|
|
|
96
97
|
}
|
|
97
98
|
throw new Error(`sandbox /health did not return 200 within ${timeoutMs}ms: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
98
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* dockerode connection options for a configured engine socket, or `undefined`
|
|
102
|
+
* to leave construction as a bare `new Docker()` — which preserves dockerode's
|
|
103
|
+
* default resolution (`DOCKER_HOST`, then the default Docker socket). A set
|
|
104
|
+
* `engineSocketPath` dials that unix socket instead (a Docker daemon or a
|
|
105
|
+
* podman Docker-compat socket).
|
|
106
|
+
*/
|
|
107
|
+
export function engineConnectionOptions(engineSocketPath) {
|
|
108
|
+
return engineSocketPath === undefined ? undefined : { socketPath: engineSocketPath };
|
|
109
|
+
}
|
|
99
110
|
/**
|
|
100
111
|
* Create the container, or adopt the one already standing under this
|
|
101
|
-
* checkpointed name
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
* name
|
|
112
|
+
* checkpointed name on a recovery re-run. Any create failure triggers an
|
|
113
|
+
* inspect of the name: engines disagree on the duplicate-name status (Docker
|
|
114
|
+
* answers 409, podman's compat API 500), so the reconcile keys on whether a
|
|
115
|
+
* container stands under the name, not on a status code. A standing container
|
|
116
|
+
* this workflow owns is adopted as-is if running, removed and recreated if
|
|
117
|
+
* stopped; one owned by a different workflow is a refused name collision — never
|
|
118
|
+
* adopt or remove someone else's container. When no container stands, the
|
|
119
|
+
* failure was not a name collision and the original create error is returned;
|
|
120
|
+
* the inspect's own failure is never surfaced, so a transient engine outage
|
|
121
|
+
* (both calls fail) still reports the create failure.
|
|
105
122
|
*/
|
|
106
|
-
async function createOrAdopt(docker, createOpts, name, ownerWorkflowId, createFailed) {
|
|
123
|
+
async function createOrAdopt(docker, createOpts, name, ownerWorkflowId, createFailed, logger) {
|
|
107
124
|
const created = await trySandbox(() => docker.createContainer(createOpts), createFailed);
|
|
108
125
|
if (created.isOk())
|
|
109
126
|
return ok({ container: created.value, alreadyRunning: false });
|
|
110
|
-
if (statusOf(created.error) !== 409)
|
|
111
|
-
return err(created.error);
|
|
112
127
|
const existing = docker.getContainer(name);
|
|
113
128
|
const inspected = await trySandbox(() => existing.inspect(), createFailed);
|
|
129
|
+
// No container stands under the name → the create failure was not a name
|
|
130
|
+
// collision, so surface the original create error rather than the inspect's.
|
|
114
131
|
if (inspected.isErr())
|
|
115
|
-
return err(
|
|
132
|
+
return err(created.error);
|
|
116
133
|
const info = inspected.value;
|
|
117
134
|
const owner = info.Config?.Labels?.[OWNER_WORKFLOW_LABEL];
|
|
118
135
|
if (owner !== ownerWorkflowId) {
|
|
119
136
|
return err({ type: "name_conflict", op: "docker.createSandbox", sandboxId: name, owner: owner ?? null });
|
|
120
137
|
}
|
|
121
|
-
|
|
138
|
+
// Both mutating reconcile outcomes are logged: adoption reuses a container the
|
|
139
|
+
// current attempt did not create, so without a record here nothing ties the
|
|
140
|
+
// step's sandbox back to the prior attempt that made it.
|
|
141
|
+
if (info.State.Running) {
|
|
142
|
+
logger.info("adopted standing container on recovery re-run", { sandboxId: name, ownerWorkflowId });
|
|
122
143
|
return ok({ container: existing, alreadyRunning: true });
|
|
144
|
+
}
|
|
123
145
|
const removed = await removeContainerIgnoreMissing(docker, name);
|
|
124
146
|
if (removed.isErr())
|
|
125
147
|
return err(removed.error);
|
|
126
148
|
const recreated = await trySandbox(() => docker.createContainer(createOpts), createFailed);
|
|
127
149
|
if (recreated.isErr())
|
|
128
150
|
return err(recreated.error);
|
|
151
|
+
logger.info("removed stopped prior attempt and recreated", { sandboxId: name, ownerWorkflowId });
|
|
129
152
|
return ok({ container: recreated.value, alreadyRunning: false });
|
|
130
153
|
}
|
|
131
154
|
function removeContainerIgnoreMissing(docker, sandboxId) {
|
|
@@ -144,7 +167,11 @@ function removeContainerIgnoreMissing(docker, sandboxId) {
|
|
|
144
167
|
statusOf(e) === 404 ? ok(undefined) : err(e));
|
|
145
168
|
}
|
|
146
169
|
export function createDockerSandboxOps(config) {
|
|
147
|
-
|
|
170
|
+
// The injected test instance wins; otherwise a configured socket dials that
|
|
171
|
+
// engine and an unset one stays a bare `new Docker()` (default resolution).
|
|
172
|
+
const connection = engineConnectionOptions(config.engineSocketPath);
|
|
173
|
+
const docker = config.docker ?? (connection ? new Docker(connection) : new Docker());
|
|
174
|
+
const logger = (config.logger ?? createNoopLogger()).named("docker-client");
|
|
148
175
|
const fetchImpl = config.fetch ?? fetch;
|
|
149
176
|
const transport = config.transport ?? "poll";
|
|
150
177
|
const pollMode = transport === "poll";
|
|
@@ -158,7 +185,7 @@ export function createDockerSandboxOps(config) {
|
|
|
158
185
|
// since an otherwise-silent libs-mount drop is invisible to operators.
|
|
159
186
|
const libsMounted = !!config.libStorePath && libStoreUsable(config.libStorePath);
|
|
160
187
|
if (config.libStorePath && !libsMounted) {
|
|
161
|
-
|
|
188
|
+
logger.warn("lib store configured but `current` is missing or incomplete at sandbox creation — mounting no library store (sandbox degrades to available:false)", { libStorePath: config.libStorePath, sandboxId });
|
|
162
189
|
}
|
|
163
190
|
const plan = buildMountPlan(meta, {
|
|
164
191
|
libs: libsMounted,
|
|
@@ -231,7 +258,7 @@ export function createDockerSandboxOps(config) {
|
|
|
231
258
|
AutoRemove: false,
|
|
232
259
|
},
|
|
233
260
|
};
|
|
234
|
-
const sandbox = await createOrAdopt(docker, createOpts, sandboxId, meta.childWorkflowId, createFailed);
|
|
261
|
+
const sandbox = await createOrAdopt(docker, createOpts, sandboxId, meta.childWorkflowId, createFailed, logger);
|
|
235
262
|
if (sandbox.isErr())
|
|
236
263
|
return err(sandbox.error);
|
|
237
264
|
if (!sandbox.value.alreadyRunning) {
|
|
@@ -16,6 +16,7 @@ import { ResultAsync } from "neverthrow";
|
|
|
16
16
|
import type { ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
17
17
|
import { type SandboxError } from "./sandbox-error.js";
|
|
18
18
|
import type { CreateSandboxMeta, ManagedSandbox, SandboxIdentity, SandboxLiveness, SandboxRef, SandboxTransport } from "./types.js";
|
|
19
|
+
import type { Logger } from "../lib/logger.js";
|
|
19
20
|
/**
|
|
20
21
|
* Coerce an arbitrary identifier into a valid K8s label value:
|
|
21
22
|
* ≤63 chars, `[A-Za-z0-9._-]` only, starting and ending alphanumeric.
|
|
@@ -27,6 +28,8 @@ import type { CreateSandboxMeta, ManagedSandbox, SandboxIdentity, SandboxLivenes
|
|
|
27
28
|
*/
|
|
28
29
|
export declare function sanitizeLabelValue(value: string): string;
|
|
29
30
|
export interface K8sClientConfig {
|
|
31
|
+
/** Operational logging seam; omitted falls back to no-op. */
|
|
32
|
+
readonly logger?: Logger;
|
|
30
33
|
image: string;
|
|
31
34
|
cortexBaseUrl: string;
|
|
32
35
|
/**
|
|
@@ -16,6 +16,7 @@ import { BatchV1Api, CoreV1Api, KubeConfig } from "@kubernetes/client-node";
|
|
|
16
16
|
import { ResultAsync, err, ok } from "neverthrow";
|
|
17
17
|
import { trySandbox } from "./sandbox-error.js";
|
|
18
18
|
import { buildMountPlan, buildSessionSubPaths } from "./mount-plan.js";
|
|
19
|
+
import { createNoopLogger } from "../lib/console-logger.js";
|
|
19
20
|
/** Read the originating HTTP status off any `SandboxError` variant that carries one. */
|
|
20
21
|
function statusOf(e) {
|
|
21
22
|
return "status" in e ? e.status : undefined;
|
|
@@ -410,12 +411,12 @@ export function createK8sSandboxOps(config) {
|
|
|
410
411
|
cause,
|
|
411
412
|
}));
|
|
412
413
|
if (registered.isErr()) {
|
|
413
|
-
await cleanupFailedJob(batchApi, config.namespace, sandboxId);
|
|
414
|
+
await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
|
|
414
415
|
return err(registered.error);
|
|
415
416
|
}
|
|
416
417
|
return ok(ref);
|
|
417
418
|
}
|
|
418
|
-
await cleanupFailedJob(batchApi, config.namespace, sandboxId);
|
|
419
|
+
await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
|
|
419
420
|
return err(ready.error);
|
|
420
421
|
})());
|
|
421
422
|
},
|
|
@@ -480,9 +481,9 @@ export function createK8sSandboxOps(config) {
|
|
|
480
481
|
* matters; a delete error here is logged and swallowed so it never masks the
|
|
481
482
|
* create error returned to the caller.
|
|
482
483
|
*/
|
|
483
|
-
async function cleanupFailedJob(batchApi, namespace, sandboxId) {
|
|
484
|
+
async function cleanupFailedJob(batchApi, namespace, sandboxId, logger) {
|
|
484
485
|
const deleted = await deleteJobIgnoreMissing(batchApi, namespace, sandboxId);
|
|
485
486
|
if (deleted.isErr()) {
|
|
486
|
-
|
|
487
|
+
logger.named("k8s-client").error("failed to delete Job after startup failure", { sandboxId, err: deleted.error });
|
|
487
488
|
}
|
|
488
489
|
}
|
|
@@ -17,14 +17,14 @@
|
|
|
17
17
|
* injectable so tests can drive the sweep against a stub.
|
|
18
18
|
*/
|
|
19
19
|
import type { Pool } from "pg";
|
|
20
|
-
import type
|
|
20
|
+
import type { Logger } from "../lib/logger.js";
|
|
21
21
|
export interface SweepDeps {
|
|
22
22
|
/**
|
|
23
23
|
* Execute the bounded DELETE. Returns the row count cleared. Injected
|
|
24
24
|
* for tests; production wires the app pool.
|
|
25
25
|
*/
|
|
26
26
|
deleteStale: (limit: number) => Promise<number>;
|
|
27
|
-
logger?:
|
|
27
|
+
logger?: Logger;
|
|
28
28
|
}
|
|
29
29
|
export declare function sweepStaleNotifications(deps: SweepDeps): Promise<number>;
|
|
30
30
|
/**
|
|
@@ -35,7 +35,7 @@ export declare function sweepStaleNotifications(deps: SweepDeps): Promise<number
|
|
|
35
35
|
export declare function makeDefaultDeleteStale(pool: Pool): (limit: number) => Promise<number>;
|
|
36
36
|
export interface RegisterNotificationSweepDeps {
|
|
37
37
|
pool: Pool;
|
|
38
|
-
logger?:
|
|
38
|
+
logger?: Logger;
|
|
39
39
|
/** Override the DELETE driver — production defaults to the app pool. */
|
|
40
40
|
deleteStale?: (limit: number) => Promise<number>;
|
|
41
41
|
}
|
|
@@ -17,16 +17,18 @@
|
|
|
17
17
|
* injectable so tests can drive the sweep against a stub.
|
|
18
18
|
*/
|
|
19
19
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
20
|
+
import { createNoopLogger } from "../lib/console-logger.js";
|
|
20
21
|
const SWEEP_CRON = "0 */5 * * * *";
|
|
21
22
|
const DELETE_BATCH_LIMIT = 10_000;
|
|
22
23
|
export async function sweepStaleNotifications(deps) {
|
|
24
|
+
const logger = (deps.logger ?? createNoopLogger()).named("notification-sweep");
|
|
23
25
|
try {
|
|
24
26
|
const cleared = await deps.deleteStale(DELETE_BATCH_LIMIT);
|
|
25
|
-
|
|
27
|
+
logger.info("cleared stale notifications", { rowsCleared: cleared });
|
|
26
28
|
return cleared;
|
|
27
29
|
}
|
|
28
30
|
catch (err) {
|
|
29
|
-
|
|
31
|
+
logger.error("sweep failed", logger.errorFields(err));
|
|
30
32
|
throw err;
|
|
31
33
|
}
|
|
32
34
|
}
|
package/dist/sandbox/reaper.d.ts
CHANGED
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
* hoisted for unit testing without a DBOS runtime; `registerSandboxReaper` is
|
|
19
19
|
* the thin production wiring.
|
|
20
20
|
*/
|
|
21
|
-
import type pino from "pino";
|
|
22
21
|
import type { Pool } from "pg";
|
|
22
|
+
import type { Logger } from "../lib/logger.js";
|
|
23
23
|
import type { SandboxClient } from "./client.js";
|
|
24
24
|
import type { ManagedSandbox } from "./types.js";
|
|
25
25
|
export type ReapDecision = "leave" | "reap";
|
|
@@ -46,7 +46,7 @@ export interface ReaperDeps {
|
|
|
46
46
|
graceMs?: number;
|
|
47
47
|
/** Wall-clock ms; injected for tests. */
|
|
48
48
|
nowMs?: () => number;
|
|
49
|
-
logger?:
|
|
49
|
+
logger?: Logger;
|
|
50
50
|
}
|
|
51
51
|
export interface ReapSummary {
|
|
52
52
|
managedCount: number;
|
|
@@ -64,7 +64,7 @@ export interface RegisterReaperDeps {
|
|
|
64
64
|
pool: Pool;
|
|
65
65
|
sandboxClient: Pick<SandboxClient, "listManagedSandboxes" | "teardownById">;
|
|
66
66
|
graceMs?: number;
|
|
67
|
-
logger?:
|
|
67
|
+
logger?: Logger;
|
|
68
68
|
}
|
|
69
69
|
/** Production registration: a single (unsharded) `@DBOS.scheduled` workflow. */
|
|
70
70
|
export declare function registerSandboxReaper(deps: RegisterReaperDeps): void;
|
package/dist/sandbox/reaper.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
* the thin production wiring.
|
|
20
20
|
*/
|
|
21
21
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
22
|
+
import { createNoopLogger } from "../lib/console-logger.js";
|
|
22
23
|
import { unwrapOrThrow } from "../lib/result.js";
|
|
23
24
|
import { reconcileReapedSandbox } from "../state/active-sandboxes.js";
|
|
24
25
|
const REAPER_CRON = "0 */5 * * * *";
|
|
@@ -65,6 +66,7 @@ export function terminalStepStatus(workflowStatus) {
|
|
|
65
66
|
* failure on one machine is logged and skipped — the next sweep retries.
|
|
66
67
|
*/
|
|
67
68
|
export async function reapOnce(deps) {
|
|
69
|
+
const logger = (deps.logger ?? createNoopLogger()).named("sandbox-reaper");
|
|
68
70
|
const nowMs = (deps.nowMs ?? (() => Date.now()))();
|
|
69
71
|
const graceMs = deps.graceMs ?? DEFAULT_GRACE_MS;
|
|
70
72
|
const managed = await deps.sandboxClient.listManagedSandboxes();
|
|
@@ -85,10 +87,10 @@ export async function reapOnce(deps) {
|
|
|
85
87
|
rowsReconciled++;
|
|
86
88
|
}
|
|
87
89
|
catch (err) {
|
|
88
|
-
|
|
90
|
+
logger.warn("reap failed — skipping this round", {
|
|
89
91
|
sandboxId: sb.sandboxId,
|
|
90
|
-
|
|
91
|
-
}
|
|
92
|
+
...logger.errorFields(err),
|
|
93
|
+
});
|
|
92
94
|
}
|
|
93
95
|
}
|
|
94
96
|
const summary = {
|
|
@@ -97,7 +99,8 @@ export async function reapOnce(deps) {
|
|
|
97
99
|
rowsReconciled,
|
|
98
100
|
leftCount,
|
|
99
101
|
};
|
|
100
|
-
|
|
102
|
+
// Spread: an interface has no implicit index signature, so it needs widening to `LogFields`.
|
|
103
|
+
logger.info("sweep completed", { ...summary });
|
|
101
104
|
return summary;
|
|
102
105
|
}
|
|
103
106
|
/** Production registration: a single (unsharded) `@DBOS.scheduled` workflow. */
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
* without spinning a DBOS runtime; `registerWatchdog` is the thin
|
|
19
19
|
* production wiring.
|
|
20
20
|
*/
|
|
21
|
-
import type pino from "pino";
|
|
22
21
|
import type { ResultAsync } from "neverthrow";
|
|
22
|
+
import type { Logger } from "../lib/logger.js";
|
|
23
23
|
import type { DbError } from "../lib/db-result.js";
|
|
24
24
|
import type { ActiveSandboxRow } from "../state/index.js";
|
|
25
25
|
import type { SandboxClient } from "./client.js";
|
|
@@ -39,7 +39,7 @@ export interface CheckShardDeps {
|
|
|
39
39
|
* this is `DBOS.send`; injected here for testability.
|
|
40
40
|
*/
|
|
41
41
|
sendSynthetic: (workflowId: string, execId: string, failure: ExecResult, reason: string) => Promise<void>;
|
|
42
|
-
logger?:
|
|
42
|
+
logger?: Logger;
|
|
43
43
|
}
|
|
44
44
|
export interface CheckShardSummary {
|
|
45
45
|
activeCount: number;
|
|
@@ -61,7 +61,7 @@ export interface WatchdogDeps {
|
|
|
61
61
|
queryActiveSandboxes: () => ResultAsync<ActiveSandboxRow[], DbError>;
|
|
62
62
|
sandboxClient: SandboxClient;
|
|
63
63
|
shardCount?: number;
|
|
64
|
-
logger?:
|
|
64
|
+
logger?: Logger;
|
|
65
65
|
}
|
|
66
66
|
/**
|
|
67
67
|
* Production registration. Builds the parent + child registered workflows
|
package/dist/sandbox/watchdog.js
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
*/
|
|
21
21
|
import { createHash } from "node:crypto";
|
|
22
22
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
23
|
+
import { createNoopLogger } from "../lib/console-logger.js";
|
|
23
24
|
import { unwrapOrThrow } from "../lib/result.js";
|
|
24
25
|
import { workflowIdFromExec } from "./exec-id.js";
|
|
25
26
|
import { syntheticFailureReason, syntheticFailureResult } from "./liveness.js";
|
|
@@ -48,6 +49,7 @@ const ACTIVE_WORKFLOW_STATUSES = new Set(["PENDING", "ENQUEUED"]);
|
|
|
48
49
|
* notification.
|
|
49
50
|
*/
|
|
50
51
|
export async function checkShard(rows, deps) {
|
|
52
|
+
const logger = (deps.logger ?? createNoopLogger()).named("sandbox-watchdog");
|
|
51
53
|
let deadCount = 0;
|
|
52
54
|
let syntheticSends = 0;
|
|
53
55
|
let liveWorkflowsSkipped = 0;
|
|
@@ -64,7 +66,10 @@ export async function checkShard(rows, deps) {
|
|
|
64
66
|
liveness = await deps.isAlive(ref);
|
|
65
67
|
}
|
|
66
68
|
catch (err) {
|
|
67
|
-
|
|
69
|
+
logger.warn("isAlive threw — skipping this round", {
|
|
70
|
+
sandboxId: row.sandboxRef.sandboxId,
|
|
71
|
+
...logger.errorFields(err),
|
|
72
|
+
});
|
|
68
73
|
continue;
|
|
69
74
|
}
|
|
70
75
|
if (liveness.alive)
|
|
@@ -92,7 +97,8 @@ export async function checkShard(rows, deps) {
|
|
|
92
97
|
syntheticSends,
|
|
93
98
|
liveWorkflowsSkipped,
|
|
94
99
|
};
|
|
95
|
-
|
|
100
|
+
// Spread: an interface has no implicit index signature, so it needs widening to `LogFields`.
|
|
101
|
+
logger.info("shard check completed", { ...summary });
|
|
96
102
|
return summary;
|
|
97
103
|
}
|
|
98
104
|
/** Encodes the synthetic-failure marker on the per-exec topic envelope. */
|
package/dist/state/init.d.ts
CHANGED
|
@@ -7,8 +7,9 @@
|
|
|
7
7
|
* simultaneous startups don't race on DDL.
|
|
8
8
|
*/
|
|
9
9
|
import type { Pool } from "pg";
|
|
10
|
+
import type { Logger } from "../lib/logger.js";
|
|
10
11
|
/**
|
|
11
12
|
* Initialize Cortex state tables. Safe to call on every startup and across
|
|
12
13
|
* concurrent replicas (advisory lock serializes the init flow).
|
|
13
14
|
*/
|
|
14
|
-
export declare function initCortexState(pool: Pool): Promise<void>;
|
|
15
|
+
export declare function initCortexState(pool: Pool, injected?: Logger): Promise<void>;
|
package/dist/state/init.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* simultaneous startups don't race on DDL.
|
|
8
8
|
*/
|
|
9
9
|
import { backfillAiSdkMessageEnvelopes } from "../memory/message-backfill.js";
|
|
10
|
+
import { createNoopLogger } from "../lib/console-logger.js";
|
|
10
11
|
const DDL = `
|
|
11
12
|
-- Analysis-level state (status, context, data-profile tracking, billing identity).
|
|
12
13
|
--
|
|
@@ -245,7 +246,8 @@ CREATE TABLE IF NOT EXISTS cortex_working_memory (
|
|
|
245
246
|
* Initialize Cortex state tables. Safe to call on every startup and across
|
|
246
247
|
* concurrent replicas (advisory lock serializes the init flow).
|
|
247
248
|
*/
|
|
248
|
-
export async function initCortexState(pool) {
|
|
249
|
+
export async function initCortexState(pool, injected) {
|
|
250
|
+
const logger = (injected ?? createNoopLogger()).named("cortex-state");
|
|
249
251
|
const client = await pool.connect();
|
|
250
252
|
try {
|
|
251
253
|
// Serialize concurrent replica startups — cheap, no extra tooling.
|
|
@@ -381,11 +383,8 @@ export async function initCortexState(pool) {
|
|
|
381
383
|
WITH (m = 16, ef_construction = 64)`);
|
|
382
384
|
}
|
|
383
385
|
catch (err) {
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
`(pgvector < 0.5.0 or operator class unavailable): ${msg}. ` +
|
|
387
|
-
`Cosine similarity queries will fall back to sequential scan. ` +
|
|
388
|
-
`Upgrade pgvector and re-run init to get the ANN index.`);
|
|
386
|
+
logger.warn("could not create HNSW index on cortex_regulatory_chunks.embedding (pgvector < 0.5.0 or operator class unavailable) — " +
|
|
387
|
+
"cosine similarity queries fall back to sequential scan; upgrade pgvector and re-run init to get the ANN index", logger.errorFields(err));
|
|
389
388
|
}
|
|
390
389
|
// No boot-time orphan sweep on cortex_runs / cortex_step_executions.
|
|
391
390
|
// Every running row is backed by a DBOS workflow (executeAnalysis,
|
|
@@ -21,6 +21,7 @@ import { type AuthContext, type RunSession } from "../auth/types.js";
|
|
|
21
21
|
import type { RunAuthorization, RunAuthorizer } from "../execution/run-authorizer.js";
|
|
22
22
|
import type { StagedInput } from "../execution/staged-input.js";
|
|
23
23
|
import type { BioToolKeys } from "../tools/bio/keys.js";
|
|
24
|
+
import type { Logger } from "../lib/logger.js";
|
|
24
25
|
import type { ChatProvider, EmbeddingProvider } from "../providers/types.js";
|
|
25
26
|
import type { SandboxClient } from "../sandbox/client.js";
|
|
26
27
|
import type { WorkspaceFilesystem } from "../workspace/filesystem.js";
|
|
@@ -29,6 +30,8 @@ import { type ProfilerOutput } from "../schemas/data-profile-schemas.js";
|
|
|
29
30
|
import { type DataProfileInputFile, type DataProfileResult } from "../state/index.js";
|
|
30
31
|
/** The body's construction-time deps — closed over at registration. */
|
|
31
32
|
export interface DataProfileDeps {
|
|
33
|
+
/** Operational logging seam; omitted falls back to no-op. */
|
|
34
|
+
readonly logger?: Logger;
|
|
32
35
|
readonly provider: ChatProvider;
|
|
33
36
|
readonly pool: Pool;
|
|
34
37
|
readonly sandboxClient: SandboxClient;
|
|
@@ -130,6 +133,8 @@ export type DataProfileTriggerResult = "started" | "restarted" | "already_runnin
|
|
|
130
133
|
* (`registerDataProfileWorkflow`); the route never holds them.
|
|
131
134
|
*/
|
|
132
135
|
export interface DataProfileTriggerDeps {
|
|
136
|
+
/** Operational logging seam; omitted falls back to no-op. */
|
|
137
|
+
readonly logger?: Logger;
|
|
133
138
|
readonly pool: Pool;
|
|
134
139
|
readonly runAuthorizer: RunAuthorizer;
|
|
135
140
|
readonly workflow: (input: DataProfileWorkflowInput) => Promise<void>;
|