@inflexa-ai/harness 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/conversation-agent.d.ts +8 -9
- package/dist/agents/conversation-agent.js +7 -2
- package/dist/agents/sandbox/shared.d.ts +2 -13
- package/dist/config/environment-stores.d.ts +42 -0
- package/dist/config/environment-stores.js +11 -0
- package/dist/contracts/chat-parts.d.ts +4 -2
- package/dist/contracts/schemas/chat-parts.d.ts +8 -0
- package/dist/contracts/schemas/chat-parts.js +1 -1
- package/dist/execution/ephemeral-runner.d.ts +2 -13
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +5 -0
- package/dist/prompts/planner.js +13 -0
- package/dist/provenance/collector.d.ts +39 -2
- package/dist/provenance/collector.js +62 -33
- package/dist/provenance/exec-frame.d.ts +14 -2
- package/dist/provenance/exec-frame.js +27 -4
- package/dist/reference-data/catalog.js +680 -1
- package/dist/tasks/data-profile.d.ts +2 -13
- package/dist/tools/research/generate-plan.d.ts +2 -8
- package/dist/tools/research/generate-plan.js +14 -7
- package/dist/tools/sandbox/list-available-packages.d.ts +2 -10
- package/dist/tools/sandbox/list-available-refs.d.ts +6 -9
- package/dist/tools/sandbox/list-available-refs.js +11 -4
- package/dist/tools/workspace/execute-command.js +1 -0
- package/dist/workflows/execute-analysis.d.ts +10 -5
- package/dist/workflows/execute-analysis.js +102 -36
- package/dist/workflows/sandbox-step.d.ts +25 -0
- package/dist/workflows/sandbox-step.js +20 -9
- package/package.json +2 -1
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import type { Pool } from "pg";
|
|
20
20
|
import { type AuthContext, type RunSession } from "../auth/types.js";
|
|
21
|
+
import type { EnvironmentStorePaths } from "../config/environment-stores.js";
|
|
21
22
|
import type { RunAuthorization, RunAuthorizer } from "../execution/run-authorizer.js";
|
|
22
23
|
import type { StagedInput } from "../execution/staged-input.js";
|
|
23
24
|
import type { BioToolKeys } from "../tools/bio/keys.js";
|
|
@@ -29,7 +30,7 @@ import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
|
29
30
|
import { type ProfilerOutput } from "../schemas/data-profile-schemas.js";
|
|
30
31
|
import { type DataProfileInputFile, type DataProfileResult } from "../state/index.js";
|
|
31
32
|
/** The body's construction-time deps — closed over at registration. */
|
|
32
|
-
export interface DataProfileDeps {
|
|
33
|
+
export interface DataProfileDeps extends EnvironmentStorePaths {
|
|
33
34
|
/** Operational logging seam; omitted falls back to no-op. */
|
|
34
35
|
readonly logger?: Logger;
|
|
35
36
|
readonly provider: ChatProvider;
|
|
@@ -52,18 +53,6 @@ export interface DataProfileDeps {
|
|
|
52
53
|
readonly embedding: EmbeddingProvider;
|
|
53
54
|
/** Absolute path to the skills tree (one subdirectory per skill). */
|
|
54
55
|
readonly skillsDir: string;
|
|
55
|
-
/**
|
|
56
|
-
* Host path of the library store's `packages.txt`. Omit when the host mounts the
|
|
57
|
-
* store at the sandbox's own path; a host whose store is baked into the image must
|
|
58
|
-
* inject its extracted copy, or the profiler reads the inventory as unknown.
|
|
59
|
-
*/
|
|
60
|
-
readonly packagesFile?: string;
|
|
61
|
-
/**
|
|
62
|
-
* Host path of the reference store. Omit only when none is provisioned: the
|
|
63
|
-
* profiler carries reference discovery, and without this it reports an installed
|
|
64
|
-
* store as absent.
|
|
65
|
-
*/
|
|
66
|
-
readonly refStorePath?: string;
|
|
67
56
|
}
|
|
68
57
|
/**
|
|
69
58
|
* Workflow input. JSON-serialisable — DBOS persists it as the workflow's
|
|
@@ -27,8 +27,9 @@
|
|
|
27
27
|
import type { Pool } from "pg";
|
|
28
28
|
import { type ChatProvider } from "../../providers/types.js";
|
|
29
29
|
import { type Tool } from "../define-tool.js";
|
|
30
|
+
import type { EnvironmentStorePaths } from "../../config/environment-stores.js";
|
|
30
31
|
import { type ResourcePolicy } from "../../config/resource-limits.js";
|
|
31
|
-
export interface GeneratePlanDeps {
|
|
32
|
+
export interface GeneratePlanDeps extends EnvironmentStorePaths {
|
|
32
33
|
/** The LLM seam the planner loop runs on. */
|
|
33
34
|
readonly provider: ChatProvider;
|
|
34
35
|
/** Database pool — plan persistence and prior-plan loading. */
|
|
@@ -41,13 +42,6 @@ export interface GeneratePlanDeps {
|
|
|
41
42
|
* default guidance and validation skips the ceiling check.
|
|
42
43
|
*/
|
|
43
44
|
readonly resourcePolicy?: ResourcePolicy;
|
|
44
|
-
/**
|
|
45
|
-
* Host path of the reference store, giving the planner reference discovery.
|
|
46
|
-
* Absent, the tool reports the store as unavailable and the planner keeps
|
|
47
|
-
* planning as though nothing is guaranteed present — which is what the prompt
|
|
48
|
-
* already tells it to do.
|
|
49
|
-
*/
|
|
50
|
-
readonly refStorePath?: string;
|
|
51
45
|
}
|
|
52
46
|
/** Build the `generate_plan` tool bound to its provider and pool. */
|
|
53
47
|
export declare function createGeneratePlanTool(deps: GeneratePlanDeps): Tool;
|
|
@@ -32,6 +32,7 @@ import { runToTerminal } from "../../loop/run-to-terminal.js";
|
|
|
32
32
|
import { passthroughStep } from "../../loop/run-step.js";
|
|
33
33
|
import { forSubAgent, scopeResource } from "../../auth/types.js";
|
|
34
34
|
import { defineTool } from "../define-tool.js";
|
|
35
|
+
import { createListAvailablePackagesTool } from "../sandbox/list-available-packages.js";
|
|
35
36
|
import { createListAvailableRefsTool } from "../sandbox/list-available-refs.js";
|
|
36
37
|
import { createReportBlockerToolFor } from "../sandbox/report-blocker.js";
|
|
37
38
|
import { DATA_PROFILE_ORIENTATION_MAX_CHARS, buildDataProfileOrientation } from "../../app/data-profile-orientation.js";
|
|
@@ -250,7 +251,10 @@ function fullyValidate(candidate, resourcePolicy) {
|
|
|
250
251
|
* shared `holder` so the outer `execute` reads the terminal outcome after the
|
|
251
252
|
* loop finishes.
|
|
252
253
|
*/
|
|
253
|
-
function buildInnerTools(holder, persistCtx, pool, resourcePolicy,
|
|
254
|
+
function buildInnerTools(holder, persistCtx, pool, resourcePolicy,
|
|
255
|
+
// The two environment stores travel together as one bag rather than as a pair
|
|
256
|
+
// of adjacent optional strings, which would be silently swappable at the call site.
|
|
257
|
+
stores) {
|
|
254
258
|
const validatePlanTool = defineTool({
|
|
255
259
|
id: "validate_plan",
|
|
256
260
|
description: "Dry-run a candidate plan and get back everything that is wrong with it. " +
|
|
@@ -354,12 +358,15 @@ function buildInnerTools(holder, persistCtx, pool, resourcePolicy, refStorePath)
|
|
|
354
358
|
"incompatible with every available agent, etc.) — not for a plan you " +
|
|
355
359
|
"could fix and submit.",
|
|
356
360
|
});
|
|
357
|
-
//
|
|
358
|
-
// reference data this environment
|
|
359
|
-
//
|
|
360
|
-
|
|
361
|
+
// Environment discovery is non-terminal and read-only: it lets the planner check what
|
|
362
|
+
// reference data this environment holds, and what a step will actually be able to
|
|
363
|
+
// import, before committing a step to needing either. Both read host-side, so no
|
|
364
|
+
// sandbox exists or is needed at plan time — which is the whole point, since a plan
|
|
365
|
+
// that assumes an absent package fails only once the run reaches that step.
|
|
366
|
+
const listAvailableRefs = createListAvailableRefsTool(stores.refStorePath === undefined ? {} : { refStorePath: stores.refStorePath });
|
|
367
|
+
const listAvailablePackages = createListAvailablePackagesTool(stores.packagesFile === undefined ? {} : { packagesFile: stores.packagesFile });
|
|
361
368
|
const terminal = [submitPlanTool, requestClarificationTool, reportBlockerTool];
|
|
362
|
-
return { all: [validatePlanTool, listAvailableRefs, ...terminal], terminal };
|
|
369
|
+
return { all: [validatePlanTool, listAvailableRefs, listAvailablePackages, ...terminal], terminal };
|
|
363
370
|
}
|
|
364
371
|
/**
|
|
365
372
|
* Translate the captured `PlannerOutcome` (plus any loop error) into the
|
|
@@ -491,7 +498,7 @@ export function createGeneratePlanTool(deps) {
|
|
|
491
498
|
analysisId,
|
|
492
499
|
parentPlanId: input.parentPlanId ?? null,
|
|
493
500
|
};
|
|
494
|
-
const innerTools = buildInnerTools(holder, persistCtx, deps.pool, deps.resourcePolicy, deps
|
|
501
|
+
const innerTools = buildInnerTools(holder, persistCtx, deps.pool, deps.resourcePolicy, deps);
|
|
495
502
|
const planner = {
|
|
496
503
|
id: PLANNER_AGENT_ID,
|
|
497
504
|
systemPrompt: composeSystemPrompt(plannerInstructions(deps.resourcePolicy)),
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* The file carries **names only — there are no version strings in it**, so this
|
|
23
23
|
* tool reports presence and language track and cannot report a version.
|
|
24
24
|
*/
|
|
25
|
+
import type { EnvironmentStorePaths } from "../../config/environment-stores.js";
|
|
25
26
|
/** One language track of the store, in `packages.txt` section order. */
|
|
26
27
|
export interface Section {
|
|
27
28
|
/** The section heading verbatim, e.g. `R (CRAN)`, `Python (pip)`. */
|
|
@@ -75,16 +76,7 @@ export interface PackagesQuery {
|
|
|
75
76
|
* is only the file read plus this call.
|
|
76
77
|
*/
|
|
77
78
|
export declare function queryPackages(sections: readonly Section[], { names, query, language, limit }: PackagesQuery): PackagesResult;
|
|
78
|
-
export
|
|
79
|
-
/**
|
|
80
|
-
* Path to the store's `packages.txt` AS THE HOST SEES IT. Defaults to the
|
|
81
|
-
* container path, which is correct only when the host mounts the same store.
|
|
82
|
-
* A host that does not — one whose store is baked into the sandbox image and
|
|
83
|
-
* never bind-mounted — must inject the path to its own extracted copy, or the
|
|
84
|
-
* read fails and every caller is told the inventory is unknown.
|
|
85
|
-
*/
|
|
86
|
-
readonly packagesFile?: string;
|
|
87
|
-
}
|
|
79
|
+
export type ListAvailablePackagesDeps = Pick<EnvironmentStorePaths, "packagesFile">;
|
|
88
80
|
/** Create the package inventory over a host-readable `packages.txt`. */
|
|
89
81
|
export declare function createListAvailablePackagesTool(deps?: ListAvailablePackagesDeps): import("../define-tool.js").Tool<{
|
|
90
82
|
names?: string[] | undefined;
|
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
* installs and user-added files alike). Narrow with `path`/`category`, filter
|
|
5
5
|
* with `query`, cap with `limit`; every response carries `returned`/`total`/
|
|
6
6
|
* `hasMore` and the `categories` present, so a listing is never an unbounded
|
|
7
|
-
* dump. Symlinks are never followed
|
|
7
|
+
* dump. Symlinks are never followed. This tool only ever READS the store: the
|
|
8
|
+
* analysis environment cannot fetch reference data, and staging more is the
|
|
9
|
+
* host's job — so an absent dataset is a fact about this moment, not a verdict
|
|
10
|
+
* that it is unobtainable.
|
|
8
11
|
*
|
|
9
12
|
* The store is read from the host filesystem at the embedder-supplied
|
|
10
13
|
* `refStorePath` and reported at `/mnt/refs`, the container path sandboxes mount
|
|
@@ -20,14 +23,8 @@
|
|
|
20
23
|
* only ever ENRICHES an entry — a file with no receipt and no catalog match is
|
|
21
24
|
* still returned, because a user-provided store is as valid as a managed one.
|
|
22
25
|
*/
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
* Host path of the reference store — the same bytes sandboxes see at
|
|
26
|
-
* `/mnt/refs`. Omitted (or absent on disk) means no store is provisioned,
|
|
27
|
-
* which is a normal state reported as `unavailable`, not an error.
|
|
28
|
-
*/
|
|
29
|
-
readonly refStorePath?: string;
|
|
30
|
-
}
|
|
26
|
+
import type { EnvironmentStorePaths } from "../../config/environment-stores.js";
|
|
27
|
+
export type ListAvailableRefsDeps = Pick<EnvironmentStorePaths, "refStorePath">;
|
|
31
28
|
export interface ReferenceInventoryMetadata {
|
|
32
29
|
readonly datasetId?: string;
|
|
33
30
|
readonly version?: string;
|
|
@@ -4,7 +4,10 @@
|
|
|
4
4
|
* installs and user-added files alike). Narrow with `path`/`category`, filter
|
|
5
5
|
* with `query`, cap with `limit`; every response carries `returned`/`total`/
|
|
6
6
|
* `hasMore` and the `categories` present, so a listing is never an unbounded
|
|
7
|
-
* dump. Symlinks are never followed
|
|
7
|
+
* dump. Symlinks are never followed. This tool only ever READS the store: the
|
|
8
|
+
* analysis environment cannot fetch reference data, and staging more is the
|
|
9
|
+
* host's job — so an absent dataset is a fact about this moment, not a verdict
|
|
10
|
+
* that it is unobtainable.
|
|
8
11
|
*
|
|
9
12
|
* The store is read from the host filesystem at the embedder-supplied
|
|
10
13
|
* `refStorePath` and reported at `/mnt/refs`, the container path sandboxes mount
|
|
@@ -456,7 +459,8 @@ function collectCategories(entries) {
|
|
|
456
459
|
}
|
|
457
460
|
function renderContent(path, scan, entries) {
|
|
458
461
|
if (scan.state === "unavailable")
|
|
459
|
-
return "No reference store is provisioned.
|
|
462
|
+
return ("No reference store is provisioned. Nothing in the analysis environment can fetch reference data — the host stages it, " +
|
|
463
|
+
"so a store provisioned through the host's own path will show up on a later call.");
|
|
460
464
|
if (scan.state === "not_found")
|
|
461
465
|
return `Reference path does not exist: ${path}`;
|
|
462
466
|
if (scan.state === "not_a_directory")
|
|
@@ -492,7 +496,8 @@ function renderContent(path, scan, entries) {
|
|
|
492
496
|
export function createListAvailableRefsTool(deps) {
|
|
493
497
|
return defineTool({
|
|
494
498
|
id: "list_available_refs",
|
|
495
|
-
description: "Resolve reference data you need into a real path. Reports the reference data
|
|
499
|
+
description: "Resolve reference data you need into a real path. Reports the reference data provisioned for this environment right now — the only reference data an analysis script can open. " +
|
|
500
|
+
"Nothing inside the analysis environment reaches the network, so never write a download into a script. " +
|
|
496
501
|
"Search by what the data IS, not by where you expect it: the store's directory layout is an installer detail and is not a stable interface. " +
|
|
497
502
|
"Catalogued files come back with the organism, the format, and the file's internal shape (key columns, identifier space), so you can pick the right reader and the right species without opening the file first. " +
|
|
498
503
|
"User-added files are returned too, with whatever labels exist — often none. Symlinks are never followed. " +
|
|
@@ -502,7 +507,9 @@ export function createListAvailableRefsTool(deps) {
|
|
|
502
507
|
"`path` inspects one directory beneath /mnt/refs (a returned path, or store-relative) — drill in with it when a listing is truncated; " +
|
|
503
508
|
'`category` is shorthand for a top-level group (e.g. "pathways", "regulatory-networks"), and the groups present come back in `categories`; ' +
|
|
504
509
|
"`limit` caps the entries returned. The response carries `returned`, `total`, and `hasMore`, so truncation is never silent. " +
|
|
505
|
-
"An empty result means the
|
|
510
|
+
"An empty result means the dataset is absent right now — not that it is unobtainable. Say so; never guess a path, and never silently substitute a different dataset. " +
|
|
511
|
+
"The store is provisioned by the host and is NOT frozen: datasets can be added through the host's own provisioning path, and whatever is added shows up on a later call. " +
|
|
512
|
+
"So before you report a gap as permanent, check whether any tool you hold reaches that provisioning path — offer that route if one exists, and call the gap permanent only when none does.",
|
|
506
513
|
inputSchema: ListAvailableRefsInputSchema,
|
|
507
514
|
execute: async ({ path, category, query, limit }) => {
|
|
508
515
|
// `category` is shorthand for a top-level path; an explicit `path` wins.
|
|
@@ -30,11 +30,16 @@
|
|
|
30
30
|
* completion
|
|
31
31
|
* - await child results via `getResult` (these are themselves cached
|
|
32
32
|
* in DBOS, so parent recovery does NOT re-run completed children)
|
|
33
|
-
* 3.
|
|
34
|
-
* -
|
|
35
|
-
*
|
|
36
|
-
* `
|
|
37
|
-
*
|
|
33
|
+
* 3. Failure isolation
|
|
34
|
+
* - a child settling `failed`/`blocked` (or throwing for a non-budget
|
|
35
|
+
* cause) dooms only its transitive dependents: they can never become
|
|
36
|
+
* dependency-satisfied, are marked `skipped` on the dag-state part, and
|
|
37
|
+
* are never dispatched. In-flight siblings and independent ready steps
|
|
38
|
+
* continue; the loop drains and the run finalises (typically `partial`).
|
|
39
|
+
* - the halt cascade (cancel in-flight via
|
|
40
|
+
* `Promise.allSettled(handles.map(h => DBOS.cancelWorkflow(h.workflowID)))`,
|
|
41
|
+
* stop scheduling) survives only for the budget paths: a
|
|
42
|
+
* `budget_exceeded` settlement and the `neverFits` plan-validation guard.
|
|
38
43
|
* 4. Pause cascade (402)
|
|
39
44
|
* - on a child cancelling itself with `budget_exceeded`: cancel siblings
|
|
40
45
|
* then self-cancel the parent to `CANCELLED` (NOT `ERROR` — ERROR
|
|
@@ -30,11 +30,16 @@
|
|
|
30
30
|
* completion
|
|
31
31
|
* - await child results via `getResult` (these are themselves cached
|
|
32
32
|
* in DBOS, so parent recovery does NOT re-run completed children)
|
|
33
|
-
* 3.
|
|
34
|
-
* -
|
|
35
|
-
*
|
|
36
|
-
* `
|
|
37
|
-
*
|
|
33
|
+
* 3. Failure isolation
|
|
34
|
+
* - a child settling `failed`/`blocked` (or throwing for a non-budget
|
|
35
|
+
* cause) dooms only its transitive dependents: they can never become
|
|
36
|
+
* dependency-satisfied, are marked `skipped` on the dag-state part, and
|
|
37
|
+
* are never dispatched. In-flight siblings and independent ready steps
|
|
38
|
+
* continue; the loop drains and the run finalises (typically `partial`).
|
|
39
|
+
* - the halt cascade (cancel in-flight via
|
|
40
|
+
* `Promise.allSettled(handles.map(h => DBOS.cancelWorkflow(h.workflowID)))`,
|
|
41
|
+
* stop scheduling) survives only for the budget paths: a
|
|
42
|
+
* `budget_exceeded` settlement and the `neverFits` plan-validation guard.
|
|
38
43
|
* 4. Pause cascade (402)
|
|
39
44
|
* - on a child cancelling itself with `budget_exceeded`: cancel siblings
|
|
40
45
|
* then self-cancel the parent to `CANCELLED` (NOT `ERROR` — ERROR
|
|
@@ -105,11 +110,23 @@ export function buildChildInput(args) {
|
|
|
105
110
|
if (!resources) {
|
|
106
111
|
throw new Error(`executeAnalysis: step "${stepId}" missing from resourcesByStepId — every step must declare resources`);
|
|
107
112
|
}
|
|
113
|
+
const planStep = input.planStepById[stepId];
|
|
114
|
+
if (!planStep) {
|
|
115
|
+
throw new Error(`executeAnalysis: step "${stepId}" missing from planStepById — every step must carry its plan data`);
|
|
116
|
+
}
|
|
108
117
|
return {
|
|
109
118
|
analysisId: input.analysisId,
|
|
110
119
|
runId,
|
|
111
120
|
stepId,
|
|
112
121
|
agentId: input.agentByStepId[stepId] ?? "scientific-executor",
|
|
122
|
+
// Carried into the child because lineage classification admits a same-run
|
|
123
|
+
// sibling edge only when the sibling was declared here — the scheduler's
|
|
124
|
+
// ordering guarantee attaches to this list and nothing else. Read from the
|
|
125
|
+
// keyed plan snapshot rather than scanning `steps`, and thrown for rather
|
|
126
|
+
// than defaulted: an empty list is indistinguishable from a step that
|
|
127
|
+
// genuinely declared nothing, and it silently deletes every same-run edge
|
|
128
|
+
// the step was entitled to.
|
|
129
|
+
dependsOn: planStep.depends_on,
|
|
113
130
|
level,
|
|
114
131
|
prompt,
|
|
115
132
|
parentWorkflowId: workflowId,
|
|
@@ -319,7 +336,7 @@ export async function runExecuteAnalysisBody(input, deps) {
|
|
|
319
336
|
stepCount: input.steps.length,
|
|
320
337
|
atMs: startedAtMs,
|
|
321
338
|
});
|
|
322
|
-
// (2-4) Scheduler loop +
|
|
339
|
+
// (2-4) Scheduler loop + failure isolation + pause cascade.
|
|
323
340
|
const final = await runSchedulerLoop({
|
|
324
341
|
input,
|
|
325
342
|
runId,
|
|
@@ -434,7 +451,7 @@ async function runSchedulerLoop(args) {
|
|
|
434
451
|
const canceled = new Set();
|
|
435
452
|
const inFlight = new Map();
|
|
436
453
|
const budgetExceededChildIds = new Set();
|
|
437
|
-
// Child workflow ids the parent itself cancelled (
|
|
454
|
+
// Child workflow ids the parent itself cancelled (budget / neverFits halt cascade). Their `getResult`
|
|
438
455
|
// rejects with a DBOS cancellation error and lands in the settlement error branch — but the step was
|
|
439
456
|
// CANCELED by us, not failed on its own merits. Keyed here (deterministic: `cancelInFlight` runs the
|
|
440
457
|
// same way on a replay) so that branch records it as canceled everywhere, matching how a child that
|
|
@@ -444,7 +461,40 @@ async function runSchedulerLoop(args) {
|
|
|
444
461
|
for (const step of input.steps) {
|
|
445
462
|
stepRuntime.set(step.id, { status: "pending" });
|
|
446
463
|
}
|
|
447
|
-
|
|
464
|
+
// Reverse adjacency over the static plan — who depends on whom.
|
|
465
|
+
const dependentsByStepId = new Map();
|
|
466
|
+
for (const step of input.steps) {
|
|
467
|
+
for (const dep of step.depends_on) {
|
|
468
|
+
const list = dependentsByStepId.get(dep);
|
|
469
|
+
if (list)
|
|
470
|
+
list.push(step.id);
|
|
471
|
+
else
|
|
472
|
+
dependentsByStepId.set(dep, [step.id]);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
/**
|
|
476
|
+
* Mark every transitive dependent of a failed/blocked step `skipped` in the
|
|
477
|
+
* dag snapshot: a failed step never enters `completed`, so its dependents
|
|
478
|
+
* can never become dependency-satisfied and will never be dispatched.
|
|
479
|
+
* Stream-only — ledger rows stay `pending` until the terminal sweep flips
|
|
480
|
+
* them to `skipped`. Dependents of an unfinished step are necessarily
|
|
481
|
+
* `pending` (never queued/running); an already-`skipped` one is not
|
|
482
|
+
* re-walked.
|
|
483
|
+
*/
|
|
484
|
+
const markDependentsSkipped = (failedStepId) => {
|
|
485
|
+
const stack = [...(dependentsByStepId.get(failedStepId) ?? [])];
|
|
486
|
+
while (stack.length > 0) {
|
|
487
|
+
const id = stack.pop();
|
|
488
|
+
if (stepRuntime.get(id).status === "pending") {
|
|
489
|
+
stepRuntime.set(id, { status: "skipped" });
|
|
490
|
+
stack.push(...(dependentsByStepId.get(id) ?? []));
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
};
|
|
494
|
+
// Halts scheduling entirely — reserved for the budget paths (a
|
|
495
|
+
// `budget_exceeded` settlement, the `neverFits` guard). An ordinary step
|
|
496
|
+
// failure does NOT set it: it dooms only that step's transitive dependents.
|
|
497
|
+
let halted = false;
|
|
448
498
|
let budgetExceeded = false;
|
|
449
499
|
let failureReason = null;
|
|
450
500
|
const stepIndexById = new Map(input.steps.map((s, i) => [s.id, i]));
|
|
@@ -473,20 +523,28 @@ async function runSchedulerLoop(args) {
|
|
|
473
523
|
};
|
|
474
524
|
const dbosParentId = DBOS.workflowID ?? workflowId;
|
|
475
525
|
const dispatchReady = async () => {
|
|
476
|
-
if (
|
|
526
|
+
if (halted)
|
|
477
527
|
return;
|
|
478
528
|
const inFlightStepIds = new Set([...inFlight.values()].map((h) => h.stepId));
|
|
529
|
+
// A settled-but-not-completed step (failed / blocked / canceled) is not
|
|
530
|
+
// in `completed` and not in flight, so `scheduleReady` would offer it
|
|
531
|
+
// again — filter it out of the candidate list. This also stops its
|
|
532
|
+
// declared resources from weighing against the budget: settled steps
|
|
533
|
+
// hold no capacity.
|
|
534
|
+
const candidates = input.steps.filter((s) => !failed.has(s.id) && !canceled.has(s.id));
|
|
479
535
|
let admit;
|
|
480
536
|
let queuedChanged = false;
|
|
481
537
|
if (input.budget) {
|
|
482
|
-
const scheduled = scheduleReady(
|
|
538
|
+
const scheduled = scheduleReady(candidates, completed, inFlightStepIds, {
|
|
483
539
|
budget: input.budget,
|
|
484
540
|
resourcesByStepId: input.resourcesByStepId,
|
|
485
541
|
});
|
|
486
542
|
if (scheduled.neverFits.length > 0) {
|
|
487
543
|
// An over-budget step can never be admitted — plan-time validation
|
|
488
544
|
// makes this unreachable for new plans; stored plans fail loudly
|
|
489
|
-
// instead of waiting forever.
|
|
545
|
+
// instead of waiting forever. The stored plan is invalid against
|
|
546
|
+
// this machine, so the run halts outright — unlike an ordinary
|
|
547
|
+
// step failure, which dooms only its dependents.
|
|
490
548
|
for (const stepId of scheduled.neverFits) {
|
|
491
549
|
const r = input.resourcesByStepId[stepId];
|
|
492
550
|
const declared = r ? `${r.cpu} CPU / ${r.memoryGb} GB` : "undeclared";
|
|
@@ -496,7 +554,7 @@ async function runSchedulerLoop(args) {
|
|
|
496
554
|
error: `step resources (${declared}) exceed the machine budget (${input.budget.cpu} CPU / ${input.budget.memoryGb} GB)`,
|
|
497
555
|
});
|
|
498
556
|
}
|
|
499
|
-
|
|
557
|
+
halted = true;
|
|
500
558
|
failureReason = `step "${scheduled.neverFits[0]}" exceeds the machine resource budget`;
|
|
501
559
|
await cancelInFlight(inFlight, "fail_fast", canceledByParent);
|
|
502
560
|
await emitDagSnapshot();
|
|
@@ -511,7 +569,7 @@ async function runSchedulerLoop(args) {
|
|
|
511
569
|
admit = scheduled.admit;
|
|
512
570
|
}
|
|
513
571
|
else {
|
|
514
|
-
admit = scheduleReady(
|
|
572
|
+
admit = scheduleReady(candidates, completed, inFlightStepIds);
|
|
515
573
|
}
|
|
516
574
|
if (admit.length === 0) {
|
|
517
575
|
if (queuedChanged)
|
|
@@ -590,7 +648,7 @@ async function runSchedulerLoop(args) {
|
|
|
590
648
|
const isBudgetCancel = childWasBudgetExceeded || r.error === "budget_exceeded";
|
|
591
649
|
if (isBudgetCancel && !budgetExceeded) {
|
|
592
650
|
budgetExceeded = true;
|
|
593
|
-
|
|
651
|
+
halted = true;
|
|
594
652
|
failureReason = "budget_exceeded";
|
|
595
653
|
await cancelInFlight(inFlight, "budget_exceeded", canceledByParent);
|
|
596
654
|
}
|
|
@@ -604,31 +662,31 @@ async function runSchedulerLoop(args) {
|
|
|
604
662
|
await emitDagSnapshot();
|
|
605
663
|
}
|
|
606
664
|
else {
|
|
607
|
-
// failed or blocked — a blocker (see the harness-sandbox-agents spec)
|
|
608
|
-
// failure: the step declared it cannot
|
|
609
|
-
//
|
|
665
|
+
// failed or blocked — a blocker (see the harness-sandbox-agents spec)
|
|
666
|
+
// is treated exactly like a failure: the step declared it cannot
|
|
667
|
+
// deliver, so only its transitive dependents can never run. In-flight
|
|
668
|
+
// siblings and independent ready steps continue; the dispatch round
|
|
669
|
+
// below also lets a budget-held step claim the freed capacity.
|
|
610
670
|
failed.add(settled.stepId);
|
|
611
|
-
|
|
612
|
-
failFast = true;
|
|
613
|
-
failureReason = r.error ?? (r.status === "blocked" ? "step_blocked" : "step_failed");
|
|
614
|
-
await cancelInFlight(inFlight, "fail_fast", canceledByParent);
|
|
615
|
-
}
|
|
671
|
+
failureReason ??= r.error ?? (r.status === "blocked" ? "step_blocked" : "step_failed");
|
|
616
672
|
stepRuntime.set(settled.stepId, {
|
|
617
673
|
status: "failed",
|
|
618
674
|
durationMs: r.durationMs ?? undefined,
|
|
619
675
|
error: r.error ?? (r.status === "blocked" ? "step_blocked" : "step_failed"),
|
|
620
676
|
});
|
|
677
|
+
markDependentsSkipped(settled.stepId);
|
|
621
678
|
stepStatus = "failed";
|
|
622
679
|
stepDurationMs = r.durationMs ?? undefined;
|
|
623
680
|
await emitDagSnapshot();
|
|
681
|
+
await dispatchReady();
|
|
624
682
|
}
|
|
625
683
|
}
|
|
626
684
|
else if (canceledByParent.has(settled.childId)) {
|
|
627
|
-
// The child threw because the PARENT cancelled it (
|
|
628
|
-
// it failed on its own — its `getResult` rejects with a DBOS cancellation error.
|
|
629
|
-
// canceled, mirroring the graceful `{status:"canceled"}` branch above so the
|
|
630
|
-
// the terminal result, and the `step_completed` provenance all agree. No
|
|
631
|
-
// cascade here: the run is already
|
|
685
|
+
// The child threw because the PARENT cancelled it (budget / neverFits halt cascade), not
|
|
686
|
+
// because it failed on its own — its `getResult` rejects with a DBOS cancellation error.
|
|
687
|
+
// Record it as canceled, mirroring the graceful `{status:"canceled"}` branch above so the
|
|
688
|
+
// `canceled` set, the terminal result, and the `step_completed` provenance all agree. No
|
|
689
|
+
// halt / cancel cascade here: the run is already halted (that is why this child was cancelled).
|
|
632
690
|
canceled.add(settled.stepId);
|
|
633
691
|
stepRuntime.set(settled.stepId, {
|
|
634
692
|
// `stepRuntime` (the DAG snapshot) has no canceled tier; the graceful-cancel branch above
|
|
@@ -641,18 +699,24 @@ async function runSchedulerLoop(args) {
|
|
|
641
699
|
await emitDagSnapshot();
|
|
642
700
|
}
|
|
643
701
|
else {
|
|
702
|
+
// The child threw on its own (not a parent-driven cancel). A
|
|
703
|
+
// budget_exceeded throw still halts the run — the pause cascade is a
|
|
704
|
+
// money decision, not a DAG one. Any other throw is treated exactly
|
|
705
|
+
// like a step failure: dependents doomed, siblings continue.
|
|
644
706
|
failed.add(settled.stepId);
|
|
645
|
-
const
|
|
646
|
-
if (
|
|
647
|
-
budgetExceeded
|
|
648
|
-
|
|
649
|
-
|
|
707
|
+
const isBudgetThrow = childWasBudgetExceeded || isBudgetExceeded(settled.err);
|
|
708
|
+
if (isBudgetThrow) {
|
|
709
|
+
if (!budgetExceeded) {
|
|
710
|
+
budgetExceeded = true;
|
|
711
|
+
halted = true;
|
|
712
|
+
failureReason = "budget_exceeded";
|
|
713
|
+
}
|
|
714
|
+
await cancelInFlight(inFlight, "budget_exceeded", canceledByParent);
|
|
650
715
|
}
|
|
651
|
-
else
|
|
652
|
-
|
|
653
|
-
|
|
716
|
+
else {
|
|
717
|
+
failureReason ??= settled.err instanceof Error ? settled.err.message : String(settled.err);
|
|
718
|
+
markDependentsSkipped(settled.stepId);
|
|
654
719
|
}
|
|
655
|
-
await cancelInFlight(inFlight, cause, canceledByParent);
|
|
656
720
|
stepRuntime.set(settled.stepId, {
|
|
657
721
|
status: "failed",
|
|
658
722
|
error: settled.err instanceof Error ? settled.err.message : String(settled.err),
|
|
@@ -661,6 +725,8 @@ async function runSchedulerLoop(args) {
|
|
|
661
725
|
// A child that settled by throwing carries no durable duration.
|
|
662
726
|
stepDurationMs = undefined;
|
|
663
727
|
await emitDagSnapshot();
|
|
728
|
+
if (!isBudgetThrow)
|
|
729
|
+
await dispatchReady();
|
|
664
730
|
}
|
|
665
731
|
// One `step_completed` provenance emission per settled child — this
|
|
666
732
|
// settlement loop is the only site that observes every executed step.
|
|
@@ -51,6 +51,18 @@ export interface SandboxStepInput {
|
|
|
51
51
|
readonly runId: string;
|
|
52
52
|
readonly stepId: string;
|
|
53
53
|
readonly agentId: string;
|
|
54
|
+
/**
|
|
55
|
+
* Step ids this step declared as dependencies. Load-bearing for lineage:
|
|
56
|
+
* classification admits a same-run sibling edge only on declaration, so a
|
|
57
|
+
* collector seeded without this refuses every same-run read — legitimate
|
|
58
|
+
* declared dependencies included.
|
|
59
|
+
*
|
|
60
|
+
* Optional because this is durable DBOS workflow input: a workflow started
|
|
61
|
+
* under an input shape predating the field recovers without it. Absence
|
|
62
|
+
* reads as an empty declaration list, which under-captures sibling edges
|
|
63
|
+
* rather than admitting unstable ones.
|
|
64
|
+
*/
|
|
65
|
+
readonly dependsOn?: readonly string[];
|
|
54
66
|
/** Topological level — persisted on `cortex_step_executions.wave`. */
|
|
55
67
|
readonly level: number;
|
|
56
68
|
/** User-content prompt the agent receives as its initial message. */
|
|
@@ -240,6 +252,19 @@ export interface PostStepContext {
|
|
|
240
252
|
/** Same step-scoped collector the agent loop fed each exec frame into. */
|
|
241
253
|
readonly lineageCollector: ProvenanceCollector;
|
|
242
254
|
}
|
|
255
|
+
/**
|
|
256
|
+
* Build the step's lineage collector from its durable input. The agent loop
|
|
257
|
+
* feeds each exec's provenance frame into it; post-step registration reads the
|
|
258
|
+
* resulting input/script edges. Reconstructed deterministically on replay — the
|
|
259
|
+
* frames it consumes are cached recv outputs and the command strings are cached
|
|
260
|
+
* LLM tool-use (see the harness-thread-store and harness-durable-runtime specs).
|
|
261
|
+
*
|
|
262
|
+
* Split out of the body, and taking only the fields it reads, so the seeding is
|
|
263
|
+
* exercisable without a live workflow: `dependsOn` is what makes a same-run edge
|
|
264
|
+
* admissible at all, so a collector built without it silently erases every
|
|
265
|
+
* legitimate declared-dependency edge while leaving each layer's own tests green.
|
|
266
|
+
*/
|
|
267
|
+
export declare function createLineageCollector(input: Pick<SandboxStepInput, "stepId" | "runId" | "dependsOn">): ProvenanceCollector;
|
|
243
268
|
/**
|
|
244
269
|
* Register the sandbox-step child workflow with DBOS. Returns the
|
|
245
270
|
* registered callable so callers can dispatch via `DBOS.startWorkflow`.
|
|
@@ -82,6 +82,25 @@ function stepPartId(kind, runId, stepId) {
|
|
|
82
82
|
function userFacingStepFailure(errorClass) {
|
|
83
83
|
return errorClass === "lineage_attestation" ? "Step results could not be finalized." : "Step failed during execution.";
|
|
84
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* Build the step's lineage collector from its durable input. The agent loop
|
|
87
|
+
* feeds each exec's provenance frame into it; post-step registration reads the
|
|
88
|
+
* resulting input/script edges. Reconstructed deterministically on replay — the
|
|
89
|
+
* frames it consumes are cached recv outputs and the command strings are cached
|
|
90
|
+
* LLM tool-use (see the harness-thread-store and harness-durable-runtime specs).
|
|
91
|
+
*
|
|
92
|
+
* Split out of the body, and taking only the fields it reads, so the seeding is
|
|
93
|
+
* exercisable without a live workflow: `dependsOn` is what makes a same-run edge
|
|
94
|
+
* admissible at all, so a collector built without it silently erases every
|
|
95
|
+
* legitimate declared-dependency edge while leaving each layer's own tests green.
|
|
96
|
+
*/
|
|
97
|
+
export function createLineageCollector(input) {
|
|
98
|
+
return new ProvenanceCollector({
|
|
99
|
+
stepId: input.stepId,
|
|
100
|
+
runId: input.runId,
|
|
101
|
+
dependsOn: [...(input.dependsOn ?? [])],
|
|
102
|
+
});
|
|
103
|
+
}
|
|
85
104
|
// ── The child workflow body ───────────────────────────────────────────
|
|
86
105
|
/**
|
|
87
106
|
* Register the sandbox-step child workflow with DBOS. Returns the
|
|
@@ -132,15 +151,7 @@ async function runSandboxStepBody(input, deps) {
|
|
|
132
151
|
// `session.provenance.agentId`, and billing tags ride the same provenance.
|
|
133
152
|
const session = forSubAgent(input.runSession, input.agentId);
|
|
134
153
|
const writePrefix = deps.resolveWritePrefix(input);
|
|
135
|
-
|
|
136
|
-
// provenance frame into it; post-step registration reads the resulting
|
|
137
|
-
// input/script edges. Reconstructed deterministically on replay — the
|
|
138
|
-
// frames it consumes are cached recv outputs and the command strings are
|
|
139
|
-
// cached LLM tool-use (see the harness-thread-store and harness-durable-runtime specs).
|
|
140
|
-
const lineageCollector = new ProvenanceCollector({
|
|
141
|
-
stepId: input.stepId,
|
|
142
|
-
runId: input.runId,
|
|
143
|
-
});
|
|
154
|
+
const lineageCollector = createLineageCollector(input);
|
|
144
155
|
// (2a) sandbox.mint — checkpoint the machine's identity (name + HMAC secret)
|
|
145
156
|
// BEFORE it is spawned (see the harness-sandbox-exec spec). Durable-before-create is what makes (2b)
|
|
146
157
|
// idempotent: a crash between spawn and the spawn checkpoint re-runs (2b)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@inflexa-ai/harness",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.2",
|
|
4
4
|
"description": "Host-agnostic bioinformatics agent harness \u2014 hand-rolled agent loop, DBOS-durable workflows, sandboxed compute.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
"prepare": "tsc -p tsconfig.json",
|
|
41
41
|
"prepublishOnly": "bun run build && bun run smoke",
|
|
42
42
|
"smoke": "node scripts/smoke.mjs",
|
|
43
|
+
"test:full": "bash scripts/test-full.sh",
|
|
43
44
|
"typecheck": "tsc --noEmit"
|
|
44
45
|
},
|
|
45
46
|
"dependencies": {
|