harness-dispatch 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +560 -1
- package/README.md +74 -43
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js.map +1 -1
- package/dist/bin.d.ts.map +1 -1
- package/dist/bin.js +166 -29
- package/dist/bin.js.map +1 -1
- package/dist/config/coercions.d.ts +5 -0
- package/dist/config/coercions.d.ts.map +1 -1
- package/dist/config/coercions.js +8 -0
- package/dist/config/coercions.js.map +1 -1
- package/dist/config/protocol.js.map +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js.map +1 -1
- package/dist/config.d.ts +11 -3
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +111 -148
- package/dist/config.js.map +1 -1
- package/dist/configure-yaml.d.ts.map +1 -1
- package/dist/configure-yaml.js +57 -13
- package/dist/configure-yaml.js.map +1 -1
- package/dist/dispatch-log.d.ts.map +1 -1
- package/dist/dispatch-log.js +3 -1
- package/dist/dispatch-log.js.map +1 -1
- package/dist/dispatchers/generic-cli.d.ts.map +1 -1
- package/dist/dispatchers/generic-cli.js +189 -97
- package/dist/dispatchers/generic-cli.js.map +1 -1
- package/dist/dispatchers/openai-compatible.d.ts +0 -5
- package/dist/dispatchers/openai-compatible.d.ts.map +1 -1
- package/dist/dispatchers/openai-compatible.js +129 -82
- package/dist/dispatchers/openai-compatible.js.map +1 -1
- package/dist/dispatchers/shared/harness-login.d.ts.map +1 -1
- package/dist/dispatchers/shared/harness-login.js +7 -2
- package/dist/dispatchers/shared/harness-login.js.map +1 -1
- package/dist/dispatchers/shared/stream-subprocess.d.ts.map +1 -1
- package/dist/dispatchers/shared/stream-subprocess.js +7 -60
- package/dist/dispatchers/shared/stream-subprocess.js.map +1 -1
- package/dist/file-lock.d.ts.map +1 -1
- package/dist/file-lock.js +31 -19
- package/dist/file-lock.js.map +1 -1
- package/dist/harness-presets.d.ts +51 -0
- package/dist/harness-presets.d.ts.map +1 -0
- package/dist/harness-presets.js +154 -0
- package/dist/harness-presets.js.map +1 -0
- package/dist/http/parse.d.ts +8 -1
- package/dist/http/parse.d.ts.map +1 -1
- package/dist/http/parse.js +10 -3
- package/dist/http/parse.js.map +1 -1
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +9 -2
- package/dist/http/server.js.map +1 -1
- package/dist/job-runner.js +6 -0
- package/dist/job-runner.js.map +1 -1
- package/dist/jobs/lifecycle.d.ts +100 -0
- package/dist/jobs/lifecycle.d.ts.map +1 -0
- package/dist/jobs/lifecycle.js +263 -0
- package/dist/jobs/lifecycle.js.map +1 -0
- package/dist/jobs/read.d.ts +17 -0
- package/dist/jobs/read.d.ts.map +1 -0
- package/dist/jobs/read.js +103 -0
- package/dist/jobs/read.js.map +1 -0
- package/dist/jobs/run.d.ts +45 -0
- package/dist/jobs/run.d.ts.map +1 -0
- package/dist/jobs/run.js +351 -0
- package/dist/jobs/run.js.map +1 -0
- package/dist/jobs/start.d.ts +9 -0
- package/dist/jobs/start.d.ts.map +1 -0
- package/dist/jobs/start.js +126 -0
- package/dist/jobs/start.js.map +1 -0
- package/dist/jobs/store.d.ts.map +1 -1
- package/dist/jobs/store.js +13 -3
- package/dist/jobs/store.js.map +1 -1
- package/dist/jobs/supervisor.d.ts +151 -0
- package/dist/jobs/supervisor.d.ts.map +1 -0
- package/dist/jobs/supervisor.js +679 -0
- package/dist/jobs/supervisor.js.map +1 -0
- package/dist/jobs.d.ts +15 -233
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +14 -1409
- package/dist/jobs.js.map +1 -1
- package/dist/leaderboard.d.ts.map +1 -1
- package/dist/leaderboard.js +5 -1
- package/dist/leaderboard.js.map +1 -1
- package/dist/mcp/config-hot-reload.d.ts.map +1 -1
- package/dist/mcp/config-hot-reload.js +4 -1
- package/dist/mcp/config-hot-reload.js.map +1 -1
- package/dist/mcp/dispatcher-factory.js +1 -1
- package/dist/mcp/dispatcher-factory.js.map +1 -1
- package/dist/mcp/resources.d.ts.map +1 -1
- package/dist/mcp/resources.js +3 -2
- package/dist/mcp/resources.js.map +1 -1
- package/dist/mcp/tools.d.ts +9 -2
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +45 -6
- package/dist/mcp/tools.js.map +1 -1
- package/dist/observability/spans.d.ts.map +1 -1
- package/dist/observability/spans.js +10 -2
- package/dist/observability/spans.js.map +1 -1
- package/dist/redaction.d.ts +90 -0
- package/dist/redaction.d.ts.map +1 -0
- package/dist/redaction.js +221 -0
- package/dist/redaction.js.map +1 -0
- package/dist/router.d.ts +12 -23
- package/dist/router.d.ts.map +1 -1
- package/dist/router.js +83 -224
- package/dist/router.js.map +1 -1
- package/dist/state-dir.d.ts.map +1 -1
- package/dist/state-dir.js +15 -1
- package/dist/state-dir.js.map +1 -1
- package/dist/status.d.ts +14 -3
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +36 -3
- package/dist/status.js.map +1 -1
- package/dist/workspace-lock.d.ts +20 -0
- package/dist/workspace-lock.d.ts.map +1 -1
- package/dist/workspace-lock.js +63 -0
- package/dist/workspace-lock.js.map +1 -1
- package/dist/workspace-resolve.d.ts +0 -9
- package/dist/workspace-resolve.d.ts.map +1 -1
- package/dist/workspace-resolve.js +38 -183
- package/dist/workspace-resolve.js.map +1 -1
- package/dist/workspaces.d.ts.map +1 -1
- package/dist/workspaces.js +63 -139
- package/dist/workspaces.js.map +1 -1
- package/package.json +5 -4
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The lifecycle verbs: stop a run, resolve its workspace, run it again.
|
|
3
|
+
*
|
|
4
|
+
* These sit above start/read because a retry starts a new job and a cancel
|
|
5
|
+
* reads one, so the dependency runs one way — lifecycle imports start, never
|
|
6
|
+
* the reverse.
|
|
7
|
+
*/
|
|
8
|
+
import type { JobDeps, JobStatus } from "./types.js";
|
|
9
|
+
/** What a cancel request actually did — the caller needs to tell these apart. */
|
|
10
|
+
export interface CancelOutcome {
|
|
11
|
+
jobId: string;
|
|
12
|
+
/**
|
|
13
|
+
* `cancelled` — it was queued and never started; stopped outright.
|
|
14
|
+
* `cancelling` — it is running; teardown requested and lands shortly.
|
|
15
|
+
* `already_finished` — it had already reached a terminal state; nothing done.
|
|
16
|
+
*/
|
|
17
|
+
outcome: "cancelled" | "cancelling" | "already_finished";
|
|
18
|
+
status: JobStatus["status"];
|
|
19
|
+
message: string;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Ask a job to stop.
|
|
23
|
+
*
|
|
24
|
+
* Cancellation cannot be a signal here: jobs run inside POOLED supervisors,
|
|
25
|
+
* and the only pid recorded against a job belongs to a process that is also
|
|
26
|
+
* running other jobs, so signalling it would cancel work nobody asked to
|
|
27
|
+
* cancel. Instead this writes a marker the run itself honours — it drops out
|
|
28
|
+
* of its event stream, which triggers the dispatcher's teardown (killTree on
|
|
29
|
+
* the agent CLI and its children) and releases the workspace lock through the
|
|
30
|
+
* same path a normal finish uses.
|
|
31
|
+
*
|
|
32
|
+
* Two consequences worth stating plainly, because a caller who assumes
|
|
33
|
+
* otherwise will be surprised:
|
|
34
|
+
*
|
|
35
|
+
* 1. It is not instantaneous. A running job stops within about a second;
|
|
36
|
+
* `cancelling` means requested, not done. Poll job_status to see it land.
|
|
37
|
+
* 2. Work already done is NOT undone. A cancelled agent may have already
|
|
38
|
+
* edited files in the workspace, and those edits stay. Cancelling stops
|
|
39
|
+
* further work; it is not a rollback.
|
|
40
|
+
*
|
|
41
|
+
* A cancelled run is deliberately not recorded as a failure: the route's
|
|
42
|
+
* circuit breaker and failure count never see it, because the caller changing
|
|
43
|
+
* their mind says nothing about whether the route works.
|
|
44
|
+
*/
|
|
45
|
+
export declare function cancelJob(jobId: string, reason?: string): Promise<CancelOutcome>;
|
|
46
|
+
/**
|
|
47
|
+
* Inspect or resolve the isolated workspace a finished job left behind.
|
|
48
|
+
*
|
|
49
|
+
* Looks the job up the same way job_status does, then hands off to
|
|
50
|
+
* workspace-resolve.ts. Kept here so the caller only ever needs a jobId —
|
|
51
|
+
* where the workspace lives, and which policy produced it, are details
|
|
52
|
+
* recorded in the job's own result.
|
|
53
|
+
*/
|
|
54
|
+
export declare function resolveJobWorkspace(jobId: string, action: "diff" | "apply" | "discard", opts?: {
|
|
55
|
+
force?: boolean;
|
|
56
|
+
}): Promise<unknown>;
|
|
57
|
+
export interface RetryOutcome {
|
|
58
|
+
jobId: string;
|
|
59
|
+
retryOf: string;
|
|
60
|
+
service?: string;
|
|
61
|
+
reusedFrom: {
|
|
62
|
+
prompt: boolean;
|
|
63
|
+
files: number;
|
|
64
|
+
workingDir: string;
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* The original's model, when the retry's route does not declare it.
|
|
68
|
+
*
|
|
69
|
+
* `retryJob` has always SET this and the MCP tool description has always
|
|
70
|
+
* documented it, but the interface never listed it — the object is built
|
|
71
|
+
* with a conditional spread, which TypeScript does not excess-property
|
|
72
|
+
* check. So a typed caller could not see a field that ships, and the tests
|
|
73
|
+
* asserting it only compiled because the tests were not typechecked.
|
|
74
|
+
*/
|
|
75
|
+
droppedModel?: string;
|
|
76
|
+
message: string;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Run a finished job's task again.
|
|
80
|
+
*
|
|
81
|
+
* The last verb missing from the job lifecycle: you could start work, watch
|
|
82
|
+
* it, stop it, and resolve its workspace — but if it failed, reproducing it
|
|
83
|
+
* meant reconstructing the prompt, the file list, the working directory and
|
|
84
|
+
* the hints by hand, from a job record that already holds all four. The
|
|
85
|
+
* machinery to execute a job bundle existed (executeJobDir) and simply was
|
|
86
|
+
* not reachable from outside.
|
|
87
|
+
*
|
|
88
|
+
* The prompt is taken from prompt.md, which is the FROZEN prompt — including
|
|
89
|
+
* any context preamble the original dispatch rendered in. A retry therefore
|
|
90
|
+
* reproduces what the delegate actually saw, not what the caller typed.
|
|
91
|
+
*
|
|
92
|
+
* `service` retargets the attempt, which is the common case rather than an
|
|
93
|
+
* afterthought: the reason a run failed is often the route, not the task
|
|
94
|
+
* ("codex hit its usage limit — try claude"). Omit it to reuse the original
|
|
95
|
+
* route, or to let the router pick again if the original had none.
|
|
96
|
+
*/
|
|
97
|
+
export declare function retryJob(jobId: string, deps: JobDeps, opts?: {
|
|
98
|
+
service?: string;
|
|
99
|
+
}): Promise<RetryOutcome>;
|
|
100
|
+
//# sourceMappingURL=lifecycle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../src/jobs/lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAqBH,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACrD,iFAAiF;AACjF,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,WAAW,GAAG,YAAY,GAAG,kBAAkB,CAAC;IACzD,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAqFtF;AAED;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,EACpC,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC7B,OAAO,CAAC,OAAO,CAAC,CAsBlB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IACnE;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,OAAO,EACb,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GAC9B,OAAO,CAAC,YAAY,CAAC,CAoFvB"}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The lifecycle verbs: stop a run, resolve its workspace, run it again.
|
|
3
|
+
*
|
|
4
|
+
* These sit above start/read because a retry starts a new job and a cancel
|
|
5
|
+
* reads one, so the dependency runs one way — lifecycle imports start, never
|
|
6
|
+
* the reverse.
|
|
7
|
+
*/
|
|
8
|
+
import { getAsyncJob } from "./read.js";
|
|
9
|
+
import { startAsyncJobTracked } from "./start.js";
|
|
10
|
+
import { readFile } from "node:fs/promises";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { declaresModel } from "../router.js";
|
|
13
|
+
import { applyWorkspace, discardWorkspace, isResolvable, workspaceDiff, } from "../workspace-resolve.js";
|
|
14
|
+
import { requestCancel, jobsRoot, readJson, timestamp, updateStatus, } from "./store.js";
|
|
15
|
+
/**
|
|
16
|
+
* Ask a job to stop.
|
|
17
|
+
*
|
|
18
|
+
* Cancellation cannot be a signal here: jobs run inside POOLED supervisors,
|
|
19
|
+
* and the only pid recorded against a job belongs to a process that is also
|
|
20
|
+
* running other jobs, so signalling it would cancel work nobody asked to
|
|
21
|
+
* cancel. Instead this writes a marker the run itself honours — it drops out
|
|
22
|
+
* of its event stream, which triggers the dispatcher's teardown (killTree on
|
|
23
|
+
* the agent CLI and its children) and releases the workspace lock through the
|
|
24
|
+
* same path a normal finish uses.
|
|
25
|
+
*
|
|
26
|
+
* Two consequences worth stating plainly, because a caller who assumes
|
|
27
|
+
* otherwise will be surprised:
|
|
28
|
+
*
|
|
29
|
+
* 1. It is not instantaneous. A running job stops within about a second;
|
|
30
|
+
* `cancelling` means requested, not done. Poll job_status to see it land.
|
|
31
|
+
* 2. Work already done is NOT undone. A cancelled agent may have already
|
|
32
|
+
* edited files in the workspace, and those edits stay. Cancelling stops
|
|
33
|
+
* further work; it is not a rollback.
|
|
34
|
+
*
|
|
35
|
+
* A cancelled run is deliberately not recorded as a failure: the route's
|
|
36
|
+
* circuit breaker and failure count never see it, because the caller changing
|
|
37
|
+
* their mind says nothing about whether the route works.
|
|
38
|
+
*/
|
|
39
|
+
export async function cancelJob(jobId, reason) {
|
|
40
|
+
const job = await getAsyncJob(jobId); // throws the friendly "No such job" for a stranger
|
|
41
|
+
const current = job.status.status;
|
|
42
|
+
// There are TWO kinds of orphaned job and they need opposite answers.
|
|
43
|
+
//
|
|
44
|
+
// WRITTEN — `drainSlotQueue` marks a slot-queued job orphaned on disk when
|
|
45
|
+
// the server exits before it ever starts. That job is genuinely
|
|
46
|
+
// terminal: its own error text says it is not resumed automatically and
|
|
47
|
+
// to use retry_job. Cancelling it is a no-op that would leave a marker
|
|
48
|
+
// nothing reads.
|
|
49
|
+
// DERIVED — `withOrphanCheck` reports a job orphaned when its heartbeat
|
|
50
|
+
// goes stale, while the FILE still says `queued` or `running`. That job
|
|
51
|
+
// is not inert: once the dead owner's claim ages out, claimNextJob will
|
|
52
|
+
// pick it up and run it. Answering "had already finished; nothing to
|
|
53
|
+
// cancel" was wrong about work that could still start, and left the
|
|
54
|
+
// caller no way to stop it.
|
|
55
|
+
//
|
|
56
|
+
// So the raw status decides, not the derived one. `getAsyncJob` has already
|
|
57
|
+
// applied the orphan check, which is why this re-reads the file.
|
|
58
|
+
const rawStatus = await readJson(path.join(jobsRoot(), jobId, "status.json")).catch(() => undefined);
|
|
59
|
+
const terminalOnDisk = rawStatus?.status === "orphaned";
|
|
60
|
+
if (current === "completed" ||
|
|
61
|
+
current === "failed" ||
|
|
62
|
+
current === "cancelled" ||
|
|
63
|
+
terminalOnDisk) {
|
|
64
|
+
return {
|
|
65
|
+
jobId,
|
|
66
|
+
outcome: "already_finished",
|
|
67
|
+
status: current,
|
|
68
|
+
message: `Job ${jobId} had already finished (${current}); nothing to cancel.`,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
const jobDir = path.join(jobsRoot(), jobId);
|
|
72
|
+
await requestCancel(jobDir, reason);
|
|
73
|
+
// A job still waiting for a slot has no runner to notice the marker, so
|
|
74
|
+
// stop it here. claimNextJob also refuses to claim a marked job, which
|
|
75
|
+
// closes the window where a supervisor picks it up between these two steps.
|
|
76
|
+
// `orphaned` joins `queued` here rather than falling through to "the runner
|
|
77
|
+
// will notice the marker": an orphaned job has no live runner BY
|
|
78
|
+
// DEFINITION, so nothing would ever act on the marker and the job would sit
|
|
79
|
+
// at "cancelling" forever. The marker written above still matters — it is
|
|
80
|
+
// what stops a supervisor reclaiming it — but the status has to be settled
|
|
81
|
+
// here, by the only process still involved.
|
|
82
|
+
if (current === "queued" || current === "orphaned") {
|
|
83
|
+
await updateStatus(jobDir, {
|
|
84
|
+
...job.status,
|
|
85
|
+
status: "cancelled",
|
|
86
|
+
updatedAt: timestamp(),
|
|
87
|
+
success: false,
|
|
88
|
+
error: reason !== undefined ? `Cancelled: ${reason}` : "Cancelled before it started.",
|
|
89
|
+
});
|
|
90
|
+
// Deliberately NOT draining the slot queue here. Freeing this job's slot
|
|
91
|
+
// makes room for a waiting one, but drainSlotQueue can SPAWN supervisor
|
|
92
|
+
// processes, and a cancel — the operation whose whole point is to stop
|
|
93
|
+
// work — must not start any. Every dispatch and every runner exit already
|
|
94
|
+
// drains, which is the same "resumes on the next event" contract the
|
|
95
|
+
// queue documents elsewhere.
|
|
96
|
+
return {
|
|
97
|
+
jobId,
|
|
98
|
+
outcome: "cancelled",
|
|
99
|
+
status: "cancelled",
|
|
100
|
+
message: current === "orphaned"
|
|
101
|
+
? `Job ${jobId} was orphaned — the process running it is gone — and is now ` +
|
|
102
|
+
`marked cancelled, so no supervisor can reclaim it. Any partial output it ` +
|
|
103
|
+
`wrote is still available from job_status.`
|
|
104
|
+
: `Job ${jobId} was waiting for a slot and has been cancelled; it never started.`,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
jobId,
|
|
109
|
+
outcome: "cancelling",
|
|
110
|
+
status: current,
|
|
111
|
+
message: `Cancellation requested for ${jobId}. The run stops within a second or so — poll ` +
|
|
112
|
+
`job_status to confirm. Any files the agent already changed are NOT reverted.`,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Inspect or resolve the isolated workspace a finished job left behind.
|
|
117
|
+
*
|
|
118
|
+
* Looks the job up the same way job_status does, then hands off to
|
|
119
|
+
* workspace-resolve.ts. Kept here so the caller only ever needs a jobId —
|
|
120
|
+
* where the workspace lives, and which policy produced it, are details
|
|
121
|
+
* recorded in the job's own result.
|
|
122
|
+
*/
|
|
123
|
+
export async function resolveJobWorkspace(jobId, action, opts = {}) {
|
|
124
|
+
const job = await getAsyncJob(jobId);
|
|
125
|
+
const run = job.result?.result?.workspace;
|
|
126
|
+
if (!isResolvable(run)) {
|
|
127
|
+
// Read through a separate binding: the type guard narrows `run` to never
|
|
128
|
+
// on this branch, which would make the diagnostic unable to say WHICH
|
|
129
|
+
// policy the caller actually got.
|
|
130
|
+
const raw = job.result?.result?.workspace;
|
|
131
|
+
const policy = raw?.policy ?? "shared";
|
|
132
|
+
throw new Error(`Job ${jobId} has no isolated workspace to ${action} (workspace policy: ${policy}). ` +
|
|
133
|
+
`Only 'copy' and 'git_worktree' dispatches produce one — a 'shared' or ` +
|
|
134
|
+
`'shared_locked' run edited ${raw?.originalWorkingDir ?? "the working directory"} ` +
|
|
135
|
+
`directly, so there is nothing separate to inspect, apply or throw away.`);
|
|
136
|
+
}
|
|
137
|
+
const jobDir = path.join(jobsRoot(), jobId);
|
|
138
|
+
if (action === "diff")
|
|
139
|
+
return workspaceDiff(jobId, jobDir, run);
|
|
140
|
+
if (action === "apply")
|
|
141
|
+
return applyWorkspace(jobId, jobDir, run, opts);
|
|
142
|
+
// force reaches discard too: it now refuses to destroy work the project
|
|
143
|
+
// does not have, and the caller needs the same override apply offers.
|
|
144
|
+
return discardWorkspace(jobId, run, opts);
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Run a finished job's task again.
|
|
148
|
+
*
|
|
149
|
+
* The last verb missing from the job lifecycle: you could start work, watch
|
|
150
|
+
* it, stop it, and resolve its workspace — but if it failed, reproducing it
|
|
151
|
+
* meant reconstructing the prompt, the file list, the working directory and
|
|
152
|
+
* the hints by hand, from a job record that already holds all four. The
|
|
153
|
+
* machinery to execute a job bundle existed (executeJobDir) and simply was
|
|
154
|
+
* not reachable from outside.
|
|
155
|
+
*
|
|
156
|
+
* The prompt is taken from prompt.md, which is the FROZEN prompt — including
|
|
157
|
+
* any context preamble the original dispatch rendered in. A retry therefore
|
|
158
|
+
* reproduces what the delegate actually saw, not what the caller typed.
|
|
159
|
+
*
|
|
160
|
+
* `service` retargets the attempt, which is the common case rather than an
|
|
161
|
+
* afterthought: the reason a run failed is often the route, not the task
|
|
162
|
+
* ("codex hit its usage limit — try claude"). Omit it to reuse the original
|
|
163
|
+
* route, or to let the router pick again if the original had none.
|
|
164
|
+
*/
|
|
165
|
+
export async function retryJob(jobId, deps, opts = {}) {
|
|
166
|
+
const prior = await getAsyncJob(jobId); // friendly "No such job" for a stranger
|
|
167
|
+
const state = prior.status.status;
|
|
168
|
+
if (state === "running" || state === "queued") {
|
|
169
|
+
throw new Error(`Job ${jobId} is still ${state}. Let it finish, or cancel it first with ` +
|
|
170
|
+
`cancel_job — retrying a live run would leave two attempts racing on the ` +
|
|
171
|
+
`same working directory.`);
|
|
172
|
+
}
|
|
173
|
+
// A DERIVED orphan produces exactly the outcome the message above forbids,
|
|
174
|
+
// and this guard could not see it.
|
|
175
|
+
//
|
|
176
|
+
// `getAsyncJob` reports orphaned when the heartbeat is stale, but the status
|
|
177
|
+
// FILE still says `queued`, and `claimNextJob` filters on the raw file — so
|
|
178
|
+
// a supervisor can still claim the original while the retry runs. The guard
|
|
179
|
+
// reads the derived status, which is neither `running` nor `queued`, so it
|
|
180
|
+
// let the retry through. `cancelJob` was given this reasoning when it learnt
|
|
181
|
+
// to tell the two kinds of orphan apart; retry was not, so the fix stopped
|
|
182
|
+
// one square short.
|
|
183
|
+
//
|
|
184
|
+
// Marking the original cancelled is what closes it: claimNextJob refuses a
|
|
185
|
+
// marked job, so the retry becomes the only attempt. A job orphaned while
|
|
186
|
+
// RUNNING has no live runner either, so the marker is equally correct there
|
|
187
|
+
// and simply has nothing left to interrupt.
|
|
188
|
+
if (state === "orphaned") {
|
|
189
|
+
const rawStatus = await readJson(path.join(jobsRoot(), jobId, "status.json")).catch(() => undefined);
|
|
190
|
+
if (rawStatus?.status === "queued" || rawStatus?.status === "running") {
|
|
191
|
+
await cancelJob(jobId, `superseded by a retry`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const manifest = prior.manifest;
|
|
195
|
+
const prompt = await readFile(manifest.promptPath, "utf8");
|
|
196
|
+
if (opts.service !== undefined && !(opts.service in deps.holder.state.config.services)) {
|
|
197
|
+
throw new Error(`Unknown service: ${opts.service}. Valid route ids: ` +
|
|
198
|
+
`${Object.keys(deps.holder.state.config.services).join(", ")}.`);
|
|
199
|
+
}
|
|
200
|
+
const service = opts.service ?? manifest.service;
|
|
201
|
+
const { hints, droppedModel } = hintsForRetry(manifest.hints, prior.status.route ?? manifest.service, service, deps.holder.state.config.services);
|
|
202
|
+
const { status } = await startAsyncJobTracked(deps, {
|
|
203
|
+
prompt,
|
|
204
|
+
files: manifest.files.map((f) => f.originalPath),
|
|
205
|
+
workingDir: manifest.workingDir,
|
|
206
|
+
retryOf: jobId,
|
|
207
|
+
...(hints !== undefined ? { hints } : {}),
|
|
208
|
+
...(manifest.workspacePolicy !== undefined
|
|
209
|
+
? { workspacePolicy: manifest.workspacePolicy }
|
|
210
|
+
: {}),
|
|
211
|
+
...(service !== undefined ? { service } : {}),
|
|
212
|
+
});
|
|
213
|
+
return {
|
|
214
|
+
jobId: status.jobId,
|
|
215
|
+
retryOf: jobId,
|
|
216
|
+
...(service !== undefined ? { service } : {}),
|
|
217
|
+
...(droppedModel !== undefined ? { droppedModel } : {}),
|
|
218
|
+
reusedFrom: {
|
|
219
|
+
prompt: true,
|
|
220
|
+
files: manifest.files.length,
|
|
221
|
+
workingDir: manifest.workingDir,
|
|
222
|
+
},
|
|
223
|
+
message: `Started ${status.jobId} from ${jobId}'s prompt, files and working directory` +
|
|
224
|
+
`${opts.service !== undefined ? `, retargeted to ${opts.service}` : ""}. ` +
|
|
225
|
+
`${droppedModel !== undefined
|
|
226
|
+
? `Left behind the original's model "${droppedModel}", which ${service} does ` +
|
|
227
|
+
`not declare — a model name belongs to the route it was chosen for. Pass ` +
|
|
228
|
+
`hints.model on a fresh dispatch if you want a specific model here. `
|
|
229
|
+
: ""}` +
|
|
230
|
+
`Check it with job_status; the original job is untouched.`,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Carry the original's hints into a retry — except a model that belonged to
|
|
235
|
+
* the route being left behind.
|
|
236
|
+
*
|
|
237
|
+
* Retrying somewhere else is the documented reason this tool exists ("the task
|
|
238
|
+
* was fine and the route was not"), and reusing the model verbatim defeated
|
|
239
|
+
* exactly that case: model names are route-scoped, so the retry failed for the
|
|
240
|
+
* same reason as the original. Observed end to end — a Cursor run that died on
|
|
241
|
+
* `Cannot use this model` was retried onto Claude and died on
|
|
242
|
+
* `unrecognized_model`, having never reached the task.
|
|
243
|
+
*
|
|
244
|
+
* Narrow on purpose. The model is kept when the retry stays on the original's
|
|
245
|
+
* route (that is a plain "try again"), and when the new route declares it
|
|
246
|
+
* anyway. Only a model the destination does not know is dropped, and the
|
|
247
|
+
* caller is told — a silently changed model is the failure this project keeps
|
|
248
|
+
* finding, so it is reported in the response rather than inferred from a
|
|
249
|
+
* different result.
|
|
250
|
+
*/
|
|
251
|
+
function hintsForRetry(hints, priorRoute, service, services) {
|
|
252
|
+
if (hints === undefined)
|
|
253
|
+
return { hints };
|
|
254
|
+
const model = hints.model;
|
|
255
|
+
if (model === undefined || service === undefined || service === priorRoute)
|
|
256
|
+
return { hints };
|
|
257
|
+
const target = services[service];
|
|
258
|
+
if (target === undefined || declaresModel(target, model))
|
|
259
|
+
return { hints };
|
|
260
|
+
const { model: _dropped, ...rest } = hints;
|
|
261
|
+
return { hints: rest, droppedModel: model };
|
|
262
|
+
}
|
|
263
|
+
//# sourceMappingURL=lifecycle.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lifecycle.js","sourceRoot":"","sources":["../../src/jobs/lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,aAAa,GACd,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,aAAa,EACb,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,YAAY,GACb,MAAM,YAAY,CAAC;AAepB;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,MAAe;IAC5D,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,mDAAmD;IACzF,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;IAElC,sEAAsE;IACtE,EAAE;IACF,6EAA6E;IAC7E,oEAAoE;IACpE,4EAA4E;IAC5E,2EAA2E;IAC3E,qBAAqB;IACrB,0EAA0E;IAC1E,4EAA4E;IAC5E,4EAA4E;IAC5E,yEAAyE;IACzE,wEAAwE;IACxE,gCAAgC;IAChC,EAAE;IACF,4EAA4E;IAC5E,iEAAiE;IACjE,MAAM,SAAS,GAAG,MAAM,QAAQ,CAC9B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,CAC5C,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzB,MAAM,cAAc,GAAG,SAAS,EAAE,MAAM,KAAK,UAAU,CAAC;IACxD,IACE,OAAO,KAAK,WAAW;QACvB,OAAO,KAAK,QAAQ;QACpB,OAAO,KAAK,WAAW;QACvB,cAAc,EACd,CAAC;QACD,OAAO;YACL,KAAK;YACL,OAAO,EAAE,kBAAkB;YAC3B,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,OAAO,KAAK,0BAA0B,OAAO,uBAAuB;SAC9E,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;IAC5C,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEpC,wEAAwE;IACxE,uEAAuE;IACvE,4EAA4E;IAC5E,4EAA4E;IAC5E,iEAAiE;IACjE,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,4CAA4C;IAC5C,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,MAAM,EAAE;YACzB,GAAG,GAAG,CAAC,MAAM;YACb,MAAM,EAAE,WAAW;YACnB,SAAS,EAAE,SAAS,EAAE;YACtB,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,MAAM,EAAE,CAAC,CAAC,CAAC,8BAA8B;SACtF,CAAC,CAAC;QACH,yEAAyE;QACzE,wEAAwE;QACxE,uEAAuE;QACvE,0EAA0E;QAC1E,qEAAqE;QACrE,6BAA6B;QAC7B,OAAO;YACL,KAAK;YACL,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,WAAW;YACnB,OAAO,EACL,OAAO,KAAK,UAAU;gBACpB,CAAC,CAAC,OAAO,KAAK,8DAA8D;oBAC1E,2EAA2E;oBAC3E,2CAA2C;gBAC7C,CAAC,CAAC,OAAO,KAAK,mEAAmE;SACtF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK;QACL,OAAO,EAAE,YAAY;QACrB,MAAM,EAAE,OAAO;QACf,OAAO,EACL,8BAA8B,KAAK,+CAA+C;YAClF,8EAA8E;KACjF,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAa,EACb,MAAoC,EACpC,OAA4B,EAAE;IAE9B,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC;IAC1C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,yEAAyE;QACzE,sEAAsE;QACtE,kCAAkC;QAClC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC;QAC1C,MAAM,MAAM,GAAG,GAAG,EAAE,MAAM,IAAI,QAAQ,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,OAAO,KAAK,iCAAiC,MAAM,uBAAuB,MAAM,KAAK;YACnF,wEAAwE;YACxE,8BAA8B,GAAG,EAAE,kBAAkB,IAAI,uBAAuB,GAAG;YACnF,yEAAyE,CAC5E,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;IAC5C,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IAChE,IAAI,MAAM,KAAK,OAAO;QAAE,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACxE,wEAAwE;IACxE,sEAAsE;IACtE,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;AAC5C,CAAC;AAoBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,IAAa,EACb,OAA6B,EAAE;IAE/B,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,wCAAwC;IAChF,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IAClC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,OAAO,KAAK,aAAa,KAAK,2CAA2C;YACvE,0EAA0E;YAC1E,yBAAyB,CAC5B,CAAC;IACJ,CAAC;IACD,2EAA2E;IAC3E,mCAAmC;IACnC,EAAE;IACF,6EAA6E;IAC7E,4EAA4E;IAC5E,4EAA4E;IAC5E,2EAA2E;IAC3E,6EAA6E;IAC7E,2EAA2E;IAC3E,oBAAoB;IACpB,EAAE;IACF,2EAA2E;IAC3E,0EAA0E;IAC1E,4EAA4E;IAC5E,4CAA4C;IAC5C,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAC9B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,CAC5C,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzB,IAAI,SAAS,EAAE,MAAM,KAAK,QAAQ,IAAI,SAAS,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YACtE,MAAM,SAAS,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAChC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC3D,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvF,MAAM,IAAI,KAAK,CACb,oBAAoB,IAAI,CAAC,OAAO,qBAAqB;YACnD,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAClE,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;IACjD,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,aAAa,CAC3C,QAAQ,CAAC,KAAK,EACd,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC,OAAO,EACtC,OAAO,EACP,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAClC,CAAC;IAEF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE;QAClD,MAAM;QACN,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;QAChD,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,OAAO,EAAE,KAAK;QACd,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS;YACxC,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE;YAC/C,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC,CAAC;IAEH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,KAAK;QACd,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,UAAU,EAAE;YACV,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;YAC5B,UAAU,EAAE,QAAQ,CAAC,UAAU;SAChC;QACD,OAAO,EACL,WAAW,MAAM,CAAC,KAAK,SAAS,KAAK,wCAAwC;YAC7E,GAAG,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI;YAC1E,GACE,YAAY,KAAK,SAAS;gBACxB,CAAC,CAAC,qCAAqC,YAAY,YAAY,OAAO,QAAQ;oBAC5E,0EAA0E;oBAC1E,qEAAqE;gBACvE,CAAC,CAAC,EACN,EAAE;YACF,0DAA0D;KAC7D,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAS,aAAa,CACpB,KAA6B,EAC7B,UAA8B,EAC9B,OAA2B,EAC3B,QAAuC;IAEvC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,UAAU;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7F,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IACjC,IAAI,MAAM,KAAK,SAAS,IAAI,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC3E,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;IAC3C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a job back: one with its partial output, or the recent list.
|
|
3
|
+
*
|
|
4
|
+
* Separate from start.ts because the supervisor reads jobs while deciding
|
|
5
|
+
* what to run, and start.ts needs the supervisor — keeping the reads beside
|
|
6
|
+
* the start verbs closed that loop.
|
|
7
|
+
*/
|
|
8
|
+
import type { JobManifest, JobResultPayload, JobStatus } from "./types.js";
|
|
9
|
+
export declare function getAsyncJob(jobId: string): Promise<{
|
|
10
|
+
manifest: JobManifest;
|
|
11
|
+
status: JobStatus;
|
|
12
|
+
result?: JobResultPayload;
|
|
13
|
+
/** Tail of live stdout/stderr while the job is still running. */
|
|
14
|
+
partialOutput?: string;
|
|
15
|
+
}>;
|
|
16
|
+
export declare function listAsyncJobs(): Promise<JobStatus[]>;
|
|
17
|
+
//# sourceMappingURL=read.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read.d.ts","sourceRoot":"","sources":["../../src/jobs/read.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAaH,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAG3E,wBAAsB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACxD,QAAQ,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,SAAS,CAAC;IAClB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,iEAAiE;IACjE,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC,CAsED;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,CAgB1D"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a job back: one with its partial output, or the recent list.
|
|
3
|
+
*
|
|
4
|
+
* Separate from start.ts because the supervisor reads jobs while deciding
|
|
5
|
+
* what to run, and start.ts needs the supervisor — keeping the reads beside
|
|
6
|
+
* the start verbs closed that loop.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync } from "node:fs";
|
|
9
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { assertValidJobId, jobsRoot, pollInstructions, readJson, SUGGESTED_POLL_SECONDS, withOrphanCheck, } from "./store.js";
|
|
12
|
+
const MAX_PARTIAL_OUTPUT_CHARS = 4000;
|
|
13
|
+
export async function getAsyncJob(jobId) {
|
|
14
|
+
assertValidJobId(jobId);
|
|
15
|
+
const jobDir = path.join(jobsRoot(), jobId);
|
|
16
|
+
// A well-formed id for a job that is gone is the ORDINARY case, not an
|
|
17
|
+
// internal error: retention prunes finished jobs, so any caller holding an
|
|
18
|
+
// id long enough will hit this. It used to surface as a raw Node ENOENT
|
|
19
|
+
// quoting an absolute path inside the jobs directory, which tells the caller
|
|
20
|
+
// nothing actionable and leaks the layout.
|
|
21
|
+
const noSuchJob = () => new Error(`No such job: ${jobId}. It may have been pruned by the retention window, ` +
|
|
22
|
+
`or it was never started on this machine.`);
|
|
23
|
+
if (!existsSync(path.join(jobDir, "manifest.json")))
|
|
24
|
+
throw noSuchJob();
|
|
25
|
+
let manifest;
|
|
26
|
+
let status;
|
|
27
|
+
let result;
|
|
28
|
+
try {
|
|
29
|
+
manifest = await readJson(path.join(jobDir, "manifest.json"));
|
|
30
|
+
status = withOrphanCheck(await readJson(path.join(jobDir, "status.json")));
|
|
31
|
+
const resultPath = path.join(jobDir, "output", "result.json");
|
|
32
|
+
result = existsSync(resultPath) ? await readJson(resultPath) : undefined;
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
// existsSync-then-read is a TOCTOU window: retention pruning can delete
|
|
36
|
+
// the directory between the two calls, resurfacing the exact raw-ENOENT-
|
|
37
|
+
// with-an-absolute-path error this function's message exists to replace.
|
|
38
|
+
if (err?.code === "ENOENT")
|
|
39
|
+
throw noSuchJob();
|
|
40
|
+
throw err;
|
|
41
|
+
}
|
|
42
|
+
if (result !== undefined) {
|
|
43
|
+
return { manifest, status, result };
|
|
44
|
+
}
|
|
45
|
+
// Terminal: no poll guidance — polling will never resolve an orphaned job.
|
|
46
|
+
// But it still gets its partial output, which is the whole point of the
|
|
47
|
+
// record surviving the runner.
|
|
48
|
+
//
|
|
49
|
+
// This branch used to `return` here, ABOVE the partial-output read below, so
|
|
50
|
+
// an orphaned job handed back nothing at all while its progress sat in
|
|
51
|
+
// output/stdout.partial.log. PRODUCT.md's success criterion is explicit that
|
|
52
|
+
// a dispatch must never die returning nothing — "at worst it fails and hands
|
|
53
|
+
// back its latest progress… a wasted attempt with no trail is the defining
|
|
54
|
+
// failure" — and orphaning is exactly the case that criterion was written
|
|
55
|
+
// for: the supervisor died, so there is no result.json and the partial log
|
|
56
|
+
// is all that survived. An acceptance pass measured eight chunks of progress
|
|
57
|
+
// on disk and an empty response.
|
|
58
|
+
const terminalOrphan = status.status === "orphaned";
|
|
59
|
+
const out = {
|
|
60
|
+
manifest,
|
|
61
|
+
status: terminalOrphan
|
|
62
|
+
? status
|
|
63
|
+
: {
|
|
64
|
+
...status,
|
|
65
|
+
nextPollSeconds: SUGGESTED_POLL_SECONDS,
|
|
66
|
+
instructions: pollInstructions(jobId),
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
const partialPath = path.join(jobDir, "output", "stdout.partial.log");
|
|
70
|
+
if (existsSync(partialPath)) {
|
|
71
|
+
try {
|
|
72
|
+
const partial = await readFile(partialPath, "utf8");
|
|
73
|
+
out.partialOutput =
|
|
74
|
+
partial.length <= MAX_PARTIAL_OUTPUT_CHARS
|
|
75
|
+
? partial
|
|
76
|
+
: `… [${partial.length - MAX_PARTIAL_OUTPUT_CHARS} chars omitted] …` +
|
|
77
|
+
partial.slice(-MAX_PARTIAL_OUTPUT_CHARS);
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
// Best-effort; absence of partial output isn't an error.
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
export async function listAsyncJobs() {
|
|
86
|
+
const root = jobsRoot();
|
|
87
|
+
if (!existsSync(root))
|
|
88
|
+
return [];
|
|
89
|
+
const entries = await readdir(root, { withFileTypes: true });
|
|
90
|
+
const statuses = [];
|
|
91
|
+
for (const entry of entries) {
|
|
92
|
+
if (!entry.isDirectory())
|
|
93
|
+
continue;
|
|
94
|
+
try {
|
|
95
|
+
statuses.push(withOrphanCheck(await readJson(path.join(root, entry.name, "status.json"))));
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
// Ignore incomplete or manually edited job directories.
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return statuses.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=read.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read.js","sourceRoot":"","sources":["../../src/jobs/read.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACL,gBAAgB,EAChB,QAAQ,EACR,gBAAgB,EAChB,QAAQ,EACR,sBAAsB,EACtB,eAAe,GAChB,MAAM,YAAY,CAAC;AAEpB,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAEtC,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAa;IAO7C,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACxB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;IAC5C,uEAAuE;IACvE,2EAA2E;IAC3E,wEAAwE;IACxE,6EAA6E;IAC7E,2CAA2C;IAC3C,MAAM,SAAS,GAAG,GAAG,EAAE,CACrB,IAAI,KAAK,CACP,gBAAgB,KAAK,qDAAqD;QACxE,0CAA0C,CAC7C,CAAC;IACJ,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAAE,MAAM,SAAS,EAAE,CAAC;IACvE,IAAI,QAAqB,CAAC;IAC1B,IAAI,MAAiB,CAAC;IACtB,IAAI,MAAoC,CAAC;IACzC,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,QAAQ,CAAc,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC;QAC3E,MAAM,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAY,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;QACtF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC9D,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAmB,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7F,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,wEAAwE;QACxE,yEAAyE;QACzE,yEAAyE;QACzE,IAAK,GAA6B,EAAE,IAAI,KAAK,QAAQ;YAAE,MAAM,SAAS,EAAE,CAAC;QACzE,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IACtC,CAAC;IACD,2EAA2E;IAC3E,wEAAwE;IACxE,+BAA+B;IAC/B,EAAE;IACF,6EAA6E;IAC7E,uEAAuE;IACvE,6EAA6E;IAC7E,6EAA6E;IAC7E,2EAA2E;IAC3E,0EAA0E;IAC1E,2EAA2E;IAC3E,6EAA6E;IAC7E,iCAAiC;IACjC,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC;IACpD,MAAM,GAAG,GAAyE;QAChF,QAAQ;QACR,MAAM,EAAE,cAAc;YACpB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC;gBACE,GAAG,MAAM;gBACT,eAAe,EAAE,sBAAsB;gBACvC,YAAY,EAAE,gBAAgB,CAAC,KAAK,CAAC;aACtC;KACN,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IACtE,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACpD,GAAG,CAAC,aAAa;gBACf,OAAO,CAAC,MAAM,IAAI,wBAAwB;oBACxC,CAAC,CAAC,OAAO;oBACT,CAAC,CAAC,MAAM,OAAO,CAAC,MAAM,GAAG,wBAAwB,mBAAmB;wBAClE,OAAO,CAAC,KAAK,CAAC,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;QAC3D,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAgB,EAAE,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CACX,eAAe,CAAC,MAAM,QAAQ,CAAY,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,CACvF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACzE,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running one job: the work a runner process actually does.
|
|
3
|
+
*
|
|
4
|
+
* Split out of jobs.ts, which had grown to hold five concerns at 1,600 lines
|
|
5
|
+
* — this one, admission and the supervisor pool, the start/read verbs, and
|
|
6
|
+
* the lifecycle verbs. Nothing here was rewritten; the code moved.
|
|
7
|
+
*/
|
|
8
|
+
import type { JobDeps, JobManifest, StartJobInput } from "./types.js";
|
|
9
|
+
export declare function runJob(deps: JobDeps, jobDir: string, manifest: JobManifest, input: StartJobInput): Promise<void>;
|
|
10
|
+
/**
|
|
11
|
+
* Fallback dispatch timeout for jobs. Dispatchers hard-code a short default
|
|
12
|
+
* (10 min for CLI harnesses, 2 min for openai_compatible) meant to catch a
|
|
13
|
+
* genuinely hung process — waiting on stdin that'll never come, a stalled
|
|
14
|
+
* network call — not to cap a slow-but-healthy run. That default made sense
|
|
15
|
+
* as-is for `code`, which blocks an MCP call anyway, but `job` runs in the
|
|
16
|
+
* background and is polled, so nothing about it requires killing a process
|
|
17
|
+
* that's still making progress after 10 minutes. Below both an explicit
|
|
18
|
+
* `hints.timeoutMs` and the route's own configured `timeoutMs` in
|
|
19
|
+
* precedence, so this only fills the gap when nobody set either.
|
|
20
|
+
*
|
|
21
|
+
* Router.stream() treats this specific value as a budget for the WHOLE call
|
|
22
|
+
* (including router fallback retries), not a per-attempt allowance — without
|
|
23
|
+
* that, a job that falls back twice (the router's default maxFallbacks: 2)
|
|
24
|
+
* could burn up to 3x this value before failing conclusively.
|
|
25
|
+
*/
|
|
26
|
+
export declare const JOB_DEFAULT_TIMEOUT_MS: number;
|
|
27
|
+
/**
|
|
28
|
+
* Rebuild a job's input from its on-disk bundle and execute it. This is the
|
|
29
|
+
* detached runner's whole job; the manifest deliberately carries everything
|
|
30
|
+
* a run needs (prompt path, resolved workingDir, hints, service) precisely
|
|
31
|
+
* so execution can happen in a process that wasn't there when the job was
|
|
32
|
+
* created.
|
|
33
|
+
*/
|
|
34
|
+
export declare function executeJobDir(deps: JobDeps, jobDir: string): Promise<void>;
|
|
35
|
+
/** dist/job-runner.js next to this module (compiled), or via the package's dist/ when running from src. */
|
|
36
|
+
export declare function resolveRunnerPath(): string | undefined;
|
|
37
|
+
/**
|
|
38
|
+
* Watch a detached job's directory until it reaches a terminal state
|
|
39
|
+
* (result.json present, or a failed/orphaned status — the orphan check
|
|
40
|
+
* doubles as the exit path if the runner dies). Timer is unref'd: an
|
|
41
|
+
* exiting server abandons the watch, which is exactly the point of
|
|
42
|
+
* detached execution.
|
|
43
|
+
*/
|
|
44
|
+
export declare function watchUntilTerminal(jobDir: string): Promise<void>;
|
|
45
|
+
//# sourceMappingURL=run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/jobs/run.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAmBH,OAAO,KAAK,EAAE,OAAO,EAAE,WAAW,EAA+B,aAAa,EAAE,MAAM,YAAY,CAAC;AACnG,wBAAsB,MAAM,CAC1B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,WAAW,EACrB,KAAK,EAAE,aAAa,GACnB,OAAO,CAAC,IAAI,CAAC,CA4Of;AAMD;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AAGrD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAchF;AAED,2GAA2G;AAC3G,wBAAgB,iBAAiB,IAAI,MAAM,GAAG,SAAS,CAmBtD;AAID;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAiCtE"}
|