harness-dispatch 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1074 -2
- package/README.md +92 -36
- package/config.default.yaml +19 -3
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js.map +1 -1
- package/dist/billing.d.ts +24 -0
- package/dist/billing.d.ts.map +1 -1
- package/dist/billing.js +24 -0
- package/dist/billing.js.map +1 -1
- package/dist/bin.d.ts.map +1 -1
- package/dist/bin.js +299 -48
- package/dist/bin.js.map +1 -1
- package/dist/circuit-breaker.d.ts +9 -1
- package/dist/circuit-breaker.d.ts.map +1 -1
- package/dist/circuit-breaker.js +15 -1
- package/dist/circuit-breaker.js.map +1 -1
- package/dist/client-register.d.ts +11 -1
- package/dist/client-register.d.ts.map +1 -1
- package/dist/client-register.js +15 -6
- package/dist/client-register.js.map +1 -1
- package/dist/config/coercions.d.ts +5 -0
- package/dist/config/coercions.d.ts.map +1 -1
- package/dist/config/coercions.js +8 -0
- package/dist/config/coercions.js.map +1 -1
- package/dist/config/protocol.js.map +1 -1
- package/dist/config/validation.d.ts +12 -0
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js +133 -7
- package/dist/config/validation.js.map +1 -1
- package/dist/config.d.ts +15 -5
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +199 -152
- package/dist/config.js.map +1 -1
- package/dist/configure-yaml.d.ts +2 -0
- package/dist/configure-yaml.d.ts.map +1 -1
- package/dist/configure-yaml.js +114 -4
- package/dist/configure-yaml.js.map +1 -1
- package/dist/dispatch-log.d.ts.map +1 -1
- package/dist/dispatch-log.js +3 -1
- package/dist/dispatch-log.js.map +1 -1
- package/dist/dispatchers/generic-cli.d.ts.map +1 -1
- package/dist/dispatchers/generic-cli.js +281 -112
- package/dist/dispatchers/generic-cli.js.map +1 -1
- package/dist/dispatchers/openai-compatible.d.ts +0 -5
- package/dist/dispatchers/openai-compatible.d.ts.map +1 -1
- package/dist/dispatchers/openai-compatible.js +150 -83
- package/dist/dispatchers/openai-compatible.js.map +1 -1
- package/dist/dispatchers/shared/harness-login.d.ts +25 -0
- package/dist/dispatchers/shared/harness-login.d.ts.map +1 -0
- package/dist/dispatchers/shared/harness-login.js +73 -0
- package/dist/dispatchers/shared/harness-login.js.map +1 -0
- package/dist/dispatchers/shared/stream-subprocess.d.ts.map +1 -1
- package/dist/dispatchers/shared/stream-subprocess.js +71 -58
- package/dist/dispatchers/shared/stream-subprocess.js.map +1 -1
- package/dist/file-lock.d.ts +29 -1
- package/dist/file-lock.d.ts.map +1 -1
- package/dist/file-lock.js +53 -20
- package/dist/file-lock.js.map +1 -1
- package/dist/harness-presets.d.ts +51 -0
- package/dist/harness-presets.d.ts.map +1 -0
- package/dist/harness-presets.js +154 -0
- package/dist/harness-presets.js.map +1 -0
- package/dist/http/parse.d.ts +8 -1
- package/dist/http/parse.d.ts.map +1 -1
- package/dist/http/parse.js +10 -3
- package/dist/http/parse.js.map +1 -1
- package/dist/http/server.d.ts +2 -0
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +188 -16
- package/dist/http/server.js.map +1 -1
- package/dist/job-runner.js +6 -0
- package/dist/job-runner.js.map +1 -1
- package/dist/jobs/context.d.ts +0 -7
- package/dist/jobs/context.d.ts.map +1 -1
- package/dist/jobs/context.js +95 -7
- package/dist/jobs/context.js.map +1 -1
- package/dist/jobs/lifecycle.d.ts +100 -0
- package/dist/jobs/lifecycle.d.ts.map +1 -0
- package/dist/jobs/lifecycle.js +263 -0
- package/dist/jobs/lifecycle.js.map +1 -0
- package/dist/jobs/read.d.ts +17 -0
- package/dist/jobs/read.d.ts.map +1 -0
- package/dist/jobs/read.js +103 -0
- package/dist/jobs/read.js.map +1 -0
- package/dist/jobs/run.d.ts +45 -0
- package/dist/jobs/run.d.ts.map +1 -0
- package/dist/jobs/run.js +351 -0
- package/dist/jobs/run.js.map +1 -0
- package/dist/jobs/start.d.ts +9 -0
- package/dist/jobs/start.d.ts.map +1 -0
- package/dist/jobs/start.js +126 -0
- package/dist/jobs/start.js.map +1 -0
- package/dist/jobs/store.d.ts.map +1 -1
- package/dist/jobs/store.js +13 -3
- package/dist/jobs/store.js.map +1 -1
- package/dist/jobs/supervisor.d.ts +151 -0
- package/dist/jobs/supervisor.d.ts.map +1 -0
- package/dist/jobs/supervisor.js +679 -0
- package/dist/jobs/supervisor.js.map +1 -0
- package/dist/jobs/types.d.ts +8 -1
- package/dist/jobs/types.d.ts.map +1 -1
- package/dist/jobs.d.ts +15 -218
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +14 -1317
- package/dist/jobs.js.map +1 -1
- package/dist/leaderboard.d.ts.map +1 -1
- package/dist/leaderboard.js +44 -28
- package/dist/leaderboard.js.map +1 -1
- package/dist/mcp/config-hot-reload.d.ts +14 -3
- package/dist/mcp/config-hot-reload.d.ts.map +1 -1
- package/dist/mcp/config-hot-reload.js +27 -6
- package/dist/mcp/config-hot-reload.js.map +1 -1
- package/dist/mcp/dispatcher-factory.d.ts.map +1 -1
- package/dist/mcp/dispatcher-factory.js +15 -4
- package/dist/mcp/dispatcher-factory.js.map +1 -1
- package/dist/mcp/resources.d.ts.map +1 -1
- package/dist/mcp/resources.js +3 -2
- package/dist/mcp/resources.js.map +1 -1
- package/dist/mcp/tool-schemas.d.ts +1 -1
- package/dist/mcp/tool-schemas.d.ts.map +1 -1
- package/dist/mcp/tool-schemas.js +6 -3
- package/dist/mcp/tool-schemas.js.map +1 -1
- package/dist/mcp/tools.d.ts +20 -6
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +92 -14
- package/dist/mcp/tools.js.map +1 -1
- package/dist/mcp-clients.d.ts +9 -0
- package/dist/mcp-clients.d.ts.map +1 -1
- package/dist/mcp-clients.js +4 -0
- package/dist/mcp-clients.js.map +1 -1
- package/dist/observability/spans.d.ts.map +1 -1
- package/dist/observability/spans.js +10 -2
- package/dist/observability/spans.js.map +1 -1
- package/dist/quota.d.ts.map +1 -1
- package/dist/quota.js +16 -2
- package/dist/quota.js.map +1 -1
- package/dist/redaction.d.ts +90 -0
- package/dist/redaction.d.ts.map +1 -0
- package/dist/redaction.js +221 -0
- package/dist/redaction.js.map +1 -0
- package/dist/route-policy.d.ts +2 -1
- package/dist/route-policy.d.ts.map +1 -1
- package/dist/route-policy.js +54 -0
- package/dist/route-policy.js.map +1 -1
- package/dist/router.d.ts +12 -23
- package/dist/router.d.ts.map +1 -1
- package/dist/router.js +86 -223
- package/dist/router.js.map +1 -1
- package/dist/safety.d.ts.map +1 -1
- package/dist/safety.js +7 -1
- package/dist/safety.js.map +1 -1
- package/dist/state-dir.d.ts +10 -0
- package/dist/state-dir.d.ts.map +1 -1
- package/dist/state-dir.js +27 -1
- package/dist/state-dir.js.map +1 -1
- package/dist/status.d.ts +14 -3
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +36 -3
- package/dist/status.js.map +1 -1
- package/dist/types.d.ts +24 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/workspace-lock.d.ts +20 -0
- package/dist/workspace-lock.d.ts.map +1 -1
- package/dist/workspace-lock.js +63 -0
- package/dist/workspace-lock.js.map +1 -1
- package/dist/workspace-resolve.d.ts +0 -9
- package/dist/workspace-resolve.d.ts.map +1 -1
- package/dist/workspace-resolve.js +115 -183
- package/dist/workspace-resolve.js.map +1 -1
- package/dist/workspaces.d.ts +15 -0
- package/dist/workspaces.d.ts.map +1 -1
- package/dist/workspaces.js +330 -90
- package/dist/workspaces.js.map +1 -1
- package/package.json +5 -4
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Starting a job.
|
|
3
|
+
*
|
|
4
|
+
* Split out of jobs.ts with the rest of its concerns; nothing was rewritten.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
7
|
+
import { pruneDeadWorkspaceLocks } from "../workspace-lock.js";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { resolveWorkingDir, validateWorkingDir, workingDirWarning } from "../working-dir.js";
|
|
10
|
+
import { buildContextPreamble } from "./context.js";
|
|
11
|
+
import { jobsRoot, newJobId, pollInstructions, pruneStaleJobs, readJson, snapshotFiles, SUGGESTED_POLL_SECONDS, timestamp, updateStatus, writeJson, } from "./store.js";
|
|
12
|
+
import { resolveRunnerPath, runJob, watchUntilTerminal } from "./run.js";
|
|
13
|
+
import { configLoadError, drainSlotQueue, } from "./supervisor.js";
|
|
14
|
+
export async function startAsyncJob(deps, input) {
|
|
15
|
+
return (await startAsyncJobTracked(deps, input)).status;
|
|
16
|
+
}
|
|
17
|
+
export async function startAsyncJobTracked(deps, input) {
|
|
18
|
+
// Before anything is created on disk. Every dispatch path — MCP, HTTP,
|
|
19
|
+
// fanout — funnels through here, so this is the one place that catches a bad
|
|
20
|
+
// workingDir while the error can still name the real cause, and the only
|
|
21
|
+
// point at which failing leaves no half-built job directory behind.
|
|
22
|
+
const workingDirError = validateWorkingDir(input.workingDir);
|
|
23
|
+
if (workingDirError !== undefined)
|
|
24
|
+
throw new Error(workingDirError);
|
|
25
|
+
// The runner reads the config FILE, so a file this server can no longer load
|
|
26
|
+
// means no runner can start — and the job would sit untouched until the 90s
|
|
27
|
+
// orphan threshold reported it dead. Observed: a caller told
|
|
28
|
+
// "ended without a result (status: orphaned)" about a job whose own
|
|
29
|
+
// status.json later read completed/success. Two false statements from one
|
|
30
|
+
// broken file, ninety seconds apart.
|
|
31
|
+
//
|
|
32
|
+
// The server itself is fine: a failed hot-reload keeps the previous config
|
|
33
|
+
// in memory, which is why it can still accept the dispatch at all. That
|
|
34
|
+
// divergence between what the server runs and what the runner would read is
|
|
35
|
+
// the whole bug, so it is refused here, immediately, naming the real cause —
|
|
36
|
+
// before a job directory exists to be misreported.
|
|
37
|
+
const configError = await configLoadError(deps.holder.state.configPath);
|
|
38
|
+
if (configError !== undefined)
|
|
39
|
+
throw new Error(configError);
|
|
40
|
+
await pruneStaleJobs();
|
|
41
|
+
// Same maintenance moment, same reason: a lock whose holder is gone is only
|
|
42
|
+
// reclaimed when something contends for that exact directory, so a
|
|
43
|
+
// dispatched-once workspace leaves its file behind forever. Best effort, and
|
|
44
|
+
// deliberately not awaited-into-failure — a sweep must never block the job
|
|
45
|
+
// that was actually asked for.
|
|
46
|
+
await pruneDeadWorkspaceLocks().catch(() => undefined);
|
|
47
|
+
const jobId = newJobId();
|
|
48
|
+
const root = jobsRoot();
|
|
49
|
+
const jobDir = path.join(root, jobId);
|
|
50
|
+
await mkdir(path.join(jobDir, "context"), { recursive: true, mode: 0o700 });
|
|
51
|
+
await mkdir(path.join(jobDir, "output"), { recursive: true, mode: 0o700 });
|
|
52
|
+
const promptPath = path.join(jobDir, "prompt.md");
|
|
53
|
+
// Prepend prior-job context before the prompt is frozen to disk, so the
|
|
54
|
+
// runner, the manifest and any later inspection all see exactly what the
|
|
55
|
+
// delegate was given.
|
|
56
|
+
const preamble = await buildContextPreamble(input.contextJobs ?? []);
|
|
57
|
+
const effectivePrompt = preamble + input.prompt;
|
|
58
|
+
await writeFile(promptPath, effectivePrompt, { encoding: "utf8", mode: 0o600 });
|
|
59
|
+
const fileSnapshots = await snapshotFiles(jobDir, input.files ?? []);
|
|
60
|
+
const createdAt = timestamp();
|
|
61
|
+
const resolvedWorkingDir = resolveWorkingDir(input.workingDir);
|
|
62
|
+
const warning = workingDirWarning(resolvedWorkingDir);
|
|
63
|
+
const manifest = {
|
|
64
|
+
jobId,
|
|
65
|
+
createdAt,
|
|
66
|
+
workingDir: resolvedWorkingDir.workingDir,
|
|
67
|
+
promptPath,
|
|
68
|
+
files: fileSnapshots,
|
|
69
|
+
...(input.hints !== undefined ? { hints: input.hints } : {}),
|
|
70
|
+
...(input.workspacePolicy !== undefined ? { workspacePolicy: input.workspacePolicy } : {}),
|
|
71
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
72
|
+
...(input.retryOf !== undefined ? { retryOf: input.retryOf } : {}),
|
|
73
|
+
...(warning !== undefined ? { warning } : {}),
|
|
74
|
+
};
|
|
75
|
+
await writeJson(path.join(jobDir, "manifest.json"), manifest);
|
|
76
|
+
const status = {
|
|
77
|
+
jobId,
|
|
78
|
+
status: "queued",
|
|
79
|
+
createdAt,
|
|
80
|
+
updatedAt: createdAt,
|
|
81
|
+
jobDir,
|
|
82
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
83
|
+
nextPollSeconds: SUGGESTED_POLL_SECONDS,
|
|
84
|
+
instructions: pollInstructions(jobId),
|
|
85
|
+
...(warning !== undefined ? { warning } : {}),
|
|
86
|
+
};
|
|
87
|
+
await updateStatus(jobDir, status);
|
|
88
|
+
// Detached by default: the run must not die with this process. In-process
|
|
89
|
+
// mode exists for unit tests (injected fake dispatchers aren't visible to
|
|
90
|
+
// a separate process) and as the fallback when the runner script can't be
|
|
91
|
+
// found (running from raw src/ with no dist/ build).
|
|
92
|
+
const inproc = process.env.HARNESS_DISPATCH_INPROC_JOBS === "1";
|
|
93
|
+
const runnerPath = inproc ? undefined : resolveRunnerPath();
|
|
94
|
+
if (inproc || runnerPath === undefined) {
|
|
95
|
+
if (!inproc) {
|
|
96
|
+
console.error("harness-dispatch: dist/job-runner.js not found (unbuilt checkout?) — " +
|
|
97
|
+
"running the job in-process; it will not survive a server restart.");
|
|
98
|
+
}
|
|
99
|
+
// The DETACHED runner re-reads prompt.md, which carries the context
|
|
100
|
+
// preamble — so the in-process run must dispatch the same frozen prompt,
|
|
101
|
+
// not input.prompt. Passing the raw prompt here silently dropped
|
|
102
|
+
// contextJobs for every in-process run (unit tests with injected fakes,
|
|
103
|
+
// and the unbuilt-checkout fallback).
|
|
104
|
+
const completion = runJob(deps, jobDir, manifest, { ...input, prompt: effectivePrompt });
|
|
105
|
+
return { status, completion };
|
|
106
|
+
}
|
|
107
|
+
// Concurrency gate. Every dispatch spawns its own detached runner, so an
|
|
108
|
+
// in-process semaphore would bound nothing — the count has to come off
|
|
109
|
+
// disk. The caller still gets its jobId back immediately either way, so the
|
|
110
|
+
// API contract is unchanged and only the start time can move.
|
|
111
|
+
// Every dispatch takes this path, including an uncapped one. It used to
|
|
112
|
+
// spawn its own detached runner when the cap was lifted, which bypassed the
|
|
113
|
+
// pool and put a ~76 MB wrapper behind every job.
|
|
114
|
+
// Enqueue first, then let drainSlotQueue decide — rather than testing the
|
|
115
|
+
// limit here and spawning inline. Two reasons, both learned the hard way:
|
|
116
|
+
// this job's own `queued` status is already on disk, so an inline count
|
|
117
|
+
// included itself and deadlocked at limit 1; and a fresh dispatch arriving
|
|
118
|
+
// while others wait must not jump the queue, which only one FIFO drainer
|
|
119
|
+
// can guarantee. Whether this job starts now is then just "did the drain
|
|
120
|
+
// reach it".
|
|
121
|
+
await updateStatus(jobDir, { ...status, slotQueued: true });
|
|
122
|
+
await drainSlotQueue(deps.holder.state.config, deps.holder.state.configPath);
|
|
123
|
+
const settled = await readJson(path.join(jobDir, "status.json"));
|
|
124
|
+
return { status: settled, completion: watchUntilTerminal(jobDir) };
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=start.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"start.js","sourceRoot":"","sources":["../../src/jobs/start.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,cAAc,EACd,QAAQ,EACR,aAAa,EACb,sBAAsB,EACtB,SAAS,EACT,YAAY,EACZ,SAAS,GACV,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACzE,OAAO,EACL,eAAe,EACf,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAa,EAAE,KAAoB;IACrE,OAAO,CAAC,MAAM,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAa,EAAE,KAAoB;IAC5E,uEAAuE;IACvE,6EAA6E;IAC7E,yEAAyE;IACzE,oEAAoE;IACpE,MAAM,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7D,IAAI,eAAe,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;IAEpE,6EAA6E;IAC7E,4EAA4E;IAC5E,6DAA6D;IAC7D,oEAAoE;IACpE,0EAA0E;IAC1E,qCAAqC;IACrC,EAAE;IACF,2EAA2E;IAC3E,wEAAwE;IACxE,4EAA4E;IAC5E,6EAA6E;IAC7E,mDAAmD;IACnD,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACxE,IAAI,WAAW,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;IAE5D,MAAM,cAAc,EAAE,CAAC;IACvB,4EAA4E;IAC5E,mEAAmE;IACnE,6EAA6E;IAC7E,2EAA2E;IAC3E,+BAA+B;IAC/B,MAAM,uBAAuB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5E,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClD,wEAAwE;IACxE,yEAAyE;IACzE,sBAAsB;IACtB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC;IAChD,MAAM,SAAS,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChF,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;IAC9B,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAgB;QAC5B,KAAK;QACL,SAAS;QACT,UAAU,EAAE,kBAAkB,CAAC,UAAU;QACzC,UAAU;QACV,KAAK,EAAE,aAAa;QACpB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAc;QACxB,KAAK;QACL,MAAM,EAAE,QAAQ;QAChB,SAAS;QACT,SAAS,EAAE,SAAS;QACpB,MAAM;QACN,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,eAAe,EAAE,sBAAsB;QACvC,YAAY,EAAE,gBAAgB,CAAC,KAAK,CAAC;QACrC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnC,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,qDAAqD;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,GAAG,CAAC;IAChE,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC;IAC5D,IAAI,MAAM,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CACX,uEAAuE;gBACrE,mEAAmE,CACtE,CAAC;QACJ,CAAC;QACD,oEAAoE;QACpE,yEAAyE;QACzE,iEAAiE;QACjE,wEAAwE;QACxE,sCAAsC;QACtC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;QACzF,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,4EAA4E;IAC5E,8DAA8D;IAC9D,wEAAwE;IACxE,4EAA4E;IAC5E,kDAAkD;IAElD,0EAA0E;IAC1E,0EAA0E;IAC1E,wEAAwE;IACxE,2EAA2E;IAC3E,yEAAyE;IACzE,yEAAyE;IACzE,aAAa;IACb,MAAM,YAAY,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAY,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAC5E,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;AACrE,CAAC"}
|
package/dist/jobs/store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAUH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AASzD,iEAAiE;AACjE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,QAAS,CAAC;AAE1C;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAwB5D;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAMD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAKlE;AAED,wBAAgB,WAAW,IAAI,MAAM,CAKpC;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAmEpD;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/E;AAgBD,wBAAsB,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAE9D;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBnF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CA4BlG;AAED;;;GAGG;AACH,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAED,eAAO,MAAM,SAAS,QAA0B,CAAC;AAEjD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAMpD;AAmBD,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMlF;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEvD;AAED,mEAAmE;AACnE,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAQ9E"}
|
package/dist/jobs/store.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* Windows EPERM.
|
|
13
13
|
*/
|
|
14
14
|
import { existsSync } from "node:fs";
|
|
15
|
+
import { redact } from "../redaction.js";
|
|
15
16
|
import { chmod, copyFile, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
16
17
|
import { randomUUID } from "node:crypto";
|
|
17
18
|
import path from "node:path";
|
|
@@ -53,8 +54,16 @@ export function withOrphanCheck(status) {
|
|
|
53
54
|
...status,
|
|
54
55
|
status: "orphaned",
|
|
55
56
|
success: false,
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
// Says what is KNOWN, not which process died. This used to read "The
|
|
58
|
+
// dispatch server that started this job exited" — and a load test killed
|
|
59
|
+
// a single supervisor while the server stayed up and serving, producing
|
|
60
|
+
// that message verbatim. The status file this is derived from records a
|
|
61
|
+
// heartbeat and nothing about who was holding it, so naming a culprit was
|
|
62
|
+
// always a guess, and it sent anyone debugging to the wrong process.
|
|
63
|
+
error: "This job stopped reporting progress and the process running it is gone — " +
|
|
64
|
+
"either the run crashed or whatever was supervising it died. Nothing will " +
|
|
65
|
+
"advance it now. Its partial output is on disk; `retry_job` re-runs the same " +
|
|
66
|
+
"task, or re-dispatch it.",
|
|
58
67
|
};
|
|
59
68
|
}
|
|
60
69
|
export function boundedError(error) {
|
|
@@ -185,7 +194,8 @@ export function safeBaseName(filePath) {
|
|
|
185
194
|
}
|
|
186
195
|
export async function writeJson(filePath, value) {
|
|
187
196
|
const tmpPath = `${filePath}.${process.pid}.${Date.now()}.${randomUUID().slice(0, 8)}.tmp`;
|
|
188
|
-
|
|
197
|
+
// Sink: every job JSON on disk — status.json, result.json, manifest.json.
|
|
198
|
+
await writeFile(tmpPath, `${redact(JSON.stringify(value, null, 2))}\n`, { encoding: "utf8", mode: 0o600 });
|
|
189
199
|
await renameWithRetry(tmpPath, filePath);
|
|
190
200
|
}
|
|
191
201
|
async function renameWithRetry(tmpPath, filePath) {
|
package/dist/jobs/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1G,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,iEAAiE;AACjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,2EAA2E;IAC3E,2EAA2E;IAC3E,oEAAoE;IACpE,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,mBAAmB;QAAE,OAAO,MAAM,CAAC;IACrF,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,UAAU;QAClB,OAAO,EAAE,KAAK;QACd,KAAK,EACH,
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1G,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,iEAAiE;AACjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,2EAA2E;IAC3E,2EAA2E;IAC3E,oEAAoE;IACpE,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,mBAAmB;QAAE,OAAO,MAAM,CAAC;IACrF,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,UAAU;QAClB,OAAO,EAAE,KAAK;QACd,qEAAqE;QACrE,yEAAyE;QACzE,wEAAwE;QACxE,wEAAwE;QACxE,0EAA0E;QAC1E,qEAAqE;QACrE,KAAK,EACH,2EAA2E;YAC3E,2EAA2E;YAC3E,8EAA8E;YAC9E,0BAA0B;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAyB;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,IAAI,oBAAoB;QAAE,OAAO,KAAK,CAAC;IACvD,OAAO,CACL,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACpC,iBAAiB,KAAK,CAAC,MAAM,GAAG,oBAAoB,0CAA0C,CAC/F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,CACL,yEAAyE;QACzE,SAAS,IAAI,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,CAAC,mCAAmC;QACnF,yBAAyB,KAAK,mDAAmD;QACjF,iFAAiF;QACjF,0CAA0C,CAC3C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvD,IAAI,qBAAyC,CAAC;AAE9C;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAwB;IAC1D,qBAAqB;QACnB,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;YACtD,CAAC,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;YAC5B,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IACxD,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;AACzD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,sEAAsE;IACtE,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,gDAAgD;IAChD,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO;IAC3B,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,0EAA0E;QAC1E,uDAAuD;QACvD,0EAA0E;QAC1E,0EAA0E;QAC1E,6DAA6D;QAC7D,iEAAiE;QACjE,EAAE;QACF,wEAAwE;QACxE,0EAA0E;QAC1E,wEAAwE;QACxE,yEAAyE;QACzE,mDAAmD;QACnD,EAAE;QACF,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,yEAAyE;QACzE,mDAAmD;QACnD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,QAAQ;gBAAE,SAAS;YAC7C,4DAA4D;YAC5D,sEAAsE;YACtE,wEAAwE;YACxE,uEAAuE;YACvE,0DAA0D;YAC1D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CACd,CAAC;gBAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;gBAChD,IACE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC;oBAC3D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACrB,GAAG,GAAG,IAAI,IAAI,mBAAmB,EACjC,CAAC;oBACD,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;YACD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;QAC3E,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IAC9D,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3F,0EAA0E;IAC1E,MAAM,SAAS,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3G,MAAM,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,QAAgB;IAC9D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACpG,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO;gBAAE,MAAM,GAAG,CAAC;YACzE,MAAM,KAAK,CAAC,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAI,QAAgB;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAM,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,MAAiB;IAClE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE;YAChD,GAAG,MAAM;YACT,SAAS,EAAE,SAAS,EAAE;SACvB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,wEAAwE;QACxE,EAAE;QACF,qEAAqE;QACrE,yEAAyE;QACzE,uEAAuE;QACvE,sEAAsE;QACtE,uEAAuE;QACvE,sEAAsE;QACtE,oDAAoD;QACpD,EAAE;QACF,yEAAyE;QACzE,sEAAsE;QACtE,wEAAwE;QACxE,MAAM,IAAI,GACR,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QACzF,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO;QACrD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,KAAe;IACjE,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAiC,EAAE,YAAY,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,oBAAoB,CAAC;gBAClC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACf,SAAS;YACX,CAAC;YACD,MAAM,YAAY,GAAG,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3F,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YACvD,MAAM,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;YAC3C,MAAM,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;YACjC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC;IACjE,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAEjD;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,4CAA4C,CACnF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,aAAa,GAAG,aAAa,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,MAAe;IACjE,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAChC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAChF,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAClC,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;QACvD,OAAO,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Admission control and the supervisor pool: who runs, when, and in which
|
|
3
|
+
* process.
|
|
4
|
+
*
|
|
5
|
+
* The concurrency cap here exists because of a measured OOM, and the pool
|
|
6
|
+
* exists because a runner process per job costs ~76 MB of wrapper. Both are
|
|
7
|
+
* load-bearing; see the comments on DEFAULT_MAX_CONCURRENT_RUNS.
|
|
8
|
+
*/
|
|
9
|
+
import type { RouterConfig } from "../types.js";
|
|
10
|
+
import type { JobDeps, JobStatus } from "./types.js";
|
|
11
|
+
/**
|
|
12
|
+
* The cap, or `null` for "no cap".
|
|
13
|
+
*
|
|
14
|
+
* `null` rather than `0`, and rather than `Infinity`, because both of those
|
|
15
|
+
* were wrong in a way that mattered. `0` used to short-circuit the whole slot
|
|
16
|
+
* queue and supervisor pool, so `max_concurrent_runs: 0` — documented as
|
|
17
|
+
* lifting a limit — silently gave every job its own runner process at ~76 MB,
|
|
18
|
+
* which is the per-job cost the pool exists to remove, on the memory-bound
|
|
19
|
+
* machine the cap exists for. And `Infinity` divides badly: the pool sizes
|
|
20
|
+
* itself with `outstanding / jobsPerSupervisor(limit)`, so an infinite limit
|
|
21
|
+
* asked for ZERO supervisors. An explicit `null` makes each site say what it
|
|
22
|
+
* means about the unbounded case.
|
|
23
|
+
*/
|
|
24
|
+
export declare function maxConcurrentRuns(config: RouterConfig | undefined): number | null;
|
|
25
|
+
/**
|
|
26
|
+
* Occupied slots: jobs actually executing right now. Counts `running` (and
|
|
27
|
+
* plain `queued` — a runner spawned but not yet started) only while the
|
|
28
|
+
* heartbeat is fresh, so a crashed runner's slot is reclaimed by the same
|
|
29
|
+
* ORPHAN_THRESHOLD_MS rule that already frees its status. Slot-queued jobs
|
|
30
|
+
* are waiting for a slot, not holding one.
|
|
31
|
+
*/
|
|
32
|
+
/**
|
|
33
|
+
* What one run of a route costs against the concurrency budget.
|
|
34
|
+
*
|
|
35
|
+
* Unknown routes count as a full 1.0 on purpose. A job that has not been
|
|
36
|
+
* routed yet (no forced `service`) has no weight to look up, and this bound
|
|
37
|
+
* exists because a measured burst of 13 concurrent CLIs exhausted memory —
|
|
38
|
+
* so the safe assumption for "might be anything" is "might be heavy".
|
|
39
|
+
*/
|
|
40
|
+
export declare function resourceWeightFor(status: JobStatus, config: RouterConfig | undefined): number;
|
|
41
|
+
/**
|
|
42
|
+
* Capacity currently in use, as a weighted sum rather than a job count.
|
|
43
|
+
*
|
|
44
|
+
* With every weight at 1.0 this is exactly the old count, so an existing
|
|
45
|
+
* `max_concurrent_runs` keeps its previous meaning.
|
|
46
|
+
*/
|
|
47
|
+
export declare function activeCapacity(statuses: Array<{
|
|
48
|
+
status: JobStatus;
|
|
49
|
+
}>, config: RouterConfig | undefined): number;
|
|
50
|
+
/**
|
|
51
|
+
* How many supervisor PROCESSES may exist, regardless of how many jobs run.
|
|
52
|
+
*
|
|
53
|
+
* Previously every job got its own detached Node process. Measured on Windows
|
|
54
|
+
* with Node 24: a bare node process is 52 MB RSS and one that has bootstrapped
|
|
55
|
+
* a runtime is 65 MB, against ~54 MB for the agent CLI it exists to supervise.
|
|
56
|
+
* So more than half the memory of a concurrent run was wrapper, and it scaled
|
|
57
|
+
* linearly — 13 concurrent jobs meant 845 MB of supervision before any agent
|
|
58
|
+
* had read a file. That is the concurrency ceiling.
|
|
59
|
+
*
|
|
60
|
+
* A supervisor is almost entirely idle: it waits on a child process and writes
|
|
61
|
+
* the result. One can watch several at once for the cost of async I/O, so
|
|
62
|
+
* wrapper memory becomes O(1) in the number of jobs instead of O(N), capped
|
|
63
|
+
* here at ~260 MB.
|
|
64
|
+
*
|
|
65
|
+
* Four rather than one purely to bound blast radius: a supervisor crash strands
|
|
66
|
+
* only the jobs it held. Those are recoverable anyway — the job directory is
|
|
67
|
+
* the source of truth and the heartbeat check already marks stranded jobs
|
|
68
|
+
* orphaned — but losing a quarter of in-flight work beats losing all of it.
|
|
69
|
+
*/
|
|
70
|
+
export declare const SUPERVISOR_POOL_SIZE = 4;
|
|
71
|
+
/**
|
|
72
|
+
* Take exclusive ownership of a job directory.
|
|
73
|
+
*
|
|
74
|
+
* `wx` fails if the file exists, atomically, on both Windows and POSIX — which
|
|
75
|
+
* is what stops two supervisors racing onto the same job. A claim left behind
|
|
76
|
+
* by a crashed supervisor is reclaimed once that job's heartbeat has gone
|
|
77
|
+
* stale, by the same ORPHAN_THRESHOLD_MS rule used everywhere else.
|
|
78
|
+
*
|
|
79
|
+
* Exported for tests: the one-winner property under concurrent reclaim is the
|
|
80
|
+
* invariant, and it is only checkable by calling this directly.
|
|
81
|
+
*/
|
|
82
|
+
export declare function claimJobDir(jobDir: string, status: JobStatus): Promise<boolean>;
|
|
83
|
+
/**
|
|
84
|
+
* Supervisor main loop: claim work, run several jobs at once, exit when idle.
|
|
85
|
+
*
|
|
86
|
+
* Exiting on idle keeps the no-jobs steady state at zero processes, same as
|
|
87
|
+
* before — the pool is a way to share supervision cost while work exists, not
|
|
88
|
+
* a daemon.
|
|
89
|
+
*/
|
|
90
|
+
export declare function runSupervisor(deps: JobDeps, supervisorId?: string): Promise<void>;
|
|
91
|
+
/**
|
|
92
|
+
* Start slot-queued jobs, oldest first, until the machine is at its limit.
|
|
93
|
+
*
|
|
94
|
+
* Deliberately has no daemon behind it: this runs on every new dispatch and
|
|
95
|
+
* again as each runner exits, which between them covers every moment a slot
|
|
96
|
+
* can free. The cost of that choice is that if every runner dies while jobs
|
|
97
|
+
* are queued, the queue resumes on the next dispatch rather than immediately.
|
|
98
|
+
* Bounded waiting was the explicit alternative and was not chosen — a queued
|
|
99
|
+
* job keeps its jobId and its artifacts either way, so nothing is lost.
|
|
100
|
+
*
|
|
101
|
+
* NOT called at server start, which was tried and reverted: it silently ran
|
|
102
|
+
* jobs abandoned by a dead session. `orphanStrandedSlotQueue` runs there
|
|
103
|
+
* instead and reports them. See its comment for why reporting beats resuming.
|
|
104
|
+
*/
|
|
105
|
+
/**
|
|
106
|
+
* Mark jobs stranded in the slot queue by a server that is gone.
|
|
107
|
+
*
|
|
108
|
+
* Called once at server start, where the reasoning holds unconditionally: this
|
|
109
|
+
* process has not queued anything yet, so anything still slot-queued was
|
|
110
|
+
* queued by a session that no longer exists and nothing will ever drain it —
|
|
111
|
+
* a new dispatch would, but the caller is asking about THIS job, and until
|
|
112
|
+
* they happen to send unrelated work it reads `queued` forever.
|
|
113
|
+
*
|
|
114
|
+
* Deliberately reports rather than runs. Resuming was tried and is worse: a
|
|
115
|
+
* job queued days ago would execute at the next server start, in its original
|
|
116
|
+
* workingDir, at up to `full_auto`, with nobody watching. The job keeps its id
|
|
117
|
+
* and artifacts, so `retry_job` re-runs it as a decision rather than a side
|
|
118
|
+
* effect of opening an editor.
|
|
119
|
+
*
|
|
120
|
+
* The one status this writes back. Orphan detection elsewhere is
|
|
121
|
+
* compute-on-read and never persists its verdict, because the owner might
|
|
122
|
+
* still be alive; here the owner is definitionally gone.
|
|
123
|
+
*/
|
|
124
|
+
export declare function orphanStrandedSlotQueue(): Promise<number>;
|
|
125
|
+
export declare function drainSlotQueue(config: RouterConfig | undefined, configPath: string | undefined): Promise<void>;
|
|
126
|
+
/**
|
|
127
|
+
* Supervisors currently alive, counted from their heartbeat files.
|
|
128
|
+
*
|
|
129
|
+
* Approximate on purpose: over-counting briefly means the pool runs one short
|
|
130
|
+
* until the next drain, and under-counting means one extra supervisor that
|
|
131
|
+
* finds no work and exits within SUPERVISOR_IDLE_EXIT_MS. Neither warrants a
|
|
132
|
+
* lock, and both self-correct.
|
|
133
|
+
*/
|
|
134
|
+
declare function countLiveSupervisors(): Promise<number>;
|
|
135
|
+
/**
|
|
136
|
+
* Exported for the cleanup test, which must exercise the REAL sweep rather
|
|
137
|
+
* than a copy of its logic — the bug being pinned is that a stale heartbeat
|
|
138
|
+
* was never removed, and a reimplementation in the test would pin nothing.
|
|
139
|
+
*/
|
|
140
|
+
export declare const countLiveSupervisorsForTest: typeof countLiveSupervisors;
|
|
141
|
+
/**
|
|
142
|
+
* Why a detached runner would fail to bootstrap from this config path, if it
|
|
143
|
+
* would. `undefined` means the file loads (or there is none, which is the
|
|
144
|
+
* auto-detect case and always fine).
|
|
145
|
+
*
|
|
146
|
+
* Deliberately re-reads rather than trusting the server's in-memory config:
|
|
147
|
+
* the two disagreeing is exactly the condition being detected.
|
|
148
|
+
*/
|
|
149
|
+
export declare function configLoadError(configPath: string | undefined): Promise<string | undefined>;
|
|
150
|
+
export {};
|
|
151
|
+
//# sourceMappingURL=supervisor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervisor.d.ts","sourceRoot":"","sources":["../../src/jobs/supervisor.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAWH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAWhD,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AA6BrD;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAMjF;AAoBD;;;;;;GAMG;AACH;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,MAAM,CAQ7F;AAeD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,CAAC,EACtC,MAAM,EAAE,YAAY,GAAG,SAAS,GAC/B,MAAM,CAUR;AAOD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAoBtC;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAuCrF;AAwBD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgGvF;AAED;;;;;;;;;;;;;GAaG;AACH;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,MAAM,CAAC,CAqC/D;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,YAAY,GAAG,SAAS,EAChC,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAkGD;;;;;;;GAOG;AACH,iBAAe,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CA2CrD;AAED;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,6BAAuB,CAAC;AA0ChE;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAcjG"}
|