harness-dispatch 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +560 -1
- package/README.md +74 -43
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js.map +1 -1
- package/dist/bin.d.ts.map +1 -1
- package/dist/bin.js +166 -29
- package/dist/bin.js.map +1 -1
- package/dist/config/coercions.d.ts +5 -0
- package/dist/config/coercions.d.ts.map +1 -1
- package/dist/config/coercions.js +8 -0
- package/dist/config/coercions.js.map +1 -1
- package/dist/config/protocol.js.map +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js.map +1 -1
- package/dist/config.d.ts +11 -3
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +111 -148
- package/dist/config.js.map +1 -1
- package/dist/configure-yaml.d.ts.map +1 -1
- package/dist/configure-yaml.js +57 -13
- package/dist/configure-yaml.js.map +1 -1
- package/dist/dispatch-log.d.ts.map +1 -1
- package/dist/dispatch-log.js +3 -1
- package/dist/dispatch-log.js.map +1 -1
- package/dist/dispatchers/generic-cli.d.ts.map +1 -1
- package/dist/dispatchers/generic-cli.js +189 -97
- package/dist/dispatchers/generic-cli.js.map +1 -1
- package/dist/dispatchers/openai-compatible.d.ts +0 -5
- package/dist/dispatchers/openai-compatible.d.ts.map +1 -1
- package/dist/dispatchers/openai-compatible.js +129 -82
- package/dist/dispatchers/openai-compatible.js.map +1 -1
- package/dist/dispatchers/shared/harness-login.d.ts.map +1 -1
- package/dist/dispatchers/shared/harness-login.js +7 -2
- package/dist/dispatchers/shared/harness-login.js.map +1 -1
- package/dist/dispatchers/shared/stream-subprocess.d.ts.map +1 -1
- package/dist/dispatchers/shared/stream-subprocess.js +7 -60
- package/dist/dispatchers/shared/stream-subprocess.js.map +1 -1
- package/dist/file-lock.d.ts.map +1 -1
- package/dist/file-lock.js +31 -19
- package/dist/file-lock.js.map +1 -1
- package/dist/harness-presets.d.ts +51 -0
- package/dist/harness-presets.d.ts.map +1 -0
- package/dist/harness-presets.js +154 -0
- package/dist/harness-presets.js.map +1 -0
- package/dist/http/parse.d.ts +8 -1
- package/dist/http/parse.d.ts.map +1 -1
- package/dist/http/parse.js +10 -3
- package/dist/http/parse.js.map +1 -1
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +9 -2
- package/dist/http/server.js.map +1 -1
- package/dist/job-runner.js +6 -0
- package/dist/job-runner.js.map +1 -1
- package/dist/jobs/lifecycle.d.ts +100 -0
- package/dist/jobs/lifecycle.d.ts.map +1 -0
- package/dist/jobs/lifecycle.js +263 -0
- package/dist/jobs/lifecycle.js.map +1 -0
- package/dist/jobs/read.d.ts +17 -0
- package/dist/jobs/read.d.ts.map +1 -0
- package/dist/jobs/read.js +103 -0
- package/dist/jobs/read.js.map +1 -0
- package/dist/jobs/run.d.ts +45 -0
- package/dist/jobs/run.d.ts.map +1 -0
- package/dist/jobs/run.js +351 -0
- package/dist/jobs/run.js.map +1 -0
- package/dist/jobs/start.d.ts +9 -0
- package/dist/jobs/start.d.ts.map +1 -0
- package/dist/jobs/start.js +126 -0
- package/dist/jobs/start.js.map +1 -0
- package/dist/jobs/store.d.ts.map +1 -1
- package/dist/jobs/store.js +13 -3
- package/dist/jobs/store.js.map +1 -1
- package/dist/jobs/supervisor.d.ts +151 -0
- package/dist/jobs/supervisor.d.ts.map +1 -0
- package/dist/jobs/supervisor.js +679 -0
- package/dist/jobs/supervisor.js.map +1 -0
- package/dist/jobs.d.ts +15 -233
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +14 -1409
- package/dist/jobs.js.map +1 -1
- package/dist/leaderboard.d.ts.map +1 -1
- package/dist/leaderboard.js +5 -1
- package/dist/leaderboard.js.map +1 -1
- package/dist/mcp/config-hot-reload.d.ts.map +1 -1
- package/dist/mcp/config-hot-reload.js +4 -1
- package/dist/mcp/config-hot-reload.js.map +1 -1
- package/dist/mcp/dispatcher-factory.js +1 -1
- package/dist/mcp/dispatcher-factory.js.map +1 -1
- package/dist/mcp/resources.d.ts.map +1 -1
- package/dist/mcp/resources.js +3 -2
- package/dist/mcp/resources.js.map +1 -1
- package/dist/mcp/tools.d.ts +9 -2
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +45 -6
- package/dist/mcp/tools.js.map +1 -1
- package/dist/observability/spans.d.ts.map +1 -1
- package/dist/observability/spans.js +10 -2
- package/dist/observability/spans.js.map +1 -1
- package/dist/redaction.d.ts +90 -0
- package/dist/redaction.d.ts.map +1 -0
- package/dist/redaction.js +221 -0
- package/dist/redaction.js.map +1 -0
- package/dist/router.d.ts +12 -23
- package/dist/router.d.ts.map +1 -1
- package/dist/router.js +83 -224
- package/dist/router.js.map +1 -1
- package/dist/state-dir.d.ts.map +1 -1
- package/dist/state-dir.js +15 -1
- package/dist/state-dir.js.map +1 -1
- package/dist/status.d.ts +14 -3
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +36 -3
- package/dist/status.js.map +1 -1
- package/dist/workspace-lock.d.ts +20 -0
- package/dist/workspace-lock.d.ts.map +1 -1
- package/dist/workspace-lock.js +63 -0
- package/dist/workspace-lock.js.map +1 -1
- package/dist/workspace-resolve.d.ts +0 -9
- package/dist/workspace-resolve.d.ts.map +1 -1
- package/dist/workspace-resolve.js +38 -183
- package/dist/workspace-resolve.js.map +1 -1
- package/dist/workspaces.d.ts.map +1 -1
- package/dist/workspaces.js +63 -139
- package/dist/workspaces.js.map +1 -1
- package/package.json +5 -4
package/dist/jobs/run.js
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running one job: the work a runner process actually does.
|
|
3
|
+
*
|
|
4
|
+
* Split out of jobs.ts, which had grown to hold five concerns at 1,600 lines
|
|
5
|
+
* — this one, admission and the supervisor pool, the start/read verbs, and
|
|
6
|
+
* the lifecycle verbs. Nothing here was rewritten; the code moved.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync } from "node:fs";
|
|
9
|
+
import { redact } from "../redaction.js";
|
|
10
|
+
import { appendFile, readFile, writeFile } from "node:fs/promises";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
14
|
+
import { boundedError, cancelReason, cancelRequested, readJson, timestamp, updateStatus, withOrphanCheck, writeJson, } from "./store.js";
|
|
15
|
+
export async function runJob(deps, jobDir, manifest, input) {
|
|
16
|
+
const started = Date.now();
|
|
17
|
+
const runningStatus = () => ({
|
|
18
|
+
jobId: manifest.jobId,
|
|
19
|
+
status: "running",
|
|
20
|
+
createdAt: manifest.createdAt,
|
|
21
|
+
updatedAt: timestamp(),
|
|
22
|
+
jobDir,
|
|
23
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
24
|
+
...(manifest.warning !== undefined ? { warning: manifest.warning } : {}),
|
|
25
|
+
});
|
|
26
|
+
await updateStatus(jobDir, runningStatus());
|
|
27
|
+
// Heartbeat: bump updatedAt while the run is alive so a reader can tell
|
|
28
|
+
// "running" apart from "the server that owned this run died and left a
|
|
29
|
+
// stale status file" (getAsyncJob reports the latter as "orphaned").
|
|
30
|
+
// unref'd so an exiting process never lingers on it — which is exactly
|
|
31
|
+
// the scenario the heartbeat exists to expose. The `finished` flag stops
|
|
32
|
+
// a beat that FIRES after the terminal write; `pendingBeat` covers the
|
|
33
|
+
// beat that fired BEFORE it and is still mid-write — updateStatus's rename
|
|
34
|
+
// can back off ~900ms on Windows EPERM, long enough to land after the
|
|
35
|
+
// terminal status and re-mark a completed job "running" (then "orphaned"
|
|
36
|
+
// forever in the list view). The terminal paths await it before writing.
|
|
37
|
+
let finished = false;
|
|
38
|
+
let pendingBeat = Promise.resolve();
|
|
39
|
+
const heartbeat = setInterval(() => {
|
|
40
|
+
if (finished)
|
|
41
|
+
return;
|
|
42
|
+
pendingBeat = updateStatus(jobDir, runningStatus()).catch(() => undefined);
|
|
43
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
44
|
+
heartbeat.unref?.();
|
|
45
|
+
try {
|
|
46
|
+
const state = deps.holder.state;
|
|
47
|
+
const files = input.files ?? [];
|
|
48
|
+
// Reuse the value already resolved (and recorded) at job creation, not a
|
|
49
|
+
// fresh process.cwd() snapshot — the two must stay in sync with the
|
|
50
|
+
// warning captured in manifest.warning.
|
|
51
|
+
const workingDir = manifest.workingDir;
|
|
52
|
+
const hints = { ...(input.hints ?? {}) };
|
|
53
|
+
if (input.workspacePolicy !== undefined)
|
|
54
|
+
hints.workspacePolicy = input.workspacePolicy;
|
|
55
|
+
// Stream the dispatch so agents polling action=get can watch progress in
|
|
56
|
+
// stdout.partial.log instead of waiting blind for the final result.
|
|
57
|
+
const partialPath = path.join(jobDir, "output", "stdout.partial.log");
|
|
58
|
+
// Cancellation travels DOWN to the child process, not up through the
|
|
59
|
+
// iterator. Returning from an async generator that is suspended at an
|
|
60
|
+
// `await` does not take effect until that await settles — which for an
|
|
61
|
+
// agent CLI gone quiet is never — so the only thing that reliably stops a
|
|
62
|
+
// silent run is aborting the subprocess (or fetch) directly.
|
|
63
|
+
const cancelController = new AbortController();
|
|
64
|
+
const events = input.service
|
|
65
|
+
? state.router.streamTo(input.service, input.prompt, files, workingDir, {
|
|
66
|
+
...(hints.safetyProfile !== undefined
|
|
67
|
+
? { safetyProfile: hints.safetyProfile }
|
|
68
|
+
: {}),
|
|
69
|
+
...(hints.workspacePolicy !== undefined
|
|
70
|
+
? { workspacePolicy: hints.workspacePolicy }
|
|
71
|
+
: {}),
|
|
72
|
+
...(hints.routePolicy !== undefined
|
|
73
|
+
? { routePolicy: hints.routePolicy }
|
|
74
|
+
: {}),
|
|
75
|
+
...(hints.model !== undefined ? { model: hints.model } : {}),
|
|
76
|
+
...(hints.taskType !== undefined ? { taskType: hints.taskType } : {}),
|
|
77
|
+
...(hints.timeoutMs !== undefined ? { timeoutMs: hints.timeoutMs } : {}),
|
|
78
|
+
defaultTimeoutMs: JOB_DEFAULT_TIMEOUT_MS,
|
|
79
|
+
signal: cancelController.signal,
|
|
80
|
+
})
|
|
81
|
+
: state.router.stream(input.prompt, files, workingDir, {
|
|
82
|
+
hints,
|
|
83
|
+
maxFallbacks: 2,
|
|
84
|
+
defaultTimeoutMs: JOB_DEFAULT_TIMEOUT_MS,
|
|
85
|
+
signal: cancelController.signal,
|
|
86
|
+
});
|
|
87
|
+
let finalResult = null;
|
|
88
|
+
let finalDecision = null;
|
|
89
|
+
let cancelled = false;
|
|
90
|
+
// Driven through an explicit iterator rather than `for await`, so a
|
|
91
|
+
// cancellation can interrupt a stream that is producing NOTHING. A
|
|
92
|
+
// for-await body only runs when an event arrives, and the case that most
|
|
93
|
+
// needs cancelling is the agent that has gone quiet for twenty minutes.
|
|
94
|
+
// Racing next() against a poll lets us stop either way, and calling
|
|
95
|
+
// return() on the iterator is what tears the child process down —
|
|
96
|
+
// stream-subprocess's return() runs killTree, which on POSIX now signals
|
|
97
|
+
// the whole process group.
|
|
98
|
+
const iterator = events[Symbol.asyncIterator]();
|
|
99
|
+
const CANCEL_POLL_MS = 1_000;
|
|
100
|
+
// The in-flight next() is held ACROSS polls rather than re-issued.
|
|
101
|
+
// Racing a fresh iterator.next() each time round drops events: when the
|
|
102
|
+
// poll wins, the previous next() is still pending, and calling next()
|
|
103
|
+
// again queues a second pull whose result is the one we read — the first
|
|
104
|
+
// event resolves into nothing. Losing a `completion` that way leaves a
|
|
105
|
+
// finished run with no result.json, so the job never reaches a terminal
|
|
106
|
+
// state and the caller polls a corpse. Caught by the slot-queue test,
|
|
107
|
+
// which waits for a queued job to actually complete.
|
|
108
|
+
let pending;
|
|
109
|
+
for (;;) {
|
|
110
|
+
pending ??= iterator.next();
|
|
111
|
+
const winner = await Promise.race([
|
|
112
|
+
pending.then((r) => ({ kind: "event", r })),
|
|
113
|
+
delay(CANCEL_POLL_MS, { kind: "poll" }, { ref: false }),
|
|
114
|
+
]);
|
|
115
|
+
if (winner.kind === "poll") {
|
|
116
|
+
if (!cancelRequested(jobDir))
|
|
117
|
+
continue; // `pending` deliberately kept
|
|
118
|
+
cancelled = true;
|
|
119
|
+
cancelController.abort();
|
|
120
|
+
// Not awaited: the generator is parked on an await that only settles
|
|
121
|
+
// once the abort above kills the child, so awaiting return() here
|
|
122
|
+
// would deadlock on the very thing it is trying to stop.
|
|
123
|
+
void iterator.return?.().catch(() => undefined);
|
|
124
|
+
break;
|
|
125
|
+
}
|
|
126
|
+
pending = undefined;
|
|
127
|
+
const next = winner.r;
|
|
128
|
+
if (next.done)
|
|
129
|
+
break;
|
|
130
|
+
if (cancelRequested(jobDir)) {
|
|
131
|
+
cancelled = true;
|
|
132
|
+
cancelController.abort();
|
|
133
|
+
void iterator.return?.().catch(() => undefined);
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
const { event, decision } = next.value;
|
|
137
|
+
if (decision)
|
|
138
|
+
finalDecision = decision;
|
|
139
|
+
if (input.onEvent) {
|
|
140
|
+
try {
|
|
141
|
+
input.onEvent(event);
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
// Progress forwarding is best-effort; the job itself must not fail.
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (event.type === "stdout" || event.type === "stderr") {
|
|
148
|
+
try {
|
|
149
|
+
await appendFile(partialPath, redact(event.chunk), { encoding: "utf8", mode: 0o600 });
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
// Progress mirroring is best-effort; the final result still lands.
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
else if (event.type === "completion") {
|
|
156
|
+
// Fallback chains yield one completion per attempt; last one wins.
|
|
157
|
+
finalResult = event.result;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (cancelled) {
|
|
161
|
+
// Terminal, and deliberately NOT routed through the result/failure path:
|
|
162
|
+
// no result.json is written and the router never sees a failure, so a
|
|
163
|
+
// cancellation cannot charge the route's breaker or failure count for
|
|
164
|
+
// the caller changing their mind.
|
|
165
|
+
finished = true;
|
|
166
|
+
await pendingBeat;
|
|
167
|
+
const reason = await cancelReason(jobDir);
|
|
168
|
+
await updateStatus(jobDir, {
|
|
169
|
+
jobId: manifest.jobId,
|
|
170
|
+
status: "cancelled",
|
|
171
|
+
createdAt: manifest.createdAt,
|
|
172
|
+
updatedAt: timestamp(),
|
|
173
|
+
jobDir,
|
|
174
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
175
|
+
success: false,
|
|
176
|
+
error: reason !== undefined ? `Cancelled: ${reason}` : "Cancelled before it finished.",
|
|
177
|
+
...(manifest.warning !== undefined ? { warning: manifest.warning } : {}),
|
|
178
|
+
durationMs: Date.now() - started,
|
|
179
|
+
});
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
const result = finalResult ?? {
|
|
183
|
+
output: "",
|
|
184
|
+
service: input.service ?? "none",
|
|
185
|
+
success: false,
|
|
186
|
+
error: "Router stream ended without a completion event",
|
|
187
|
+
};
|
|
188
|
+
finished = true;
|
|
189
|
+
await pendingBeat;
|
|
190
|
+
const payload = {
|
|
191
|
+
jobId: manifest.jobId,
|
|
192
|
+
result: { ...result, ...(result.error !== undefined ? { error: boundedError(result.error) } : {}) },
|
|
193
|
+
decision: finalDecision,
|
|
194
|
+
};
|
|
195
|
+
await writeFile(path.join(jobDir, "output", "stdout.log"), redact(result.output), { encoding: "utf8", mode: 0o600 });
|
|
196
|
+
await writeFile(path.join(jobDir, "output", "stderr.log"), redact(result.error ?? ""), { encoding: "utf8", mode: 0o600 });
|
|
197
|
+
await writeJson(path.join(jobDir, "output", "result.json"), payload);
|
|
198
|
+
await writeFile(path.join(jobDir, "output", "result.md"), redact(result.output || result.error || ""), { encoding: "utf8", mode: 0o600 });
|
|
199
|
+
await updateStatus(jobDir, {
|
|
200
|
+
jobId: manifest.jobId,
|
|
201
|
+
status: result.success ? "completed" : "failed",
|
|
202
|
+
createdAt: manifest.createdAt,
|
|
203
|
+
updatedAt: timestamp(),
|
|
204
|
+
jobDir,
|
|
205
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
206
|
+
route: result.service,
|
|
207
|
+
success: result.success,
|
|
208
|
+
...(result.error !== undefined ? { error: boundedError(result.error) } : {}),
|
|
209
|
+
...(manifest.warning !== undefined ? { warning: manifest.warning } : {}),
|
|
210
|
+
durationMs: Date.now() - started,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
finished = true;
|
|
215
|
+
await pendingBeat;
|
|
216
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
217
|
+
try {
|
|
218
|
+
// The sibling of the success path 25 lines up, which redacts. Missing
|
|
219
|
+
// this one is the same one-branch-of-a-pair miss the chokepoint exists
|
|
220
|
+
// to make impossible, found in the very file the chokepoint edited.
|
|
221
|
+
await writeFile(path.join(jobDir, "output", "stderr.log"), redact(message), {
|
|
222
|
+
encoding: "utf8",
|
|
223
|
+
mode: 0o600,
|
|
224
|
+
});
|
|
225
|
+
await updateStatus(jobDir, {
|
|
226
|
+
jobId: manifest.jobId,
|
|
227
|
+
status: "failed",
|
|
228
|
+
createdAt: manifest.createdAt,
|
|
229
|
+
updatedAt: timestamp(),
|
|
230
|
+
jobDir,
|
|
231
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
232
|
+
success: false,
|
|
233
|
+
error: boundedError(message),
|
|
234
|
+
...(manifest.warning !== undefined ? { warning: manifest.warning } : {}),
|
|
235
|
+
durationMs: Date.now() - started,
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
catch {
|
|
239
|
+
// The job directory can be GONE by the time a failure is recorded —
|
|
240
|
+
// retention pruning, or a caller that tore down its state mid-run.
|
|
241
|
+
// There is nowhere to write and no reader left to care; throwing here
|
|
242
|
+
// would reject `completion`, which is documented to never reject (and
|
|
243
|
+
// surfaced in CI as an unhandled rejection out of a finished test).
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
finally {
|
|
247
|
+
clearInterval(heartbeat);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/** How often a live background run bumps its status file's updatedAt. */
|
|
251
|
+
const HEARTBEAT_INTERVAL_MS = 15_000;
|
|
252
|
+
/**
|
|
253
|
+
* Fallback dispatch timeout for jobs. Dispatchers hard-code a short default
|
|
254
|
+
* (10 min for CLI harnesses, 2 min for openai_compatible) meant to catch a
|
|
255
|
+
* genuinely hung process — waiting on stdin that'll never come, a stalled
|
|
256
|
+
* network call — not to cap a slow-but-healthy run. That default made sense
|
|
257
|
+
* as-is for `code`, which blocks an MCP call anyway, but `job` runs in the
|
|
258
|
+
* background and is polled, so nothing about it requires killing a process
|
|
259
|
+
* that's still making progress after 10 minutes. Below both an explicit
|
|
260
|
+
* `hints.timeoutMs` and the route's own configured `timeoutMs` in
|
|
261
|
+
* precedence, so this only fills the gap when nobody set either.
|
|
262
|
+
*
|
|
263
|
+
* Router.stream() treats this specific value as a budget for the WHOLE call
|
|
264
|
+
* (including router fallback retries), not a per-attempt allowance — without
|
|
265
|
+
* that, a job that falls back twice (the router's default maxFallbacks: 2)
|
|
266
|
+
* could burn up to 3x this value before failing conclusively.
|
|
267
|
+
*/
|
|
268
|
+
export const JOB_DEFAULT_TIMEOUT_MS = 60 * 60 * 1000;
|
|
269
|
+
/**
|
|
270
|
+
* Rebuild a job's input from its on-disk bundle and execute it. This is the
|
|
271
|
+
* detached runner's whole job; the manifest deliberately carries everything
|
|
272
|
+
* a run needs (prompt path, resolved workingDir, hints, service) precisely
|
|
273
|
+
* so execution can happen in a process that wasn't there when the job was
|
|
274
|
+
* created.
|
|
275
|
+
*/
|
|
276
|
+
export async function executeJobDir(deps, jobDir) {
|
|
277
|
+
const manifest = await readJson(path.join(jobDir, "manifest.json"));
|
|
278
|
+
const prompt = await readFile(manifest.promptPath, "utf8");
|
|
279
|
+
const input = {
|
|
280
|
+
prompt,
|
|
281
|
+
files: manifest.files.map((f) => f.originalPath),
|
|
282
|
+
workingDir: manifest.workingDir,
|
|
283
|
+
...(manifest.hints !== undefined ? { hints: manifest.hints } : {}),
|
|
284
|
+
...(manifest.workspacePolicy !== undefined
|
|
285
|
+
? { workspacePolicy: manifest.workspacePolicy }
|
|
286
|
+
: {}),
|
|
287
|
+
...(manifest.service !== undefined ? { service: manifest.service } : {}),
|
|
288
|
+
};
|
|
289
|
+
await runJob(deps, jobDir, manifest, input);
|
|
290
|
+
}
|
|
291
|
+
/** dist/job-runner.js next to this module (compiled), or via the package's dist/ when running from src. */
|
|
292
|
+
export function resolveRunnerPath() {
|
|
293
|
+
// Resolved against THIS module's own location, which is a trap worth
|
|
294
|
+
// stating: the candidates below have to be updated whenever this function
|
|
295
|
+
// moves between directories. Splitting jobs.ts moved it from `dist/` down
|
|
296
|
+
// into `dist/jobs/`, both candidates missed, and the function returned
|
|
297
|
+
// undefined — which is not an error anywhere, it is the signal to run the
|
|
298
|
+
// job IN-PROCESS. So every dispatch quietly stopped being detached and the
|
|
299
|
+
// concurrency gate stopped firing, with nothing failing until the job
|
|
300
|
+
// concurrency tests ran.
|
|
301
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
302
|
+
const candidates = [
|
|
303
|
+
// dist/jobs/run.js -> dist/job-runner.js (the built layout today)
|
|
304
|
+
path.join(here, "..", "job-runner.js"),
|
|
305
|
+
// Beside this file, if the build ever flattens or co-locates it.
|
|
306
|
+
path.join(here, "job-runner.js"),
|
|
307
|
+
// src/jobs/run.ts with a dist/ build present (unbuilt checkout, tests).
|
|
308
|
+
path.join(here, "..", "..", "dist", "job-runner.js"),
|
|
309
|
+
];
|
|
310
|
+
return candidates.find((candidate) => existsSync(candidate));
|
|
311
|
+
}
|
|
312
|
+
const TERMINAL_WATCH_INTERVAL_MS = 300;
|
|
313
|
+
/**
|
|
314
|
+
* Watch a detached job's directory until it reaches a terminal state
|
|
315
|
+
* (result.json present, or a failed/orphaned status — the orphan check
|
|
316
|
+
* doubles as the exit path if the runner dies). Timer is unref'd: an
|
|
317
|
+
* exiting server abandons the watch, which is exactly the point of
|
|
318
|
+
* detached execution.
|
|
319
|
+
*/
|
|
320
|
+
export async function watchUntilTerminal(jobDir) {
|
|
321
|
+
const deadline = Date.now() + JOB_DEFAULT_TIMEOUT_MS + 10 * 60 * 1000;
|
|
322
|
+
while (Date.now() < deadline) {
|
|
323
|
+
// Waits for a terminal STATUS, deliberately not for result.json.
|
|
324
|
+
//
|
|
325
|
+
// runJob writes result.json and then updates the status, so returning on
|
|
326
|
+
// result.json alone let this resolve in the window between the two: a
|
|
327
|
+
// caller could `await` a job and then read `status: "running"` from the
|
|
328
|
+
// job it had just been told was finished. Observed on Windows CI as
|
|
329
|
+
// "expected 'running' to be 'completed'".
|
|
330
|
+
//
|
|
331
|
+
// Because the status write comes last, a terminal status implies the
|
|
332
|
+
// result is already on disk — the ordering does the synchronising, so no
|
|
333
|
+
// extra check is needed here. A runner that dies between the two writes
|
|
334
|
+
// is covered by withOrphanCheck below, which is the same exit path as any
|
|
335
|
+
// other dead runner.
|
|
336
|
+
try {
|
|
337
|
+
const status = withOrphanCheck(await readJson(path.join(jobDir, "status.json")));
|
|
338
|
+
if (status.status === "completed" ||
|
|
339
|
+
status.status === "failed" ||
|
|
340
|
+
status.status === "orphaned" ||
|
|
341
|
+
status.status === "cancelled") {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
// Transient read during an atomic rename — retry next tick.
|
|
347
|
+
}
|
|
348
|
+
await delay(TERMINAL_WATCH_INTERVAL_MS, undefined, { ref: false });
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/jobs/run.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EACL,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,eAAe,EACf,SAAS,GACV,MAAM,YAAY,CAAC;AAEpB,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,IAAa,EACb,MAAc,EACd,QAAqB,EACrB,KAAoB;IAEpB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,aAAa,GAAG,GAAc,EAAE,CAAC,CAAC;QACtC,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,SAAS;QACjB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,SAAS,EAAE,SAAS,EAAE;QACtB,MAAM;QACN,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzE,CAAC,CAAC;IACH,MAAM,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;IAE5C,wEAAwE;IACxE,uEAAuE;IACvE,qEAAqE;IACrE,uEAAuE;IACvE,yEAAyE;IACzE,uEAAuE;IACvE,2EAA2E;IAC3E,sEAAsE;IACtE,yEAAyE;IACzE,yEAAyE;IACzE,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,WAAW,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;QACjC,IAAI,QAAQ;YAAE,OAAO;QACrB,WAAW,GAAG,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7E,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAC1B,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;QAChC,yEAAyE;QACzE,oEAAoE;QACpE,wCAAwC;QACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACvC,MAAM,KAAK,GAAe,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;QACrD,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS;YAAE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC,eAAe,CAAC;QAEvF,yEAAyE;QACzE,oEAAoE;QACpE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,oBAAoB,CAAC,CAAC;QACtE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,0EAA0E;QAC1E,6DAA6D;QAC7D,MAAM,gBAAgB,GAAG,IAAI,eAAe,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO;YAC1B,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE;gBACpE,GAAG,CAAC,KAAK,CAAC,aAAa,KAAK,SAAS;oBACnC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;oBACxC,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS;oBACrC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE;oBAC5C,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS;oBACjC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE;oBACpC,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,GAAG,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxE,gBAAgB,EAAE,sBAAsB;gBACxC,MAAM,EAAE,gBAAgB,CAAC,MAAM;aAChC,CAAC;YACJ,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE;gBACnD,KAAK;gBACL,YAAY,EAAE,CAAC;gBACf,gBAAgB,EAAE,sBAAsB;gBACxC,MAAM,EAAE,gBAAgB,CAAC,MAAM;aAChC,CAAC,CAAC;QAEP,IAAI,WAAW,GAA0B,IAAI,CAAC;QAC9C,IAAI,aAAa,GAA2B,IAAI,CAAC;QACjD,IAAI,SAAS,GAAG,KAAK,CAAC;QAEtB,oEAAoE;QACpE,mEAAmE;QACnE,yEAAyE;QACzE,wEAAwE;QACxE,oEAAoE;QACpE,kEAAkE;QAClE,yEAAyE;QACzE,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAChD,MAAM,cAAc,GAAG,KAAK,CAAC;QAC7B,mEAAmE;QACnE,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,uEAAuE;QACvE,wEAAwE;QACxE,sEAAsE;QACtE,qDAAqD;QACrD,IAAI,OAA2G,CAAC;QAChH,SAAS,CAAC;YACR,OAAO,KAAK,QAAQ,CAAC,IAAI,EAExB,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAChC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAgB,EAAE,CAAC,EAAE,CAAC,CAAC;gBACpD,KAAK,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,MAAe,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;aACjE,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;oBAAE,SAAS,CAAC,8BAA8B;gBACtE,SAAS,GAAG,IAAI,CAAC;gBACjB,gBAAgB,CAAC,KAAK,EAAE,CAAC;gBACzB,qEAAqE;gBACrE,kEAAkE;gBAClE,yDAAyD;gBACzD,KAAK,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAChD,MAAM;YACR,CAAC;YACD,OAAO,GAAG,SAAS,CAAC;YACpB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI;gBAAE,MAAM;YACrB,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,SAAS,GAAG,IAAI,CAAC;gBACjB,gBAAgB,CAAC,KAAK,EAAE,CAAC;gBACzB,KAAK,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAChD,MAAM;YACR,CAAC;YACD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;YACvC,IAAI,QAAQ;gBAAE,aAAa,GAAG,QAAQ,CAAC;YACvC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAClB,IAAI,CAAC;oBACH,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACvB,CAAC;gBAAC,MAAM,CAAC;oBACP,oEAAoE;gBACtE,CAAC;YACH,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,IAAI,CAAC;oBACH,MAAM,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACxF,CAAC;gBAAC,MAAM,CAAC;oBACP,mEAAmE;gBACrE,CAAC;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACvC,mEAAmE;gBACnE,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC;YAC7B,CAAC;QACH,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACd,yEAAyE;YACzE,sEAAsE;YACtE,sEAAsE;YACtE,kCAAkC;YAClC,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,WAAW,CAAC;YAClB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,YAAY,CAAC,MAAM,EAAE;gBACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,SAAS,EAAE,SAAS,EAAE;gBACtB,MAAM;gBACN,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,MAAM,EAAE,CAAC,CAAC,CAAC,+BAA+B;gBACtF,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAmB,WAAW,IAAI;YAC5C,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,MAAM;YAChC,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,gDAAgD;SACxD,CAAC;QAEF,QAAQ,GAAG,IAAI,CAAC;QAChB,MAAM,WAAW,CAAC;QAClB,MAAM,OAAO,GAAqB;YAChC,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;YACpG,QAAQ,EAAE,aAAa;SACxB,CAAC;QACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACrH,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1H,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,aAAa,CAAC,EAAE,OAAO,CAAC,CAAC;QACrE,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,EACxC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,EAC3C,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAClC,CAAC;QACF,MAAM,YAAY,CAAC,MAAM,EAAE;YACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YAC/C,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,SAAS,EAAE,SAAS,EAAE;YACtB,MAAM;YACN,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,KAAK,EAAE,MAAM,CAAC,OAAO;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,GAAG,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7E,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,GAAG,IAAI,CAAC;QAChB,MAAM,WAAW,CAAC;QAClB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,IAAI,CAAC;YACH,sEAAsE;YACtE,uEAAuE;YACvE,oEAAoE;YACpE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE;gBAC1E,QAAQ,EAAE,MAAM;gBAChB,IAAI,EAAE,KAAK;aACZ,CAAC,CAAC;YACH,MAAM,YAAY,CAAC,MAAM,EAAE;gBACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,SAAS,EAAE,SAAS,EAAE;gBACtB,MAAM;gBACN,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,YAAY,CAAC,OAAO,CAAE;gBAC7B,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,oEAAoE;YACpE,mEAAmE;YACnE,sEAAsE;YACtE,sEAAsE;YACtE,oEAAoE;QACtE,CAAC;IACH,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAGD,yEAAyE;AACzE,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAGrD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAa,EAAE,MAAc;IAC/D,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAc,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC;IACjF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAkB;QAC3B,MAAM;QACN,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;QAChD,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS;YACxC,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE;YAC/C,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzE,CAAC;IACF,MAAM,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AAC9C,CAAC;AAED,2GAA2G;AAC3G,MAAM,UAAU,iBAAiB;IAC/B,qEAAqE;IACrE,0EAA0E;IAC1E,0EAA0E;IAC1E,uEAAuE;IACvE,0EAA0E;IAC1E,2EAA2E;IAC3E,sEAAsE;IACtE,yBAAyB;IACzB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG;QACjB,kEAAkE;QAClE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,eAAe,CAAC;QACtC,iEAAiE;QACjE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC;QAChC,wEAAwE;QACxE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,CAAC;KACrD,CAAC;IACF,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAEvC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,sBAAsB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACtE,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,iEAAiE;QACjE,EAAE;QACF,yEAAyE;QACzE,sEAAsE;QACtE,wEAAwE;QACxE,oEAAoE;QACpE,0CAA0C;QAC1C,EAAE;QACF,qEAAqE;QACrE,yEAAyE;QACzE,wEAAwE;QACxE,0EAA0E;QAC1E,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,eAAe,CAC5B,MAAM,QAAQ,CAAY,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAC5D,CAAC;YACF,IACE,MAAM,CAAC,MAAM,KAAK,WAAW;gBAC7B,MAAM,CAAC,MAAM,KAAK,QAAQ;gBAC1B,MAAM,CAAC,MAAM,KAAK,UAAU;gBAC5B,MAAM,CAAC,MAAM,KAAK,WAAW,EAC7B,CAAC;gBACD,OAAO;YACT,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4DAA4D;QAC9D,CAAC;QACD,MAAM,KAAK,CAAC,0BAA0B,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Starting a job.
|
|
3
|
+
*
|
|
4
|
+
* Split out of jobs.ts with the rest of its concerns; nothing was rewritten.
|
|
5
|
+
*/
|
|
6
|
+
import type { JobDeps, JobStatus, StartedJob, StartJobInput } from "./types.js";
|
|
7
|
+
export declare function startAsyncJob(deps: JobDeps, input: StartJobInput): Promise<JobStatus>;
|
|
8
|
+
export declare function startAsyncJobTracked(deps: JobDeps, input: StartJobInput): Promise<StartedJob>;
|
|
9
|
+
//# sourceMappingURL=start.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"start.d.ts","sourceRoot":"","sources":["../../src/jobs/start.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAmBH,OAAO,KAAK,EAAE,OAAO,EAAe,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,wBAAsB,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC,CAE3F;AAED,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CAmHnG"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Starting a job.
|
|
3
|
+
*
|
|
4
|
+
* Split out of jobs.ts with the rest of its concerns; nothing was rewritten.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
7
|
+
import { pruneDeadWorkspaceLocks } from "../workspace-lock.js";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { resolveWorkingDir, validateWorkingDir, workingDirWarning } from "../working-dir.js";
|
|
10
|
+
import { buildContextPreamble } from "./context.js";
|
|
11
|
+
import { jobsRoot, newJobId, pollInstructions, pruneStaleJobs, readJson, snapshotFiles, SUGGESTED_POLL_SECONDS, timestamp, updateStatus, writeJson, } from "./store.js";
|
|
12
|
+
import { resolveRunnerPath, runJob, watchUntilTerminal } from "./run.js";
|
|
13
|
+
import { configLoadError, drainSlotQueue, } from "./supervisor.js";
|
|
14
|
+
export async function startAsyncJob(deps, input) {
|
|
15
|
+
return (await startAsyncJobTracked(deps, input)).status;
|
|
16
|
+
}
|
|
17
|
+
export async function startAsyncJobTracked(deps, input) {
|
|
18
|
+
// Before anything is created on disk. Every dispatch path — MCP, HTTP,
|
|
19
|
+
// fanout — funnels through here, so this is the one place that catches a bad
|
|
20
|
+
// workingDir while the error can still name the real cause, and the only
|
|
21
|
+
// point at which failing leaves no half-built job directory behind.
|
|
22
|
+
const workingDirError = validateWorkingDir(input.workingDir);
|
|
23
|
+
if (workingDirError !== undefined)
|
|
24
|
+
throw new Error(workingDirError);
|
|
25
|
+
// The runner reads the config FILE, so a file this server can no longer load
|
|
26
|
+
// means no runner can start — and the job would sit untouched until the 90s
|
|
27
|
+
// orphan threshold reported it dead. Observed: a caller told
|
|
28
|
+
// "ended without a result (status: orphaned)" about a job whose own
|
|
29
|
+
// status.json later read completed/success. Two false statements from one
|
|
30
|
+
// broken file, ninety seconds apart.
|
|
31
|
+
//
|
|
32
|
+
// The server itself is fine: a failed hot-reload keeps the previous config
|
|
33
|
+
// in memory, which is why it can still accept the dispatch at all. That
|
|
34
|
+
// divergence between what the server runs and what the runner would read is
|
|
35
|
+
// the whole bug, so it is refused here, immediately, naming the real cause —
|
|
36
|
+
// before a job directory exists to be misreported.
|
|
37
|
+
const configError = await configLoadError(deps.holder.state.configPath);
|
|
38
|
+
if (configError !== undefined)
|
|
39
|
+
throw new Error(configError);
|
|
40
|
+
await pruneStaleJobs();
|
|
41
|
+
// Same maintenance moment, same reason: a lock whose holder is gone is only
|
|
42
|
+
// reclaimed when something contends for that exact directory, so a
|
|
43
|
+
// dispatched-once workspace leaves its file behind forever. Best effort, and
|
|
44
|
+
// deliberately not awaited-into-failure — a sweep must never block the job
|
|
45
|
+
// that was actually asked for.
|
|
46
|
+
await pruneDeadWorkspaceLocks().catch(() => undefined);
|
|
47
|
+
const jobId = newJobId();
|
|
48
|
+
const root = jobsRoot();
|
|
49
|
+
const jobDir = path.join(root, jobId);
|
|
50
|
+
await mkdir(path.join(jobDir, "context"), { recursive: true, mode: 0o700 });
|
|
51
|
+
await mkdir(path.join(jobDir, "output"), { recursive: true, mode: 0o700 });
|
|
52
|
+
const promptPath = path.join(jobDir, "prompt.md");
|
|
53
|
+
// Prepend prior-job context before the prompt is frozen to disk, so the
|
|
54
|
+
// runner, the manifest and any later inspection all see exactly what the
|
|
55
|
+
// delegate was given.
|
|
56
|
+
const preamble = await buildContextPreamble(input.contextJobs ?? []);
|
|
57
|
+
const effectivePrompt = preamble + input.prompt;
|
|
58
|
+
await writeFile(promptPath, effectivePrompt, { encoding: "utf8", mode: 0o600 });
|
|
59
|
+
const fileSnapshots = await snapshotFiles(jobDir, input.files ?? []);
|
|
60
|
+
const createdAt = timestamp();
|
|
61
|
+
const resolvedWorkingDir = resolveWorkingDir(input.workingDir);
|
|
62
|
+
const warning = workingDirWarning(resolvedWorkingDir);
|
|
63
|
+
const manifest = {
|
|
64
|
+
jobId,
|
|
65
|
+
createdAt,
|
|
66
|
+
workingDir: resolvedWorkingDir.workingDir,
|
|
67
|
+
promptPath,
|
|
68
|
+
files: fileSnapshots,
|
|
69
|
+
...(input.hints !== undefined ? { hints: input.hints } : {}),
|
|
70
|
+
...(input.workspacePolicy !== undefined ? { workspacePolicy: input.workspacePolicy } : {}),
|
|
71
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
72
|
+
...(input.retryOf !== undefined ? { retryOf: input.retryOf } : {}),
|
|
73
|
+
...(warning !== undefined ? { warning } : {}),
|
|
74
|
+
};
|
|
75
|
+
await writeJson(path.join(jobDir, "manifest.json"), manifest);
|
|
76
|
+
const status = {
|
|
77
|
+
jobId,
|
|
78
|
+
status: "queued",
|
|
79
|
+
createdAt,
|
|
80
|
+
updatedAt: createdAt,
|
|
81
|
+
jobDir,
|
|
82
|
+
...(input.service !== undefined ? { service: input.service } : {}),
|
|
83
|
+
nextPollSeconds: SUGGESTED_POLL_SECONDS,
|
|
84
|
+
instructions: pollInstructions(jobId),
|
|
85
|
+
...(warning !== undefined ? { warning } : {}),
|
|
86
|
+
};
|
|
87
|
+
await updateStatus(jobDir, status);
|
|
88
|
+
// Detached by default: the run must not die with this process. In-process
|
|
89
|
+
// mode exists for unit tests (injected fake dispatchers aren't visible to
|
|
90
|
+
// a separate process) and as the fallback when the runner script can't be
|
|
91
|
+
// found (running from raw src/ with no dist/ build).
|
|
92
|
+
const inproc = process.env.HARNESS_DISPATCH_INPROC_JOBS === "1";
|
|
93
|
+
const runnerPath = inproc ? undefined : resolveRunnerPath();
|
|
94
|
+
if (inproc || runnerPath === undefined) {
|
|
95
|
+
if (!inproc) {
|
|
96
|
+
console.error("harness-dispatch: dist/job-runner.js not found (unbuilt checkout?) — " +
|
|
97
|
+
"running the job in-process; it will not survive a server restart.");
|
|
98
|
+
}
|
|
99
|
+
// The DETACHED runner re-reads prompt.md, which carries the context
|
|
100
|
+
// preamble — so the in-process run must dispatch the same frozen prompt,
|
|
101
|
+
// not input.prompt. Passing the raw prompt here silently dropped
|
|
102
|
+
// contextJobs for every in-process run (unit tests with injected fakes,
|
|
103
|
+
// and the unbuilt-checkout fallback).
|
|
104
|
+
const completion = runJob(deps, jobDir, manifest, { ...input, prompt: effectivePrompt });
|
|
105
|
+
return { status, completion };
|
|
106
|
+
}
|
|
107
|
+
// Concurrency gate. Every dispatch spawns its own detached runner, so an
|
|
108
|
+
// in-process semaphore would bound nothing — the count has to come off
|
|
109
|
+
// disk. The caller still gets its jobId back immediately either way, so the
|
|
110
|
+
// API contract is unchanged and only the start time can move.
|
|
111
|
+
// Every dispatch takes this path, including an uncapped one. It used to
|
|
112
|
+
// spawn its own detached runner when the cap was lifted, which bypassed the
|
|
113
|
+
// pool and put a ~76 MB wrapper behind every job.
|
|
114
|
+
// Enqueue first, then let drainSlotQueue decide — rather than testing the
|
|
115
|
+
// limit here and spawning inline. Two reasons, both learned the hard way:
|
|
116
|
+
// this job's own `queued` status is already on disk, so an inline count
|
|
117
|
+
// included itself and deadlocked at limit 1; and a fresh dispatch arriving
|
|
118
|
+
// while others wait must not jump the queue, which only one FIFO drainer
|
|
119
|
+
// can guarantee. Whether this job starts now is then just "did the drain
|
|
120
|
+
// reach it".
|
|
121
|
+
await updateStatus(jobDir, { ...status, slotQueued: true });
|
|
122
|
+
await drainSlotQueue(deps.holder.state.config, deps.holder.state.configPath);
|
|
123
|
+
const settled = await readJson(path.join(jobDir, "status.json"));
|
|
124
|
+
return { status: settled, completion: watchUntilTerminal(jobDir) };
|
|
125
|
+
}
|
|
126
|
+
//# sourceMappingURL=start.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"start.js","sourceRoot":"","sources":["../../src/jobs/start.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,cAAc,EACd,QAAQ,EACR,aAAa,EACb,sBAAsB,EACtB,SAAS,EACT,YAAY,EACZ,SAAS,GACV,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACzE,OAAO,EACL,eAAe,EACf,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAa,EAAE,KAAoB;IACrE,OAAO,CAAC,MAAM,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAa,EAAE,KAAoB;IAC5E,uEAAuE;IACvE,6EAA6E;IAC7E,yEAAyE;IACzE,oEAAoE;IACpE,MAAM,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7D,IAAI,eAAe,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;IAEpE,6EAA6E;IAC7E,4EAA4E;IAC5E,6DAA6D;IAC7D,oEAAoE;IACpE,0EAA0E;IAC1E,qCAAqC;IACrC,EAAE;IACF,2EAA2E;IAC3E,wEAAwE;IACxE,4EAA4E;IAC5E,6EAA6E;IAC7E,mDAAmD;IACnD,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACxE,IAAI,WAAW,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;IAE5D,MAAM,cAAc,EAAE,CAAC;IACvB,4EAA4E;IAC5E,mEAAmE;IACnE,6EAA6E;IAC7E,2EAA2E;IAC3E,+BAA+B;IAC/B,MAAM,uBAAuB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5E,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClD,wEAAwE;IACxE,yEAAyE;IACzE,sBAAsB;IACtB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC;IAChD,MAAM,SAAS,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChF,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;IAC9B,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAgB;QAC5B,KAAK;QACL,SAAS;QACT,UAAU,EAAE,kBAAkB,CAAC,UAAU;QACzC,UAAU;QACV,KAAK,EAAE,aAAa;QACpB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAc;QACxB,KAAK;QACL,MAAM,EAAE,QAAQ;QAChB,SAAS;QACT,SAAS,EAAE,SAAS;QACpB,MAAM;QACN,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,eAAe,EAAE,sBAAsB;QACvC,YAAY,EAAE,gBAAgB,CAAC,KAAK,CAAC;QACrC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnC,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,qDAAqD;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,GAAG,CAAC;IAChE,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC;IAC5D,IAAI,MAAM,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CACX,uEAAuE;gBACrE,mEAAmE,CACtE,CAAC;QACJ,CAAC;QACD,oEAAoE;QACpE,yEAAyE;QACzE,iEAAiE;QACjE,wEAAwE;QACxE,sCAAsC;QACtC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;QACzF,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,4EAA4E;IAC5E,8DAA8D;IAC9D,wEAAwE;IACxE,4EAA4E;IAC5E,kDAAkD;IAElD,0EAA0E;IAC1E,0EAA0E;IAC1E,wEAAwE;IACxE,2EAA2E;IAC3E,yEAAyE;IACzE,yEAAyE;IACzE,aAAa;IACb,MAAM,YAAY,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAY,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAC5E,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;AACrE,CAAC"}
|
package/dist/jobs/store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAUH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AASzD,iEAAiE;AACjE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,QAAS,CAAC;AAE1C;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAwB5D;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAMD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAKlE;AAED,wBAAgB,WAAW,IAAI,MAAM,CAKpC;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAmEpD;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/E;AAgBD,wBAAsB,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAE9D;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBnF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CA4BlG;AAED;;;GAGG;AACH,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAED,eAAO,MAAM,SAAS,QAA0B,CAAC;AAEjD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAMpD;AAmBD,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMlF;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEvD;AAED,mEAAmE;AACnE,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAQ9E"}
|
package/dist/jobs/store.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* Windows EPERM.
|
|
13
13
|
*/
|
|
14
14
|
import { existsSync } from "node:fs";
|
|
15
|
+
import { redact } from "../redaction.js";
|
|
15
16
|
import { chmod, copyFile, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
16
17
|
import { randomUUID } from "node:crypto";
|
|
17
18
|
import path from "node:path";
|
|
@@ -53,8 +54,16 @@ export function withOrphanCheck(status) {
|
|
|
53
54
|
...status,
|
|
54
55
|
status: "orphaned",
|
|
55
56
|
success: false,
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
// Says what is KNOWN, not which process died. This used to read "The
|
|
58
|
+
// dispatch server that started this job exited" — and a load test killed
|
|
59
|
+
// a single supervisor while the server stayed up and serving, producing
|
|
60
|
+
// that message verbatim. The status file this is derived from records a
|
|
61
|
+
// heartbeat and nothing about who was holding it, so naming a culprit was
|
|
62
|
+
// always a guess, and it sent anyone debugging to the wrong process.
|
|
63
|
+
error: "This job stopped reporting progress and the process running it is gone — " +
|
|
64
|
+
"either the run crashed or whatever was supervising it died. Nothing will " +
|
|
65
|
+
"advance it now. Its partial output is on disk; `retry_job` re-runs the same " +
|
|
66
|
+
"task, or re-dispatch it.",
|
|
58
67
|
};
|
|
59
68
|
}
|
|
60
69
|
export function boundedError(error) {
|
|
@@ -185,7 +194,8 @@ export function safeBaseName(filePath) {
|
|
|
185
194
|
}
|
|
186
195
|
export async function writeJson(filePath, value) {
|
|
187
196
|
const tmpPath = `${filePath}.${process.pid}.${Date.now()}.${randomUUID().slice(0, 8)}.tmp`;
|
|
188
|
-
|
|
197
|
+
// Sink: every job JSON on disk — status.json, result.json, manifest.json.
|
|
198
|
+
await writeFile(tmpPath, `${redact(JSON.stringify(value, null, 2))}\n`, { encoding: "utf8", mode: 0o600 });
|
|
189
199
|
await renameWithRetry(tmpPath, filePath);
|
|
190
200
|
}
|
|
191
201
|
async function renameWithRetry(tmpPath, filePath) {
|
package/dist/jobs/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1G,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,iEAAiE;AACjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,2EAA2E;IAC3E,2EAA2E;IAC3E,oEAAoE;IACpE,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,mBAAmB;QAAE,OAAO,MAAM,CAAC;IACrF,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,UAAU;QAClB,OAAO,EAAE,KAAK;QACd,KAAK,EACH,
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1G,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,iEAAiE;AACjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,2EAA2E;IAC3E,2EAA2E;IAC3E,oEAAoE;IACpE,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,mBAAmB;QAAE,OAAO,MAAM,CAAC;IACrF,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,UAAU;QAClB,OAAO,EAAE,KAAK;QACd,qEAAqE;QACrE,yEAAyE;QACzE,wEAAwE;QACxE,wEAAwE;QACxE,0EAA0E;QAC1E,qEAAqE;QACrE,KAAK,EACH,2EAA2E;YAC3E,2EAA2E;YAC3E,8EAA8E;YAC9E,0BAA0B;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAyB;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,IAAI,oBAAoB;QAAE,OAAO,KAAK,CAAC;IACvD,OAAO,CACL,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACpC,iBAAiB,KAAK,CAAC,MAAM,GAAG,oBAAoB,0CAA0C,CAC/F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,CACL,yEAAyE;QACzE,SAAS,IAAI,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,CAAC,mCAAmC;QACnF,yBAAyB,KAAK,mDAAmD;QACjF,iFAAiF;QACjF,0CAA0C,CAC3C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvD,IAAI,qBAAyC,CAAC;AAE9C;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAwB;IAC1D,qBAAqB;QACnB,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;YACtD,CAAC,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;YAC5B,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IACxD,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;AACzD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,sEAAsE;IACtE,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,gDAAgD;IAChD,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO;IAC3B,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,0EAA0E;QAC1E,uDAAuD;QACvD,0EAA0E;QAC1E,0EAA0E;QAC1E,6DAA6D;QAC7D,iEAAiE;QACjE,EAAE;QACF,wEAAwE;QACxE,0EAA0E;QAC1E,wEAAwE;QACxE,yEAAyE;QACzE,mDAAmD;QACnD,EAAE;QACF,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,yEAAyE;QACzE,mDAAmD;QACnD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,QAAQ;gBAAE,SAAS;YAC7C,4DAA4D;YAC5D,sEAAsE;YACtE,wEAAwE;YACxE,uEAAuE;YACvE,0DAA0D;YAC1D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CACd,CAAC;gBAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;gBAChD,IACE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC;oBAC3D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACrB,GAAG,GAAG,IAAI,IAAI,mBAAmB,EACjC,CAAC;oBACD,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;YACD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;QAC3E,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IAC9D,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3F,0EAA0E;IAC1E,MAAM,SAAS,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3G,MAAM,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,QAAgB;IAC9D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACpG,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO;gBAAE,MAAM,GAAG,CAAC;YACzE,MAAM,KAAK,CAAC,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAI,QAAgB;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAM,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,MAAiB;IAClE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE;YAChD,GAAG,MAAM;YACT,SAAS,EAAE,SAAS,EAAE;SACvB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,wEAAwE;QACxE,EAAE;QACF,qEAAqE;QACrE,yEAAyE;QACzE,uEAAuE;QACvE,sEAAsE;QACtE,uEAAuE;QACvE,sEAAsE;QACtE,oDAAoD;QACpD,EAAE;QACF,yEAAyE;QACzE,sEAAsE;QACtE,wEAAwE;QACxE,MAAM,IAAI,GACR,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QACzF,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO;QACrD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,KAAe;IACjE,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAiC,EAAE,YAAY,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,oBAAoB,CAAC;gBAClC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACf,SAAS;YACX,CAAC;YACD,MAAM,YAAY,GAAG,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3F,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YACvD,MAAM,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;YAC3C,MAAM,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;YACjC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC;IACjE,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAEjD;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,4CAA4C,CACnF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,aAAa,GAAG,aAAa,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,MAAe;IACjE,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAChC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAChF,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAClC,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;QACvD,OAAO,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
|