harness-dispatch 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/CHANGELOG.md +1074 -2
  2. package/README.md +92 -36
  3. package/config.default.yaml +19 -3
  4. package/dist/auth.d.ts.map +1 -1
  5. package/dist/auth.js.map +1 -1
  6. package/dist/billing.d.ts +24 -0
  7. package/dist/billing.d.ts.map +1 -1
  8. package/dist/billing.js +24 -0
  9. package/dist/billing.js.map +1 -1
  10. package/dist/bin.d.ts.map +1 -1
  11. package/dist/bin.js +299 -48
  12. package/dist/bin.js.map +1 -1
  13. package/dist/circuit-breaker.d.ts +9 -1
  14. package/dist/circuit-breaker.d.ts.map +1 -1
  15. package/dist/circuit-breaker.js +15 -1
  16. package/dist/circuit-breaker.js.map +1 -1
  17. package/dist/client-register.d.ts +11 -1
  18. package/dist/client-register.d.ts.map +1 -1
  19. package/dist/client-register.js +15 -6
  20. package/dist/client-register.js.map +1 -1
  21. package/dist/config/coercions.d.ts +5 -0
  22. package/dist/config/coercions.d.ts.map +1 -1
  23. package/dist/config/coercions.js +8 -0
  24. package/dist/config/coercions.js.map +1 -1
  25. package/dist/config/protocol.js.map +1 -1
  26. package/dist/config/validation.d.ts +12 -0
  27. package/dist/config/validation.d.ts.map +1 -1
  28. package/dist/config/validation.js +133 -7
  29. package/dist/config/validation.js.map +1 -1
  30. package/dist/config.d.ts +15 -5
  31. package/dist/config.d.ts.map +1 -1
  32. package/dist/config.js +199 -152
  33. package/dist/config.js.map +1 -1
  34. package/dist/configure-yaml.d.ts +2 -0
  35. package/dist/configure-yaml.d.ts.map +1 -1
  36. package/dist/configure-yaml.js +114 -4
  37. package/dist/configure-yaml.js.map +1 -1
  38. package/dist/dispatch-log.d.ts.map +1 -1
  39. package/dist/dispatch-log.js +3 -1
  40. package/dist/dispatch-log.js.map +1 -1
  41. package/dist/dispatchers/generic-cli.d.ts.map +1 -1
  42. package/dist/dispatchers/generic-cli.js +281 -112
  43. package/dist/dispatchers/generic-cli.js.map +1 -1
  44. package/dist/dispatchers/openai-compatible.d.ts +0 -5
  45. package/dist/dispatchers/openai-compatible.d.ts.map +1 -1
  46. package/dist/dispatchers/openai-compatible.js +150 -83
  47. package/dist/dispatchers/openai-compatible.js.map +1 -1
  48. package/dist/dispatchers/shared/harness-login.d.ts +25 -0
  49. package/dist/dispatchers/shared/harness-login.d.ts.map +1 -0
  50. package/dist/dispatchers/shared/harness-login.js +73 -0
  51. package/dist/dispatchers/shared/harness-login.js.map +1 -0
  52. package/dist/dispatchers/shared/stream-subprocess.d.ts.map +1 -1
  53. package/dist/dispatchers/shared/stream-subprocess.js +71 -58
  54. package/dist/dispatchers/shared/stream-subprocess.js.map +1 -1
  55. package/dist/file-lock.d.ts +29 -1
  56. package/dist/file-lock.d.ts.map +1 -1
  57. package/dist/file-lock.js +53 -20
  58. package/dist/file-lock.js.map +1 -1
  59. package/dist/harness-presets.d.ts +51 -0
  60. package/dist/harness-presets.d.ts.map +1 -0
  61. package/dist/harness-presets.js +154 -0
  62. package/dist/harness-presets.js.map +1 -0
  63. package/dist/http/parse.d.ts +8 -1
  64. package/dist/http/parse.d.ts.map +1 -1
  65. package/dist/http/parse.js +10 -3
  66. package/dist/http/parse.js.map +1 -1
  67. package/dist/http/server.d.ts +2 -0
  68. package/dist/http/server.d.ts.map +1 -1
  69. package/dist/http/server.js +188 -16
  70. package/dist/http/server.js.map +1 -1
  71. package/dist/job-runner.js +6 -0
  72. package/dist/job-runner.js.map +1 -1
  73. package/dist/jobs/context.d.ts +0 -7
  74. package/dist/jobs/context.d.ts.map +1 -1
  75. package/dist/jobs/context.js +95 -7
  76. package/dist/jobs/context.js.map +1 -1
  77. package/dist/jobs/lifecycle.d.ts +100 -0
  78. package/dist/jobs/lifecycle.d.ts.map +1 -0
  79. package/dist/jobs/lifecycle.js +263 -0
  80. package/dist/jobs/lifecycle.js.map +1 -0
  81. package/dist/jobs/read.d.ts +17 -0
  82. package/dist/jobs/read.d.ts.map +1 -0
  83. package/dist/jobs/read.js +103 -0
  84. package/dist/jobs/read.js.map +1 -0
  85. package/dist/jobs/run.d.ts +45 -0
  86. package/dist/jobs/run.d.ts.map +1 -0
  87. package/dist/jobs/run.js +351 -0
  88. package/dist/jobs/run.js.map +1 -0
  89. package/dist/jobs/start.d.ts +9 -0
  90. package/dist/jobs/start.d.ts.map +1 -0
  91. package/dist/jobs/start.js +126 -0
  92. package/dist/jobs/start.js.map +1 -0
  93. package/dist/jobs/store.d.ts.map +1 -1
  94. package/dist/jobs/store.js +13 -3
  95. package/dist/jobs/store.js.map +1 -1
  96. package/dist/jobs/supervisor.d.ts +151 -0
  97. package/dist/jobs/supervisor.d.ts.map +1 -0
  98. package/dist/jobs/supervisor.js +679 -0
  99. package/dist/jobs/supervisor.js.map +1 -0
  100. package/dist/jobs/types.d.ts +8 -1
  101. package/dist/jobs/types.d.ts.map +1 -1
  102. package/dist/jobs.d.ts +15 -218
  103. package/dist/jobs.d.ts.map +1 -1
  104. package/dist/jobs.js +14 -1317
  105. package/dist/jobs.js.map +1 -1
  106. package/dist/leaderboard.d.ts.map +1 -1
  107. package/dist/leaderboard.js +44 -28
  108. package/dist/leaderboard.js.map +1 -1
  109. package/dist/mcp/config-hot-reload.d.ts +14 -3
  110. package/dist/mcp/config-hot-reload.d.ts.map +1 -1
  111. package/dist/mcp/config-hot-reload.js +27 -6
  112. package/dist/mcp/config-hot-reload.js.map +1 -1
  113. package/dist/mcp/dispatcher-factory.d.ts.map +1 -1
  114. package/dist/mcp/dispatcher-factory.js +15 -4
  115. package/dist/mcp/dispatcher-factory.js.map +1 -1
  116. package/dist/mcp/resources.d.ts.map +1 -1
  117. package/dist/mcp/resources.js +3 -2
  118. package/dist/mcp/resources.js.map +1 -1
  119. package/dist/mcp/tool-schemas.d.ts +1 -1
  120. package/dist/mcp/tool-schemas.d.ts.map +1 -1
  121. package/dist/mcp/tool-schemas.js +6 -3
  122. package/dist/mcp/tool-schemas.js.map +1 -1
  123. package/dist/mcp/tools.d.ts +20 -6
  124. package/dist/mcp/tools.d.ts.map +1 -1
  125. package/dist/mcp/tools.js +92 -14
  126. package/dist/mcp/tools.js.map +1 -1
  127. package/dist/mcp-clients.d.ts +9 -0
  128. package/dist/mcp-clients.d.ts.map +1 -1
  129. package/dist/mcp-clients.js +4 -0
  130. package/dist/mcp-clients.js.map +1 -1
  131. package/dist/observability/spans.d.ts.map +1 -1
  132. package/dist/observability/spans.js +10 -2
  133. package/dist/observability/spans.js.map +1 -1
  134. package/dist/quota.d.ts.map +1 -1
  135. package/dist/quota.js +16 -2
  136. package/dist/quota.js.map +1 -1
  137. package/dist/redaction.d.ts +90 -0
  138. package/dist/redaction.d.ts.map +1 -0
  139. package/dist/redaction.js +221 -0
  140. package/dist/redaction.js.map +1 -0
  141. package/dist/route-policy.d.ts +2 -1
  142. package/dist/route-policy.d.ts.map +1 -1
  143. package/dist/route-policy.js +54 -0
  144. package/dist/route-policy.js.map +1 -1
  145. package/dist/router.d.ts +12 -23
  146. package/dist/router.d.ts.map +1 -1
  147. package/dist/router.js +86 -223
  148. package/dist/router.js.map +1 -1
  149. package/dist/safety.d.ts.map +1 -1
  150. package/dist/safety.js +7 -1
  151. package/dist/safety.js.map +1 -1
  152. package/dist/state-dir.d.ts +10 -0
  153. package/dist/state-dir.d.ts.map +1 -1
  154. package/dist/state-dir.js +27 -1
  155. package/dist/state-dir.js.map +1 -1
  156. package/dist/status.d.ts +14 -3
  157. package/dist/status.d.ts.map +1 -1
  158. package/dist/status.js +36 -3
  159. package/dist/status.js.map +1 -1
  160. package/dist/types.d.ts +24 -1
  161. package/dist/types.d.ts.map +1 -1
  162. package/dist/workspace-lock.d.ts +20 -0
  163. package/dist/workspace-lock.d.ts.map +1 -1
  164. package/dist/workspace-lock.js +63 -0
  165. package/dist/workspace-lock.js.map +1 -1
  166. package/dist/workspace-resolve.d.ts +0 -9
  167. package/dist/workspace-resolve.d.ts.map +1 -1
  168. package/dist/workspace-resolve.js +115 -183
  169. package/dist/workspace-resolve.js.map +1 -1
  170. package/dist/workspaces.d.ts +15 -0
  171. package/dist/workspaces.d.ts.map +1 -1
  172. package/dist/workspaces.js +330 -90
  173. package/dist/workspaces.js.map +1 -1
  174. package/package.json +5 -4
@@ -0,0 +1,126 @@
1
+ /**
2
+ * Starting a job.
3
+ *
4
+ * Split out of jobs.ts with the rest of its concerns; nothing was rewritten.
5
+ */
6
+ import { mkdir, writeFile } from "node:fs/promises";
7
+ import { pruneDeadWorkspaceLocks } from "../workspace-lock.js";
8
+ import path from "node:path";
9
+ import { resolveWorkingDir, validateWorkingDir, workingDirWarning } from "../working-dir.js";
10
+ import { buildContextPreamble } from "./context.js";
11
+ import { jobsRoot, newJobId, pollInstructions, pruneStaleJobs, readJson, snapshotFiles, SUGGESTED_POLL_SECONDS, timestamp, updateStatus, writeJson, } from "./store.js";
12
+ import { resolveRunnerPath, runJob, watchUntilTerminal } from "./run.js";
13
+ import { configLoadError, drainSlotQueue, } from "./supervisor.js";
14
+ export async function startAsyncJob(deps, input) {
15
+ return (await startAsyncJobTracked(deps, input)).status;
16
+ }
17
+ export async function startAsyncJobTracked(deps, input) {
18
+ // Before anything is created on disk. Every dispatch path — MCP, HTTP,
19
+ // fanout — funnels through here, so this is the one place that catches a bad
20
+ // workingDir while the error can still name the real cause, and the only
21
+ // point at which failing leaves no half-built job directory behind.
22
+ const workingDirError = validateWorkingDir(input.workingDir);
23
+ if (workingDirError !== undefined)
24
+ throw new Error(workingDirError);
25
+ // The runner reads the config FILE, so a file this server can no longer load
26
+ // means no runner can start — and the job would sit untouched until the 90s
27
+ // orphan threshold reported it dead. Observed: a caller told
28
+ // "ended without a result (status: orphaned)" about a job whose own
29
+ // status.json later read completed/success. Two false statements from one
30
+ // broken file, ninety seconds apart.
31
+ //
32
+ // The server itself is fine: a failed hot-reload keeps the previous config
33
+ // in memory, which is why it can still accept the dispatch at all. That
34
+ // divergence between what the server runs and what the runner would read is
35
+ // the whole bug, so it is refused here, immediately, naming the real cause —
36
+ // before a job directory exists to be misreported.
37
+ const configError = await configLoadError(deps.holder.state.configPath);
38
+ if (configError !== undefined)
39
+ throw new Error(configError);
40
+ await pruneStaleJobs();
41
+ // Same maintenance moment, same reason: a lock whose holder is gone is only
42
+ // reclaimed when something contends for that exact directory, so a
43
+ // dispatched-once workspace leaves its file behind forever. Best effort, and
44
+ // deliberately not awaited-into-failure — a sweep must never block the job
45
+ // that was actually asked for.
46
+ await pruneDeadWorkspaceLocks().catch(() => undefined);
47
+ const jobId = newJobId();
48
+ const root = jobsRoot();
49
+ const jobDir = path.join(root, jobId);
50
+ await mkdir(path.join(jobDir, "context"), { recursive: true, mode: 0o700 });
51
+ await mkdir(path.join(jobDir, "output"), { recursive: true, mode: 0o700 });
52
+ const promptPath = path.join(jobDir, "prompt.md");
53
+ // Prepend prior-job context before the prompt is frozen to disk, so the
54
+ // runner, the manifest and any later inspection all see exactly what the
55
+ // delegate was given.
56
+ const preamble = await buildContextPreamble(input.contextJobs ?? []);
57
+ const effectivePrompt = preamble + input.prompt;
58
+ await writeFile(promptPath, effectivePrompt, { encoding: "utf8", mode: 0o600 });
59
+ const fileSnapshots = await snapshotFiles(jobDir, input.files ?? []);
60
+ const createdAt = timestamp();
61
+ const resolvedWorkingDir = resolveWorkingDir(input.workingDir);
62
+ const warning = workingDirWarning(resolvedWorkingDir);
63
+ const manifest = {
64
+ jobId,
65
+ createdAt,
66
+ workingDir: resolvedWorkingDir.workingDir,
67
+ promptPath,
68
+ files: fileSnapshots,
69
+ ...(input.hints !== undefined ? { hints: input.hints } : {}),
70
+ ...(input.workspacePolicy !== undefined ? { workspacePolicy: input.workspacePolicy } : {}),
71
+ ...(input.service !== undefined ? { service: input.service } : {}),
72
+ ...(input.retryOf !== undefined ? { retryOf: input.retryOf } : {}),
73
+ ...(warning !== undefined ? { warning } : {}),
74
+ };
75
+ await writeJson(path.join(jobDir, "manifest.json"), manifest);
76
+ const status = {
77
+ jobId,
78
+ status: "queued",
79
+ createdAt,
80
+ updatedAt: createdAt,
81
+ jobDir,
82
+ ...(input.service !== undefined ? { service: input.service } : {}),
83
+ nextPollSeconds: SUGGESTED_POLL_SECONDS,
84
+ instructions: pollInstructions(jobId),
85
+ ...(warning !== undefined ? { warning } : {}),
86
+ };
87
+ await updateStatus(jobDir, status);
88
+ // Detached by default: the run must not die with this process. In-process
89
+ // mode exists for unit tests (injected fake dispatchers aren't visible to
90
+ // a separate process) and as the fallback when the runner script can't be
91
+ // found (running from raw src/ with no dist/ build).
92
+ const inproc = process.env.HARNESS_DISPATCH_INPROC_JOBS === "1";
93
+ const runnerPath = inproc ? undefined : resolveRunnerPath();
94
+ if (inproc || runnerPath === undefined) {
95
+ if (!inproc) {
96
+ console.error("harness-dispatch: dist/job-runner.js not found (unbuilt checkout?) — " +
97
+ "running the job in-process; it will not survive a server restart.");
98
+ }
99
+ // The DETACHED runner re-reads prompt.md, which carries the context
100
+ // preamble — so the in-process run must dispatch the same frozen prompt,
101
+ // not input.prompt. Passing the raw prompt here silently dropped
102
+ // contextJobs for every in-process run (unit tests with injected fakes,
103
+ // and the unbuilt-checkout fallback).
104
+ const completion = runJob(deps, jobDir, manifest, { ...input, prompt: effectivePrompt });
105
+ return { status, completion };
106
+ }
107
+ // Concurrency gate. Every dispatch spawns its own detached runner, so an
108
+ // in-process semaphore would bound nothing — the count has to come off
109
+ // disk. The caller still gets its jobId back immediately either way, so the
110
+ // API contract is unchanged and only the start time can move.
111
+ // Every dispatch takes this path, including an uncapped one. It used to
112
+ // spawn its own detached runner when the cap was lifted, which bypassed the
113
+ // pool and put a ~76 MB wrapper behind every job.
114
+ // Enqueue first, then let drainSlotQueue decide — rather than testing the
115
+ // limit here and spawning inline. Two reasons, both learned the hard way:
116
+ // this job's own `queued` status is already on disk, so an inline count
117
+ // included itself and deadlocked at limit 1; and a fresh dispatch arriving
118
+ // while others wait must not jump the queue, which only one FIFO drainer
119
+ // can guarantee. Whether this job starts now is then just "did the drain
120
+ // reach it".
121
+ await updateStatus(jobDir, { ...status, slotQueued: true });
122
+ await drainSlotQueue(deps.holder.state.config, deps.holder.state.configPath);
123
+ const settled = await readJson(path.join(jobDir, "status.json"));
124
+ return { status: settled, completion: watchUntilTerminal(jobDir) };
125
+ }
126
+ //# sourceMappingURL=start.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"start.js","sourceRoot":"","sources":["../../src/jobs/start.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,cAAc,EACd,QAAQ,EACR,aAAa,EACb,sBAAsB,EACtB,SAAS,EACT,YAAY,EACZ,SAAS,GACV,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACzE,OAAO,EACL,eAAe,EACf,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAa,EAAE,KAAoB;IACrE,OAAO,CAAC,MAAM,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAa,EAAE,KAAoB;IAC5E,uEAAuE;IACvE,6EAA6E;IAC7E,yEAAyE;IACzE,oEAAoE;IACpE,MAAM,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7D,IAAI,eAAe,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;IAEpE,6EAA6E;IAC7E,4EAA4E;IAC5E,6DAA6D;IAC7D,oEAAoE;IACpE,0EAA0E;IAC1E,qCAAqC;IACrC,EAAE;IACF,2EAA2E;IAC3E,wEAAwE;IACxE,4EAA4E;IAC5E,6EAA6E;IAC7E,mDAAmD;IACnD,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACxE,IAAI,WAAW,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;IAE5D,MAAM,cAAc,EAAE,CAAC;IACvB,4EAA4E;IAC5E,mEAAmE;IACnE,6EAA6E;IAC7E,2EAA2E;IAC3E,+BAA+B;IAC/B,MAAM,uBAAuB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5E,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClD,wEAAwE;IACxE,yEAAyE;IACzE,sBAAsB;IACtB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC;IAChD,MAAM,SAAS,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChF,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrE,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;IAC9B,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAgB;QAC5B,KAAK;QACL,SAAS;QACT,UAAU,EAAE,kBAAkB,CAAC,UAAU;QACzC,UAAU;QACV,KAAK,EAAE,aAAa;QACpB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAc;QACxB,KAAK;QACL,MAAM,EAAE,QAAQ;QAChB,SAAS;QACT,SAAS,EAAE,SAAS;QACpB,MAAM;QACN,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,eAAe,EAAE,sBAAsB;QACvC,YAAY,EAAE,gBAAgB,CAAC,KAAK,CAAC;QACrC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnC,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,qDAAqD;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,GAAG,CAAC;IAChE,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC;IAC5D,IAAI,MAAM,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CACX,uEAAuE;gBACrE,mEAAmE,CACtE,CAAC;QACJ,CAAC;QACD,oEAAoE;QACpE,yEAAyE;QACzE,iEAAiE;QACjE,wEAAwE;QACxE,sCAAsC;QACtC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;QACzF,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,4EAA4E;IAC5E,8DAA8D;IAC9D,wEAAwE;IACxE,4EAA4E;IAC5E,kDAAkD;IAElD,0EAA0E;IAC1E,0EAA0E;IAC1E,wEAAwE;IACxE,2EAA2E;IAC3E,yEAAyE;IACzE,yEAAyE;IACzE,aAAa;IACb,MAAM,YAAY,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAY,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAC5E,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;AACrE,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AASzD,iEAAiE;AACjE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,QAAS,CAAC;AAE1C;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAgB5D;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAMD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAKlE;AAED,wBAAgB,WAAW,IAAI,MAAM,CAKpC;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAmEpD;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAI/E;AAgBD,wBAAsB,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAE9D;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBnF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CA4BlG;AAED;;;GAGG;AACH,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAED,eAAO,MAAM,SAAS,QAA0B,CAAC;AAEjD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAMpD;AAmBD,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMlF;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEvD;AAED,mEAAmE;AACnE,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAQ9E"}
1
+ {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAUH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AASzD,iEAAiE;AACjE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,QAAS,CAAC;AAE1C;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAwB5D;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAMD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAKlE;AAED,wBAAgB,WAAW,IAAI,MAAM,CAKpC;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAmEpD;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/E;AAgBD,wBAAsB,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAE9D;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBnF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CA4BlG;AAED;;;GAGG;AACH,wBAAgB,QAAQ,IAAI,MAAM,CAEjC;AAED,eAAO,MAAM,SAAS,QAA0B,CAAC;AAEjD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAMpD;AAmBD,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMlF;AAED,gFAAgF;AAChF,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEvD;AAED,mEAAmE;AACnE,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAQ9E"}
@@ -12,6 +12,7 @@
12
12
  * Windows EPERM.
13
13
  */
14
14
  import { existsSync } from "node:fs";
15
+ import { redact } from "../redaction.js";
15
16
  import { chmod, copyFile, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
16
17
  import { randomUUID } from "node:crypto";
17
18
  import path from "node:path";
@@ -53,8 +54,16 @@ export function withOrphanCheck(status) {
53
54
  ...status,
54
55
  status: "orphaned",
55
56
  success: false,
56
- error: "The dispatch server that started this job exited before the run finished " +
57
- "the background run died with it. Re-dispatch the task; this job will never complete.",
57
+ // Says what is KNOWN, not which process died. This used to read "The
58
+ // dispatch server that started this job exited" and a load test killed
59
+ // a single supervisor while the server stayed up and serving, producing
60
+ // that message verbatim. The status file this is derived from records a
61
+ // heartbeat and nothing about who was holding it, so naming a culprit was
62
+ // always a guess, and it sent anyone debugging to the wrong process.
63
+ error: "This job stopped reporting progress and the process running it is gone — " +
64
+ "either the run crashed or whatever was supervising it died. Nothing will " +
65
+ "advance it now. Its partial output is on disk; `retry_job` re-runs the same " +
66
+ "task, or re-dispatch it.",
58
67
  };
59
68
  }
60
69
  export function boundedError(error) {
@@ -185,7 +194,8 @@ export function safeBaseName(filePath) {
185
194
  }
186
195
  export async function writeJson(filePath, value) {
187
196
  const tmpPath = `${filePath}.${process.pid}.${Date.now()}.${randomUUID().slice(0, 8)}.tmp`;
188
- await writeFile(tmpPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
197
+ // Sink: every job JSON on disk status.json, result.json, manifest.json.
198
+ await writeFile(tmpPath, `${redact(JSON.stringify(value, null, 2))}\n`, { encoding: "utf8", mode: 0o600 });
189
199
  await renameWithRetry(tmpPath, filePath);
190
200
  }
191
201
  async function renameWithRetry(tmpPath, filePath) {
@@ -1 +1 @@
1
- {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1G,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,iEAAiE;AACjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,2EAA2E;IAC3E,2EAA2E;IAC3E,oEAAoE;IACpE,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,mBAAmB;QAAE,OAAO,MAAM,CAAC;IACrF,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,UAAU;QAClB,OAAO,EAAE,KAAK;QACd,KAAK,EACH,6EAA6E;YAC7E,sFAAsF;KACzF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAyB;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,IAAI,oBAAoB;QAAE,OAAO,KAAK,CAAC;IACvD,OAAO,CACL,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACpC,iBAAiB,KAAK,CAAC,MAAM,GAAG,oBAAoB,0CAA0C,CAC/F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,CACL,yEAAyE;QACzE,SAAS,IAAI,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,CAAC,mCAAmC;QACnF,yBAAyB,KAAK,mDAAmD;QACjF,iFAAiF;QACjF,0CAA0C,CAC3C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvD,IAAI,qBAAyC,CAAC;AAE9C;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAwB;IAC1D,qBAAqB;QACnB,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;YACtD,CAAC,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;YAC5B,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IACxD,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;AACzD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,sEAAsE;IACtE,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,gDAAgD;IAChD,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO;IAC3B,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,0EAA0E;QAC1E,uDAAuD;QACvD,0EAA0E;QAC1E,0EAA0E;QAC1E,6DAA6D;QAC7D,iEAAiE;QACjE,EAAE;QACF,wEAAwE;QACxE,0EAA0E;QAC1E,wEAAwE;QACxE,yEAAyE;QACzE,mDAAmD;QACnD,EAAE;QACF,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,yEAAyE;QACzE,mDAAmD;QACnD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,QAAQ;gBAAE,SAAS;YAC7C,4DAA4D;YAC5D,sEAAsE;YACtE,wEAAwE;YACxE,uEAAuE;YACvE,0DAA0D;YAC1D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CACd,CAAC;gBAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;gBAChD,IACE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC;oBAC3D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACrB,GAAG,GAAG,IAAI,IAAI,mBAAmB,EACjC,CAAC;oBACD,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;YACD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;QAC3E,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IAC9D,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3F,MAAM,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACnG,MAAM,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,QAAgB;IAC9D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACpG,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO;gBAAE,MAAM,GAAG,CAAC;YACzE,MAAM,KAAK,CAAC,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAI,QAAgB;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAM,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,MAAiB;IAClE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE;YAChD,GAAG,MAAM;YACT,SAAS,EAAE,SAAS,EAAE;SACvB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,wEAAwE;QACxE,EAAE;QACF,qEAAqE;QACrE,yEAAyE;QACzE,uEAAuE;QACvE,sEAAsE;QACtE,uEAAuE;QACvE,sEAAsE;QACtE,oDAAoD;QACpD,EAAE;QACF,yEAAyE;QACzE,sEAAsE;QACtE,wEAAwE;QACxE,MAAM,IAAI,GACR,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QACzF,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO;QACrD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,KAAe;IACjE,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAiC,EAAE,YAAY,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,oBAAoB,CAAC;gBAClC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACf,SAAS;YACX,CAAC;YACD,MAAM,YAAY,GAAG,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3F,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YACvD,MAAM,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;YAC3C,MAAM,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;YACjC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC;IACjE,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAEjD;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,4CAA4C,CACnF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,aAAa,GAAG,aAAa,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,MAAe;IACjE,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAChC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAChF,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAClC,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;QACvD,OAAO,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/jobs/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1G,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,iEAAiE;AACjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,2EAA2E;IAC3E,2EAA2E;IAC3E,oEAAoE;IACpE,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,mBAAmB;QAAE,OAAO,MAAM,CAAC;IACrF,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,UAAU;QAClB,OAAO,EAAE,KAAK;QACd,qEAAqE;QACrE,yEAAyE;QACzE,wEAAwE;QACxE,wEAAwE;QACxE,0EAA0E;QAC1E,qEAAqE;QACrE,KAAK,EACH,2EAA2E;YAC3E,2EAA2E;YAC3E,8EAA8E;YAC9E,0BAA0B;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAyB;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,IAAI,oBAAoB;QAAE,OAAO,KAAK,CAAC;IACvD,OAAO,CACL,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACpC,iBAAiB,KAAK,CAAC,MAAM,GAAG,oBAAoB,0CAA0C,CAC/F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,CACL,yEAAyE;QACzE,SAAS,IAAI,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,CAAC,mCAAmC;QACnF,yBAAyB,KAAK,mDAAmD;QACjF,iFAAiF;QACjF,0CAA0C,CAC3C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvD,IAAI,qBAAyC,CAAC;AAE9C;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAwB;IAC1D,qBAAqB;QACnB,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;YACtD,CAAC,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;YAC5B,CAAC,CAAC,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IACxD,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;AACzD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,sEAAsE;IACtE,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,gDAAgD;IAChD,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO;IAC3B,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,0EAA0E;QAC1E,uDAAuD;QACvD,0EAA0E;QAC1E,0EAA0E;QAC1E,6DAA6D;QAC7D,iEAAiE;QACjE,EAAE;QACF,wEAAwE;QACxE,0EAA0E;QAC1E,wEAAwE;QACxE,yEAAyE;QACzE,mDAAmD;QACnD,EAAE;QACF,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,yEAAyE;QACzE,mDAAmD;QACnD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,QAAQ;gBAAE,SAAS;YAC7C,4DAA4D;YAC5D,sEAAsE;YACtE,wEAAwE;YACxE,uEAAuE;YACvE,0DAA0D;YAC1D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CACd,CAAC;gBAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;gBAChD,IACE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC;oBAC3D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACrB,GAAG,GAAG,IAAI,IAAI,mBAAmB,EACjC,CAAC;oBACD,SAAS;gBACX,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;YACD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;QAC3E,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IAC9D,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3F,0EAA0E;IAC1E,MAAM,SAAS,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3G,MAAM,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,QAAgB;IAC9D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACpG,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO;gBAAE,MAAM,GAAG,CAAC;YACzE,MAAM,KAAK,CAAC,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAI,QAAgB;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAM,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,MAAiB;IAClE,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE;YAChD,GAAG,MAAM;YACT,SAAS,EAAE,SAAS,EAAE;SACvB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,wEAAwE;QACxE,EAAE;QACF,qEAAqE;QACrE,yEAAyE;QACzE,uEAAuE;QACvE,sEAAsE;QACtE,uEAAuE;QACvE,sEAAsE;QACtE,oDAAoD;QACpD,EAAE;QACF,yEAAyE;QACzE,sEAAsE;QACtE,wEAAwE;QACxE,MAAM,IAAI,GACR,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAE,GAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QACzF,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO;QACrD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,KAAe;IACjE,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAiC,EAAE,YAAY,EAAE,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,oBAAoB,CAAC;gBAClC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACf,SAAS;YACX,CAAC;YACD,MAAM,YAAY,GAAG,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3F,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YACvD,MAAM,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;YAC3C,MAAM,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;YACjC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC;IACjE,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ;IACtB,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAEjD;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,4CAA4C,CACnF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,aAAa,GAAG,aAAa,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,MAAe;IACjE,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAChC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAChF,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAClC,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;QACvD,OAAO,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
@@ -0,0 +1,151 @@
1
+ /**
2
+ * Admission control and the supervisor pool: who runs, when, and in which
3
+ * process.
4
+ *
5
+ * The concurrency cap here exists because of a measured OOM, and the pool
6
+ * exists because a runner process per job costs ~76 MB of wrapper. Both are
7
+ * load-bearing; see the comments on DEFAULT_MAX_CONCURRENT_RUNS.
8
+ */
9
+ import type { RouterConfig } from "../types.js";
10
+ import type { JobDeps, JobStatus } from "./types.js";
11
+ /**
12
+ * The cap, or `null` for "no cap".
13
+ *
14
+ * `null` rather than `0`, and rather than `Infinity`, because both of those
15
+ * were wrong in a way that mattered. `0` used to short-circuit the whole slot
16
+ * queue and supervisor pool, so `max_concurrent_runs: 0` — documented as
17
+ * lifting a limit — silently gave every job its own runner process at ~76 MB,
18
+ * which is the per-job cost the pool exists to remove, on the memory-bound
19
+ * machine the cap exists for. And `Infinity` divides badly: the pool sizes
20
+ * itself with `outstanding / jobsPerSupervisor(limit)`, so an infinite limit
21
+ * asked for ZERO supervisors. An explicit `null` makes each site say what it
22
+ * means about the unbounded case.
23
+ */
24
+ export declare function maxConcurrentRuns(config: RouterConfig | undefined): number | null;
25
+ /**
26
+ * Occupied slots: jobs actually executing right now. Counts `running` (and
27
+ * plain `queued` — a runner spawned but not yet started) only while the
28
+ * heartbeat is fresh, so a crashed runner's slot is reclaimed by the same
29
+ * ORPHAN_THRESHOLD_MS rule that already frees its status. Slot-queued jobs
30
+ * are waiting for a slot, not holding one.
31
+ */
32
+ /**
33
+ * What one run of a route costs against the concurrency budget.
34
+ *
35
+ * Unknown routes count as a full 1.0 on purpose. A job that has not been
36
+ * routed yet (no forced `service`) has no weight to look up, and this bound
37
+ * exists because a measured burst of 13 concurrent CLIs exhausted memory —
38
+ * so the safe assumption for "might be anything" is "might be heavy".
39
+ */
40
+ export declare function resourceWeightFor(status: JobStatus, config: RouterConfig | undefined): number;
41
+ /**
42
+ * Capacity currently in use, as a weighted sum rather than a job count.
43
+ *
44
+ * With every weight at 1.0 this is exactly the old count, so an existing
45
+ * `max_concurrent_runs` keeps its previous meaning.
46
+ */
47
+ export declare function activeCapacity(statuses: Array<{
48
+ status: JobStatus;
49
+ }>, config: RouterConfig | undefined): number;
50
+ /**
51
+ * How many supervisor PROCESSES may exist, regardless of how many jobs run.
52
+ *
53
+ * Previously every job got its own detached Node process. Measured on Windows
54
+ * with Node 24: a bare node process is 52 MB RSS and one that has bootstrapped
55
+ * a runtime is 65 MB, against ~54 MB for the agent CLI it exists to supervise.
56
+ * So more than half the memory of a concurrent run was wrapper, and it scaled
57
+ * linearly — 13 concurrent jobs meant 845 MB of supervision before any agent
58
+ * had read a file. That is the concurrency ceiling.
59
+ *
60
+ * A supervisor is almost entirely idle: it waits on a child process and writes
61
+ * the result. One can watch several at once for the cost of async I/O, so
62
+ * wrapper memory becomes O(1) in the number of jobs instead of O(N), capped
63
+ * here at ~260 MB.
64
+ *
65
+ * Four rather than one purely to bound blast radius: a supervisor crash strands
66
+ * only the jobs it held. Those are recoverable anyway — the job directory is
67
+ * the source of truth and the heartbeat check already marks stranded jobs
68
+ * orphaned — but losing a quarter of in-flight work beats losing all of it.
69
+ */
70
+ export declare const SUPERVISOR_POOL_SIZE = 4;
71
+ /**
72
+ * Take exclusive ownership of a job directory.
73
+ *
74
+ * `wx` fails if the file exists, atomically, on both Windows and POSIX — which
75
+ * is what stops two supervisors racing onto the same job. A claim left behind
76
+ * by a crashed supervisor is reclaimed once that job's heartbeat has gone
77
+ * stale, by the same ORPHAN_THRESHOLD_MS rule used everywhere else.
78
+ *
79
+ * Exported for tests: the one-winner property under concurrent reclaim is the
80
+ * invariant, and it is only checkable by calling this directly.
81
+ */
82
+ export declare function claimJobDir(jobDir: string, status: JobStatus): Promise<boolean>;
83
+ /**
84
+ * Supervisor main loop: claim work, run several jobs at once, exit when idle.
85
+ *
86
+ * Exiting on idle keeps the no-jobs steady state at zero processes, same as
87
+ * before — the pool is a way to share supervision cost while work exists, not
88
+ * a daemon.
89
+ */
90
+ export declare function runSupervisor(deps: JobDeps, supervisorId?: string): Promise<void>;
91
+ /**
92
+ * Start slot-queued jobs, oldest first, until the machine is at its limit.
93
+ *
94
+ * Deliberately has no daemon behind it: this runs on every new dispatch and
95
+ * again as each runner exits, which between them covers every moment a slot
96
+ * can free. The cost of that choice is that if every runner dies while jobs
97
+ * are queued, the queue resumes on the next dispatch rather than immediately.
98
+ * Bounded waiting was the explicit alternative and was not chosen — a queued
99
+ * job keeps its jobId and its artifacts either way, so nothing is lost.
100
+ *
101
+ * NOT called at server start, which was tried and reverted: it silently ran
102
+ * jobs abandoned by a dead session. `orphanStrandedSlotQueue` runs there
103
+ * instead and reports them. See its comment for why reporting beats resuming.
104
+ */
105
+ /**
106
+ * Mark jobs stranded in the slot queue by a server that is gone.
107
+ *
108
+ * Called once at server start, where the reasoning holds unconditionally: this
109
+ * process has not queued anything yet, so anything still slot-queued was
110
+ * queued by a session that no longer exists and nothing will ever drain it —
111
+ * a new dispatch would, but the caller is asking about THIS job, and until
112
+ * they happen to send unrelated work it reads `queued` forever.
113
+ *
114
+ * Deliberately reports rather than runs. Resuming was tried and is worse: a
115
+ * job queued days ago would execute at the next server start, in its original
116
+ * workingDir, at up to `full_auto`, with nobody watching. The job keeps its id
117
+ * and artifacts, so `retry_job` re-runs it as a decision rather than a side
118
+ * effect of opening an editor.
119
+ *
120
+ * The one status this writes back. Orphan detection elsewhere is
121
+ * compute-on-read and never persists its verdict, because the owner might
122
+ * still be alive; here the owner is definitionally gone.
123
+ */
124
+ export declare function orphanStrandedSlotQueue(): Promise<number>;
125
+ export declare function drainSlotQueue(config: RouterConfig | undefined, configPath: string | undefined): Promise<void>;
126
+ /**
127
+ * Supervisors currently alive, counted from their heartbeat files.
128
+ *
129
+ * Approximate on purpose: over-counting briefly means the pool runs one short
130
+ * until the next drain, and under-counting means one extra supervisor that
131
+ * finds no work and exits within SUPERVISOR_IDLE_EXIT_MS. Neither warrants a
132
+ * lock, and both self-correct.
133
+ */
134
+ declare function countLiveSupervisors(): Promise<number>;
135
+ /**
136
+ * Exported for the cleanup test, which must exercise the REAL sweep rather
137
+ * than a copy of its logic — the bug being pinned is that a stale heartbeat
138
+ * was never removed, and a reimplementation in the test would pin nothing.
139
+ */
140
+ export declare const countLiveSupervisorsForTest: typeof countLiveSupervisors;
141
+ /**
142
+ * Why a detached runner would fail to bootstrap from this config path, if it
143
+ * would. `undefined` means the file loads (or there is none, which is the
144
+ * auto-detect case and always fine).
145
+ *
146
+ * Deliberately re-reads rather than trusting the server's in-memory config:
147
+ * the two disagreeing is exactly the condition being detected.
148
+ */
149
+ export declare function configLoadError(configPath: string | undefined): Promise<string | undefined>;
150
+ export {};
151
+ //# sourceMappingURL=supervisor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"supervisor.d.ts","sourceRoot":"","sources":["../../src/jobs/supervisor.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAWH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAWhD,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AA6BrD;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAMjF;AAoBD;;;;;;GAMG;AACH;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,MAAM,CAQ7F;AAeD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,CAAC,EACtC,MAAM,EAAE,YAAY,GAAG,SAAS,GAC/B,MAAM,CAUR;AAOD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAoBtC;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAuCrF;AAwBD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgGvF;AAED;;;;;;;;;;;;;GAaG;AACH;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,MAAM,CAAC,CAqC/D;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,YAAY,GAAG,SAAS,EAChC,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAkGD;;;;;;;GAOG;AACH,iBAAe,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CA2CrD;AAED;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,6BAAuB,CAAC;AA0ChE;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAcjG"}