@luckydraw/cumulus 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/gateway/daemon.d.ts.map +1 -1
- package/dist/gateway/daemon.js +53 -18
- package/dist/gateway/daemon.js.map +1 -1
- package/dist/gateway/gateway-agents-mcp.js +133 -0
- package/dist/gateway/gateway-agents-mcp.js.map +1 -1
- package/dist/gateway/jobs.d.ts +158 -0
- package/dist/gateway/jobs.d.ts.map +1 -0
- package/dist/gateway/jobs.js +497 -0
- package/dist/gateway/jobs.js.map +1 -0
- package/dist/gateway/scheduler.d.ts.map +1 -1
- package/dist/gateway/scheduler.js +9 -30
- package/dist/gateway/scheduler.js.map +1 -1
- package/dist/gateway/server.d.ts +16 -0
- package/dist/gateway/server.d.ts.map +1 -1
- package/dist/gateway/server.js +183 -2
- package/dist/gateway/server.js.map +1 -1
- package/dist/gateway/setup.d.ts.map +1 -1
- package/dist/gateway/setup.js +10 -0
- package/dist/gateway/setup.js.map +1 -1
- package/dist/lib/config.d.ts +11 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +21 -0
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/gateway.d.ts +35 -45
- package/dist/lib/gateway.d.ts.map +1 -1
- package/dist/lib/gateway.js +98 -117
- package/dist/lib/gateway.js.map +1 -1
- package/dist/lib/tool-inventory.d.ts +140 -0
- package/dist/lib/tool-inventory.d.ts.map +1 -0
- package/dist/lib/tool-inventory.js +317 -0
- package/dist/lib/tool-inventory.js.map +1 -0
- package/docs/conditional-continuation.md +102 -147
- package/docs/web-app-agent-guide.md +74 -3
- package/examples/web-app-agent/README.md +40 -8
- package/examples/web-app-agent/thread-config.visitor.example.json +30 -2
- package/package.json +1 -1
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job registry — daemon-owned background jobs (task 139).
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* Cumulus tried three times to make background work reliable by TELLING the
|
|
6
|
+
* model how to do it: task 108 (the redirect cure), 109 (the two-leg
|
|
7
|
+
* watcher+deadline rule), 116 (the worked one-liner in the content store).
|
|
8
|
+
* All three are correct and all three still failed live, because the cure has
|
|
9
|
+
* to be applied by the driver AND every child, and because no shell trick
|
|
10
|
+
* escapes the third killer at all. This is task 115's lesson applied again:
|
|
11
|
+
* stop prescribing, enforce at the seam.
|
|
12
|
+
*
|
|
13
|
+
* WHAT OWNING IT BUYS, BY CONSTRUCTION
|
|
14
|
+
* 1. Turn end → SIGPIPE. The daemon spawns the job with NO pipes: stdout and
|
|
15
|
+
* stderr are a file descriptor, stdin is /dev/null. There is nothing whose
|
|
16
|
+
* read end can close, so SIGPIPE (task 108's entire failure mode) stops
|
|
17
|
+
* being reachable rather than being cured.
|
|
18
|
+
* 2. Interjection → SIGTERM. The job is not in the turn's process tree, so
|
|
19
|
+
* killing the turn cannot reach it.
|
|
20
|
+
* 3. Gateway reload → cgroup reap. NOT solved by spawning here — `detached`
|
|
21
|
+
* gives a new session, and a session does not leave a cgroup. It needs
|
|
22
|
+
* `KillMode=process` on the unit (shipped in setup.ts's template). What
|
|
23
|
+
* this module guarantees regardless is that a job killed by a restart is
|
|
24
|
+
* REPORTED as interrupted rather than silently vanishing.
|
|
25
|
+
*
|
|
26
|
+
* The exit code is written to disk BY THE JOB ITSELF, not read off the child
|
|
27
|
+
* `exit` event. The event is authoritative while this process lives and is
|
|
28
|
+
* unavailable after a restart; the file survives both, which is what makes
|
|
29
|
+
* adoption able to distinguish "exited 7 while we were down" from "killed".
|
|
30
|
+
*/
|
|
31
|
+
export type JobStatus = 'running' | 'done' | 'failed' | 'cancelled' | 'interrupted';
|
|
32
|
+
export interface JobRecord {
|
|
33
|
+
id: string;
|
|
34
|
+
thread: string;
|
|
35
|
+
label: string;
|
|
36
|
+
command: string;
|
|
37
|
+
cwd: string;
|
|
38
|
+
pid?: number;
|
|
39
|
+
/**
|
|
40
|
+
* OS-level start token, guarding against pid reuse across a gateway restart.
|
|
41
|
+
* A pid alone is not identity: on a busy box the number can belong to an
|
|
42
|
+
* unrelated process by the time we adopt, and we would then report a live
|
|
43
|
+
* job for something that died. Linux reads field 22 of /proc/<pid>/stat
|
|
44
|
+
* (start time in clock ticks); elsewhere it is absent and liveness degrades
|
|
45
|
+
* to a bare kill(pid, 0).
|
|
46
|
+
*/
|
|
47
|
+
startToken?: string;
|
|
48
|
+
startedAt: string;
|
|
49
|
+
endedAt?: string;
|
|
50
|
+
status: JobStatus;
|
|
51
|
+
exitCode?: number;
|
|
52
|
+
signal?: string;
|
|
53
|
+
logPath: string;
|
|
54
|
+
}
|
|
55
|
+
export interface CreateJobRequest {
|
|
56
|
+
thread: string;
|
|
57
|
+
command: string;
|
|
58
|
+
label?: string;
|
|
59
|
+
cwd: string;
|
|
60
|
+
}
|
|
61
|
+
/** Max concurrently-running jobs for one thread. */
|
|
62
|
+
export declare const MAX_RUNNING_PER_THREAD = 5;
|
|
63
|
+
/** Max concurrently-running jobs across all threads. */
|
|
64
|
+
export declare const MAX_RUNNING_TOTAL = 20;
|
|
65
|
+
/** Finished records retained per thread; older ones are pruned with their logs. */
|
|
66
|
+
export declare const MAX_FINISHED_PER_THREAD = 25;
|
|
67
|
+
/** Lines of log tail carried in the completion report. */
|
|
68
|
+
export declare const COMPLETION_TAIL_LINES = 40;
|
|
69
|
+
/** How often adopted jobs (no live child handle) are checked for death. */
|
|
70
|
+
export declare const ADOPTION_POLL_MS = 5000;
|
|
71
|
+
export interface ValidationResult {
|
|
72
|
+
ok: boolean;
|
|
73
|
+
error?: string;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Validate a job request. Deliberately permissive about the command itself —
|
|
77
|
+
* a thread that reaches this seam has already been checked for `Bash`
|
|
78
|
+
* availability by the caller, so the command is no more privileged than what
|
|
79
|
+
* it could already run. What is rejected is what would make the RECORD
|
|
80
|
+
* unusable: an empty command, or a label that would corrupt the report.
|
|
81
|
+
*/
|
|
82
|
+
export declare function validateJobRequest(req: Partial<CreateJobRequest>): ValidationResult;
|
|
83
|
+
/** Map a process outcome onto a durable status. */
|
|
84
|
+
export declare function classifyExit(exitCode: number | undefined, signal?: string): JobStatus;
|
|
85
|
+
/**
|
|
86
|
+
* Wrap the command so the job records its OWN exit status before leaving.
|
|
87
|
+
*
|
|
88
|
+
* An EXIT trap, NOT trailing statements. Trailing statements are unreachable
|
|
89
|
+
* whenever the command ends in `exit N` (or `exec`) — which is ordinary in a
|
|
90
|
+
* driver script, and which silently produced "interrupted" for jobs that had
|
|
91
|
+
* in fact finished cleanly. A trap fires on both paths. `$?` inside it is the
|
|
92
|
+
* status that triggered it, and nothing runs before the capture.
|
|
93
|
+
*
|
|
94
|
+
* The path arrives through the environment rather than interpolated, so no
|
|
95
|
+
* quoting of ours can collide with the command's.
|
|
96
|
+
*
|
|
97
|
+
* Known limit: a command that installs its own EXIT trap replaces this one, and
|
|
98
|
+
* that job reports as interrupted. Rare, and it degrades to the honest answer.
|
|
99
|
+
*/
|
|
100
|
+
export declare function wrapCommand(command: string): string;
|
|
101
|
+
/** Read the OS start token for a pid (Linux only; undefined elsewhere). */
|
|
102
|
+
export declare function readStartToken(pid: number): string | undefined;
|
|
103
|
+
/**
|
|
104
|
+
* Is this pid still the process we started?
|
|
105
|
+
*
|
|
106
|
+
* `kill(pid, 0)` answers "does a process with this number exist", which is a
|
|
107
|
+
* weaker question. When a start token was recorded we also require it to
|
|
108
|
+
* match, so a recycled pid reads as dead rather than as our job.
|
|
109
|
+
*/
|
|
110
|
+
export declare function isProcessAlive(pid: number | undefined, startToken?: string): boolean;
|
|
111
|
+
/**
|
|
112
|
+
* The message the finished job sends back into its thread.
|
|
113
|
+
*
|
|
114
|
+
* Self-describing on purpose: it may arrive either as a fresh turn (idle) or
|
|
115
|
+
* inside a batched queue drain (busy), and it has to read correctly both ways.
|
|
116
|
+
* It carries the exit code and a log tail so the woken turn can DIAGNOSE
|
|
117
|
+
* without re-running — task 109's "fire on outcome, not on success".
|
|
118
|
+
*/
|
|
119
|
+
export declare function formatCompletionMessage(job: JobRecord, logTail: string): string;
|
|
120
|
+
/** Read the last N lines of a file, bounded so a huge log cannot be slurped. */
|
|
121
|
+
export declare function tailFile(filePath: string, lines: number, maxBytes?: number): string;
|
|
122
|
+
/**
|
|
123
|
+
* Trim finished records for a thread down to the retention cap.
|
|
124
|
+
* Returns the records to keep and the ones evicted (whose files the caller
|
|
125
|
+
* deletes) — pure so the retention rule is testable without a filesystem.
|
|
126
|
+
*/
|
|
127
|
+
export declare function pruneFinished(records: JobRecord[], thread: string, keep?: number): {
|
|
128
|
+
kept: JobRecord[];
|
|
129
|
+
evicted: JobRecord[];
|
|
130
|
+
};
|
|
131
|
+
export interface JobRegistryOptions {
|
|
132
|
+
/** Directory holding jobs.json and the per-job log/exit files. */
|
|
133
|
+
jobsDir?: string;
|
|
134
|
+
log?: (msg: string, data?: Record<string, unknown>) => void;
|
|
135
|
+
/**
|
|
136
|
+
* Deliver a completion report as a turn on the job's thread. Injected rather
|
|
137
|
+
* than imported so the registry has no dependency on the HTTP server (and so
|
|
138
|
+
* tests can observe delivery without spawning Claude). The daemon wires this
|
|
139
|
+
* to server.ts's `deliverAgentTurn`, which is the SAME busy-gate + queue used
|
|
140
|
+
* by the scheduler and every other inject path (tasks 100/120).
|
|
141
|
+
*/
|
|
142
|
+
deliver: (thread: string, text: string, sender: string) => void | Promise<void>;
|
|
143
|
+
pollIntervalMs?: number;
|
|
144
|
+
}
|
|
145
|
+
export interface JobRegistryHandle {
|
|
146
|
+
create(req: CreateJobRequest): JobRecord;
|
|
147
|
+
list(thread: string): JobRecord[];
|
|
148
|
+
get(thread: string, id: string): JobRecord | undefined;
|
|
149
|
+
cancel(thread: string, id: string): boolean;
|
|
150
|
+
tail(thread: string, id: string, lines?: number): string | undefined;
|
|
151
|
+
/** Adopt jobs recorded by a previous daemon process. Called once at startup. */
|
|
152
|
+
adopt(): void;
|
|
153
|
+
stop(): void;
|
|
154
|
+
/** Test seam: run one adoption-poll pass immediately. */
|
|
155
|
+
pollOnce(): void;
|
|
156
|
+
}
|
|
157
|
+
export declare function createJobRegistry(opts: JobRegistryOptions): JobRegistryHandle;
|
|
158
|
+
//# sourceMappingURL=jobs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../src/gateway/jobs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAQH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,GAAG,aAAa,CAAC;AAEpF,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;CACb;AAID,oDAAoD;AACpD,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,wDAAwD;AACxD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,mFAAmF;AACnF,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAC1C,0DAA0D;AAC1D,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC,2EAA2E;AAC3E,eAAO,MAAM,gBAAgB,OAAQ,CAAC;AAEtC,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,gBAAgB,CAmBnF;AAED,mDAAmD;AACnD,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAIrF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAY9D;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAYpF;AAUD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAmC/E;AAED,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,SAAa,GAAG,MAAM,CAoBvF;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,EAAE,EACpB,MAAM,EAAE,MAAM,EACd,IAAI,SAA0B,GAC7B;IAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAE,CAQ7C;AAID,MAAM,WAAW,kBAAkB;IACjC,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAC5D;;;;;;OAMG;IACH,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChF,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,GAAG,EAAE,gBAAgB,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,CAAC;IAClC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAC;IACvD,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC5C,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IACrE,gFAAgF;IAChF,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,IAAI,CAAC;IACb,yDAAyD;IACzD,QAAQ,IAAI,IAAI,CAAC;CAClB;AAMD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,kBAAkB,GAAG,iBAAiB,CAiS7E"}
|
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job registry — daemon-owned background jobs (task 139).
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* Cumulus tried three times to make background work reliable by TELLING the
|
|
6
|
+
* model how to do it: task 108 (the redirect cure), 109 (the two-leg
|
|
7
|
+
* watcher+deadline rule), 116 (the worked one-liner in the content store).
|
|
8
|
+
* All three are correct and all three still failed live, because the cure has
|
|
9
|
+
* to be applied by the driver AND every child, and because no shell trick
|
|
10
|
+
* escapes the third killer at all. This is task 115's lesson applied again:
|
|
11
|
+
* stop prescribing, enforce at the seam.
|
|
12
|
+
*
|
|
13
|
+
* WHAT OWNING IT BUYS, BY CONSTRUCTION
|
|
14
|
+
* 1. Turn end → SIGPIPE. The daemon spawns the job with NO pipes: stdout and
|
|
15
|
+
* stderr are a file descriptor, stdin is /dev/null. There is nothing whose
|
|
16
|
+
* read end can close, so SIGPIPE (task 108's entire failure mode) stops
|
|
17
|
+
* being reachable rather than being cured.
|
|
18
|
+
* 2. Interjection → SIGTERM. The job is not in the turn's process tree, so
|
|
19
|
+
* killing the turn cannot reach it.
|
|
20
|
+
* 3. Gateway reload → cgroup reap. NOT solved by spawning here — `detached`
|
|
21
|
+
* gives a new session, and a session does not leave a cgroup. It needs
|
|
22
|
+
* `KillMode=process` on the unit (shipped in setup.ts's template). What
|
|
23
|
+
* this module guarantees regardless is that a job killed by a restart is
|
|
24
|
+
* REPORTED as interrupted rather than silently vanishing.
|
|
25
|
+
*
|
|
26
|
+
* The exit code is written to disk BY THE JOB ITSELF, not read off the child
|
|
27
|
+
* `exit` event. The event is authoritative while this process lives and is
|
|
28
|
+
* unavailable after a restart; the file survives both, which is what makes
|
|
29
|
+
* adoption able to distinguish "exited 7 while we were down" from "killed".
|
|
30
|
+
*/
|
|
31
|
+
import { spawn } from 'child_process';
|
|
32
|
+
import * as crypto from 'crypto';
|
|
33
|
+
import * as fs from 'fs';
|
|
34
|
+
import * as os from 'os';
|
|
35
|
+
import * as path from 'path';
|
|
36
|
+
// ─── Pure helpers ────────────────────────────────────────────────────────────
|
|
37
|
+
/** Max concurrently-running jobs for one thread. */
|
|
38
|
+
export const MAX_RUNNING_PER_THREAD = 5;
|
|
39
|
+
/** Max concurrently-running jobs across all threads. */
|
|
40
|
+
export const MAX_RUNNING_TOTAL = 20;
|
|
41
|
+
/** Finished records retained per thread; older ones are pruned with their logs. */
|
|
42
|
+
export const MAX_FINISHED_PER_THREAD = 25;
|
|
43
|
+
/** Lines of log tail carried in the completion report. */
|
|
44
|
+
export const COMPLETION_TAIL_LINES = 40;
|
|
45
|
+
/** How often adopted jobs (no live child handle) are checked for death. */
|
|
46
|
+
export const ADOPTION_POLL_MS = 5_000;
|
|
47
|
+
/**
|
|
48
|
+
* Validate a job request. Deliberately permissive about the command itself —
|
|
49
|
+
* a thread that reaches this seam has already been checked for `Bash`
|
|
50
|
+
* availability by the caller, so the command is no more privileged than what
|
|
51
|
+
* it could already run. What is rejected is what would make the RECORD
|
|
52
|
+
* unusable: an empty command, or a label that would corrupt the report.
|
|
53
|
+
*/
|
|
54
|
+
export function validateJobRequest(req) {
|
|
55
|
+
if (typeof req.command !== 'string' || req.command.trim().length === 0) {
|
|
56
|
+
return { ok: false, error: 'command is required and must be a non-empty string' };
|
|
57
|
+
}
|
|
58
|
+
if (req.command.length > 8_000) {
|
|
59
|
+
return { ok: false, error: 'command exceeds 8000 characters' };
|
|
60
|
+
}
|
|
61
|
+
if (req.label !== undefined) {
|
|
62
|
+
if (typeof req.label !== 'string') {
|
|
63
|
+
return { ok: false, error: 'label must be a string' };
|
|
64
|
+
}
|
|
65
|
+
if (req.label.length > 120) {
|
|
66
|
+
return { ok: false, error: 'label exceeds 120 characters' };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (req.cwd !== undefined && typeof req.cwd !== 'string') {
|
|
70
|
+
return { ok: false, error: 'cwd must be a string' };
|
|
71
|
+
}
|
|
72
|
+
return { ok: true };
|
|
73
|
+
}
|
|
74
|
+
/** Map a process outcome onto a durable status. */
|
|
75
|
+
export function classifyExit(exitCode, signal) {
|
|
76
|
+
if (signal)
|
|
77
|
+
return 'failed';
|
|
78
|
+
if (exitCode === undefined)
|
|
79
|
+
return 'interrupted';
|
|
80
|
+
return exitCode === 0 ? 'done' : 'failed';
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Wrap the command so the job records its OWN exit status before leaving.
|
|
84
|
+
*
|
|
85
|
+
* An EXIT trap, NOT trailing statements. Trailing statements are unreachable
|
|
86
|
+
* whenever the command ends in `exit N` (or `exec`) — which is ordinary in a
|
|
87
|
+
* driver script, and which silently produced "interrupted" for jobs that had
|
|
88
|
+
* in fact finished cleanly. A trap fires on both paths. `$?` inside it is the
|
|
89
|
+
* status that triggered it, and nothing runs before the capture.
|
|
90
|
+
*
|
|
91
|
+
* The path arrives through the environment rather than interpolated, so no
|
|
92
|
+
* quoting of ours can collide with the command's.
|
|
93
|
+
*
|
|
94
|
+
* Known limit: a command that installs its own EXIT trap replaces this one, and
|
|
95
|
+
* that job reports as interrupted. Rare, and it degrades to the honest answer.
|
|
96
|
+
*/
|
|
97
|
+
export function wrapCommand(command) {
|
|
98
|
+
return [
|
|
99
|
+
`trap '__cumulus_rc=$?; printf %s "$__cumulus_rc" > "$CUMULUS_JOB_EXIT_FILE" 2>/dev/null' EXIT`,
|
|
100
|
+
command,
|
|
101
|
+
].join('\n');
|
|
102
|
+
}
|
|
103
|
+
/** Read the OS start token for a pid (Linux only; undefined elsewhere). */
|
|
104
|
+
export function readStartToken(pid) {
|
|
105
|
+
try {
|
|
106
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf-8');
|
|
107
|
+
// Field 2 (comm) can contain spaces and parentheses, so split after the
|
|
108
|
+
// LAST ')' rather than tokenising the whole line.
|
|
109
|
+
const after = stat.slice(stat.lastIndexOf(')') + 2);
|
|
110
|
+
const fields = after.split(' ');
|
|
111
|
+
// starttime is field 22 overall = index 19 of the post-comm fields
|
|
112
|
+
return fields[19];
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Is this pid still the process we started?
|
|
120
|
+
*
|
|
121
|
+
* `kill(pid, 0)` answers "does a process with this number exist", which is a
|
|
122
|
+
* weaker question. When a start token was recorded we also require it to
|
|
123
|
+
* match, so a recycled pid reads as dead rather than as our job.
|
|
124
|
+
*/
|
|
125
|
+
export function isProcessAlive(pid, startToken) {
|
|
126
|
+
if (!pid || pid <= 0)
|
|
127
|
+
return false;
|
|
128
|
+
try {
|
|
129
|
+
process.kill(pid, 0);
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
// EPERM means it EXISTS but belongs to someone else — alive, not gone.
|
|
133
|
+
return err?.code === 'EPERM';
|
|
134
|
+
}
|
|
135
|
+
if (startToken === undefined)
|
|
136
|
+
return true;
|
|
137
|
+
const current = readStartToken(pid);
|
|
138
|
+
if (current === undefined)
|
|
139
|
+
return true; // can't tell — don't declare it dead
|
|
140
|
+
return current === startToken;
|
|
141
|
+
}
|
|
142
|
+
function humanDuration(ms) {
|
|
143
|
+
const s = Math.round(ms / 1000);
|
|
144
|
+
if (s < 60)
|
|
145
|
+
return `${s}s`;
|
|
146
|
+
const m = Math.floor(s / 60);
|
|
147
|
+
if (m < 60)
|
|
148
|
+
return `${m}m ${s % 60}s`;
|
|
149
|
+
return `${Math.floor(m / 60)}h ${m % 60}m`;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* The message the finished job sends back into its thread.
|
|
153
|
+
*
|
|
154
|
+
* Self-describing on purpose: it may arrive either as a fresh turn (idle) or
|
|
155
|
+
* inside a batched queue drain (busy), and it has to read correctly both ways.
|
|
156
|
+
* It carries the exit code and a log tail so the woken turn can DIAGNOSE
|
|
157
|
+
* without re-running — task 109's "fire on outcome, not on success".
|
|
158
|
+
*/
|
|
159
|
+
export function formatCompletionMessage(job, logTail) {
|
|
160
|
+
const started = new Date(job.startedAt).getTime();
|
|
161
|
+
const ended = job.endedAt ? new Date(job.endedAt).getTime() : Date.now();
|
|
162
|
+
const duration = humanDuration(Math.max(0, ended - started));
|
|
163
|
+
const outcome = job.status === 'done'
|
|
164
|
+
? 'succeeded (exit 0)'
|
|
165
|
+
: job.status === 'cancelled'
|
|
166
|
+
? 'was cancelled'
|
|
167
|
+
: job.status === 'interrupted'
|
|
168
|
+
? 'was INTERRUPTED — the gateway restarted while it was running, and it did not record an exit status. It may or may not have completed its work; verify before assuming either.'
|
|
169
|
+
: job.signal
|
|
170
|
+
? `was killed by ${job.signal}`
|
|
171
|
+
: `FAILED (exit ${job.exitCode})`;
|
|
172
|
+
const lines = [
|
|
173
|
+
`[background job ${job.id} "${job.label}"] ${outcome} after ${duration}.`,
|
|
174
|
+
`Command: ${job.command}`,
|
|
175
|
+
`Working directory: ${job.cwd}`,
|
|
176
|
+
`Full log: ${job.logPath}`,
|
|
177
|
+
];
|
|
178
|
+
if (logTail.trim().length > 0) {
|
|
179
|
+
lines.push('', `--- last ${COMPLETION_TAIL_LINES} lines of output ---`, logTail.trimEnd());
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
lines.push('', '(the job produced no output)');
|
|
183
|
+
}
|
|
184
|
+
lines.push('', 'This is the job you started reporting its own outcome. Continue the work it was part of; read the full log if the tail is not enough.');
|
|
185
|
+
return lines.join('\n');
|
|
186
|
+
}
|
|
187
|
+
/** Read the last N lines of a file, bounded so a huge log cannot be slurped. */
|
|
188
|
+
export function tailFile(filePath, lines, maxBytes = 256 * 1024) {
|
|
189
|
+
try {
|
|
190
|
+
const stat = fs.statSync(filePath);
|
|
191
|
+
const start = Math.max(0, stat.size - maxBytes);
|
|
192
|
+
const fd = fs.openSync(filePath, 'r');
|
|
193
|
+
try {
|
|
194
|
+
const length = stat.size - start;
|
|
195
|
+
const buf = Buffer.alloc(length);
|
|
196
|
+
fs.readSync(fd, buf, 0, length, start);
|
|
197
|
+
const text = buf.toString('utf-8');
|
|
198
|
+
const all = text.split('\n');
|
|
199
|
+
// A partial first line is an artifact of the byte window, not content.
|
|
200
|
+
if (start > 0 && all.length > 1)
|
|
201
|
+
all.shift();
|
|
202
|
+
return all.slice(-lines).join('\n');
|
|
203
|
+
}
|
|
204
|
+
finally {
|
|
205
|
+
fs.closeSync(fd);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
return '';
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Trim finished records for a thread down to the retention cap.
|
|
214
|
+
* Returns the records to keep and the ones evicted (whose files the caller
|
|
215
|
+
* deletes) — pure so the retention rule is testable without a filesystem.
|
|
216
|
+
*/
|
|
217
|
+
export function pruneFinished(records, thread, keep = MAX_FINISHED_PER_THREAD) {
|
|
218
|
+
const finished = records
|
|
219
|
+
.filter(r => r.thread === thread && r.status !== 'running')
|
|
220
|
+
.sort((a, b) => (a.endedAt ?? a.startedAt).localeCompare(b.endedAt ?? b.startedAt));
|
|
221
|
+
if (finished.length <= keep)
|
|
222
|
+
return { kept: records, evicted: [] };
|
|
223
|
+
const evicted = finished.slice(0, finished.length - keep);
|
|
224
|
+
const evictedIds = new Set(evicted.map(r => r.id));
|
|
225
|
+
return { kept: records.filter(r => !evictedIds.has(r.id)), evicted };
|
|
226
|
+
}
|
|
227
|
+
function defaultJobsDir() {
|
|
228
|
+
return path.join(process.env.CUMULUS_DIR || path.join(os.homedir(), '.cumulus'), 'jobs');
|
|
229
|
+
}
|
|
230
|
+
export function createJobRegistry(opts) {
|
|
231
|
+
const jobsDir = opts.jobsDir ?? defaultJobsDir();
|
|
232
|
+
const registryPath = path.join(jobsDir, 'jobs.json');
|
|
233
|
+
const log = opts.log ?? ((msg) => console.log(`[Jobs] ${msg}`));
|
|
234
|
+
const pollMs = opts.pollIntervalMs ?? ADOPTION_POLL_MS;
|
|
235
|
+
let records = [];
|
|
236
|
+
/** Jobs spawned by THIS process — their exit arrives as an event, not a poll. */
|
|
237
|
+
const liveChildren = new Map();
|
|
238
|
+
let poller = null;
|
|
239
|
+
let stopped = false;
|
|
240
|
+
fs.mkdirSync(jobsDir, { recursive: true });
|
|
241
|
+
function loadRegistry() {
|
|
242
|
+
try {
|
|
243
|
+
records = JSON.parse(fs.readFileSync(registryPath, 'utf-8'));
|
|
244
|
+
if (!Array.isArray(records))
|
|
245
|
+
records = [];
|
|
246
|
+
}
|
|
247
|
+
catch {
|
|
248
|
+
records = [];
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
function persist() {
|
|
252
|
+
try {
|
|
253
|
+
const tmp = `${registryPath}.tmp`;
|
|
254
|
+
fs.writeFileSync(tmp, JSON.stringify(records, null, 2));
|
|
255
|
+
fs.renameSync(tmp, registryPath);
|
|
256
|
+
}
|
|
257
|
+
catch (err) {
|
|
258
|
+
log(`Failed to persist job registry: ${err}`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
function exitFilePath(id) {
|
|
262
|
+
return path.join(jobsDir, `${id}.exit`);
|
|
263
|
+
}
|
|
264
|
+
function readRecordedExit(id) {
|
|
265
|
+
try {
|
|
266
|
+
const raw = fs.readFileSync(exitFilePath(id), 'utf-8').trim();
|
|
267
|
+
const code = parseInt(raw, 10);
|
|
268
|
+
return Number.isNaN(code) ? undefined : code;
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
function prune(thread) {
|
|
275
|
+
const { kept, evicted } = pruneFinished(records, thread);
|
|
276
|
+
if (evicted.length === 0)
|
|
277
|
+
return;
|
|
278
|
+
records = kept;
|
|
279
|
+
for (const job of evicted) {
|
|
280
|
+
try {
|
|
281
|
+
fs.unlinkSync(job.logPath);
|
|
282
|
+
}
|
|
283
|
+
catch {
|
|
284
|
+
/* already gone */
|
|
285
|
+
}
|
|
286
|
+
try {
|
|
287
|
+
fs.unlinkSync(exitFilePath(job.id));
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
/* already gone */
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Finalize a job and wake its thread. Idempotent — a job reports once.
|
|
296
|
+
*
|
|
297
|
+
* `notify` is false only for an explicit cancel: the thread that cancelled is
|
|
298
|
+
* mid-turn, so a report would be queued and then spend a whole extra turn
|
|
299
|
+
* telling it something it just did.
|
|
300
|
+
*/
|
|
301
|
+
function finish(job, status, exitCode, signal, notify = true) {
|
|
302
|
+
if (job.status !== 'running')
|
|
303
|
+
return;
|
|
304
|
+
job.status = status;
|
|
305
|
+
job.exitCode = exitCode;
|
|
306
|
+
if (signal)
|
|
307
|
+
job.signal = signal;
|
|
308
|
+
job.endedAt = new Date().toISOString();
|
|
309
|
+
liveChildren.delete(job.id);
|
|
310
|
+
prune(job.thread);
|
|
311
|
+
persist();
|
|
312
|
+
log(`Job ${job.id} ("${job.label}") ${status}`, {
|
|
313
|
+
thread: job.thread,
|
|
314
|
+
exitCode,
|
|
315
|
+
signal,
|
|
316
|
+
durationMs: Date.now() - new Date(job.startedAt).getTime(),
|
|
317
|
+
});
|
|
318
|
+
if (!notify)
|
|
319
|
+
return;
|
|
320
|
+
const tail = tailFile(job.logPath, COMPLETION_TAIL_LINES);
|
|
321
|
+
void Promise.resolve(opts.deliver(job.thread, formatCompletionMessage(job, tail), `job:${job.id}`)).catch(err => log(`Failed to deliver completion for ${job.id}: ${err}`));
|
|
322
|
+
}
|
|
323
|
+
function create(req) {
|
|
324
|
+
const id = `job_${crypto.randomBytes(4).toString('hex')}`;
|
|
325
|
+
const logPath = path.join(jobsDir, `${id}.log`);
|
|
326
|
+
const label = req.label?.trim() || req.command.trim().split('\n')[0].slice(0, 60);
|
|
327
|
+
const job = {
|
|
328
|
+
id,
|
|
329
|
+
thread: req.thread,
|
|
330
|
+
label,
|
|
331
|
+
command: req.command,
|
|
332
|
+
cwd: req.cwd,
|
|
333
|
+
startedAt: new Date().toISOString(),
|
|
334
|
+
status: 'running',
|
|
335
|
+
logPath,
|
|
336
|
+
};
|
|
337
|
+
// A stale exit file from a recycled id would be read as this job's status.
|
|
338
|
+
try {
|
|
339
|
+
fs.unlinkSync(exitFilePath(id));
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
/* expected: no such file */
|
|
343
|
+
}
|
|
344
|
+
// stdout AND stderr go to one fd. Both matter — task 108 measured that an
|
|
345
|
+
// inherited stderr pipe is a second SIGPIPE source, and the fd here is
|
|
346
|
+
// exactly what makes both unreachable.
|
|
347
|
+
const fd = fs.openSync(logPath, 'a');
|
|
348
|
+
try {
|
|
349
|
+
const isWindows = process.platform === 'win32';
|
|
350
|
+
const child = spawn(isWindows ? req.command : wrapCommand(req.command), {
|
|
351
|
+
cwd: req.cwd,
|
|
352
|
+
shell: true,
|
|
353
|
+
detached: true,
|
|
354
|
+
stdio: ['ignore', fd, fd],
|
|
355
|
+
windowsHide: true,
|
|
356
|
+
env: {
|
|
357
|
+
...process.env,
|
|
358
|
+
CUMULUS_JOB_ID: id,
|
|
359
|
+
CUMULUS_JOB_EXIT_FILE: exitFilePath(id),
|
|
360
|
+
},
|
|
361
|
+
});
|
|
362
|
+
job.pid = child.pid;
|
|
363
|
+
if (child.pid)
|
|
364
|
+
job.startToken = readStartToken(child.pid);
|
|
365
|
+
child.on('exit', (code, signal) => {
|
|
366
|
+
finish(job, classifyExit(code ?? undefined, signal ?? undefined), code ?? undefined, signal ?? undefined);
|
|
367
|
+
});
|
|
368
|
+
child.on('error', err => {
|
|
369
|
+
log(`Job ${id} failed to start: ${err}`);
|
|
370
|
+
finish(job, 'failed', undefined);
|
|
371
|
+
});
|
|
372
|
+
// The whole point: the daemon must not wait on this, and must be able to
|
|
373
|
+
// exit without it.
|
|
374
|
+
child.unref();
|
|
375
|
+
liveChildren.set(id, child);
|
|
376
|
+
}
|
|
377
|
+
finally {
|
|
378
|
+
// The child dup'd the descriptor; holding ours open would leak one fd per
|
|
379
|
+
// job for the lifetime of the daemon.
|
|
380
|
+
fs.closeSync(fd);
|
|
381
|
+
}
|
|
382
|
+
records.push(job);
|
|
383
|
+
persist();
|
|
384
|
+
log(`Job ${id} started ("${label}")`, { thread: req.thread, pid: job.pid, cwd: req.cwd });
|
|
385
|
+
return job;
|
|
386
|
+
}
|
|
387
|
+
function runningCount(thread) {
|
|
388
|
+
return records.filter(r => r.status === 'running' && (!thread || r.thread === thread)).length;
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Check adopted jobs (those with no live child handle) for death. A job we
|
|
392
|
+
* spawned reports through its `exit` event; one inherited from a previous
|
|
393
|
+
* daemon has no event to fire, so its death is only observable by polling.
|
|
394
|
+
*/
|
|
395
|
+
function pollOnce() {
|
|
396
|
+
if (stopped)
|
|
397
|
+
return;
|
|
398
|
+
for (const job of records) {
|
|
399
|
+
if (job.status !== 'running')
|
|
400
|
+
continue;
|
|
401
|
+
if (liveChildren.has(job.id))
|
|
402
|
+
continue;
|
|
403
|
+
if (isProcessAlive(job.pid, job.startToken))
|
|
404
|
+
continue;
|
|
405
|
+
const code = readRecordedExit(job.id);
|
|
406
|
+
finish(job, code === undefined ? 'interrupted' : classifyExit(code), code);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
function adopt() {
|
|
410
|
+
loadRegistry();
|
|
411
|
+
const running = records.filter(r => r.status === 'running');
|
|
412
|
+
if (running.length === 0)
|
|
413
|
+
return;
|
|
414
|
+
let alive = 0;
|
|
415
|
+
for (const job of running) {
|
|
416
|
+
if (isProcessAlive(job.pid, job.startToken)) {
|
|
417
|
+
alive++;
|
|
418
|
+
log(`Adopted still-running job ${job.id} ("${job.label}")`, {
|
|
419
|
+
thread: job.thread,
|
|
420
|
+
pid: job.pid,
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
log(`Adoption: ${alive} of ${running.length} recorded job(s) still running`);
|
|
425
|
+
// Anything not alive is finalized here — with its recorded exit code if the
|
|
426
|
+
// job got far enough to write one, otherwise honestly as `interrupted`.
|
|
427
|
+
pollOnce();
|
|
428
|
+
}
|
|
429
|
+
loadRegistry();
|
|
430
|
+
poller = setInterval(pollOnce, pollMs);
|
|
431
|
+
poller.unref();
|
|
432
|
+
return {
|
|
433
|
+
create(req) {
|
|
434
|
+
if (runningCount(req.thread) >= MAX_RUNNING_PER_THREAD) {
|
|
435
|
+
throw new Error(`Thread already has ${MAX_RUNNING_PER_THREAD} running jobs — wait for one to finish or cancel it`);
|
|
436
|
+
}
|
|
437
|
+
if (runningCount() >= MAX_RUNNING_TOTAL) {
|
|
438
|
+
throw new Error(`Gateway already has ${MAX_RUNNING_TOTAL} running jobs`);
|
|
439
|
+
}
|
|
440
|
+
return create(req);
|
|
441
|
+
},
|
|
442
|
+
list(thread) {
|
|
443
|
+
return records.filter(r => r.thread === thread);
|
|
444
|
+
},
|
|
445
|
+
get(thread, id) {
|
|
446
|
+
return records.find(r => r.thread === thread && r.id === id);
|
|
447
|
+
},
|
|
448
|
+
cancel(thread, id) {
|
|
449
|
+
const job = records.find(r => r.thread === thread && r.id === id);
|
|
450
|
+
if (!job || job.status !== 'running')
|
|
451
|
+
return false;
|
|
452
|
+
if (job.pid) {
|
|
453
|
+
try {
|
|
454
|
+
// Negative pid = the whole process group. `detached: true` made the
|
|
455
|
+
// job a group leader, so this reaches the children it spawned too —
|
|
456
|
+
// without it, cancelling a driver script orphans everything it ran.
|
|
457
|
+
process.kill(-job.pid, 'SIGTERM');
|
|
458
|
+
}
|
|
459
|
+
catch {
|
|
460
|
+
try {
|
|
461
|
+
process.kill(job.pid, 'SIGTERM');
|
|
462
|
+
}
|
|
463
|
+
catch {
|
|
464
|
+
/* already gone */
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
finish(job, 'cancelled', undefined, undefined, false);
|
|
469
|
+
return true;
|
|
470
|
+
},
|
|
471
|
+
tail(thread, id, lines = COMPLETION_TAIL_LINES) {
|
|
472
|
+
const job = records.find(r => r.thread === thread && r.id === id);
|
|
473
|
+
if (!job)
|
|
474
|
+
return undefined;
|
|
475
|
+
return tailFile(job.logPath, lines);
|
|
476
|
+
},
|
|
477
|
+
adopt,
|
|
478
|
+
pollOnce,
|
|
479
|
+
stop() {
|
|
480
|
+
stopped = true;
|
|
481
|
+
if (poller) {
|
|
482
|
+
clearInterval(poller);
|
|
483
|
+
poller = null;
|
|
484
|
+
}
|
|
485
|
+
// Deliberately NOT killing running jobs: outliving this process is the
|
|
486
|
+
// entire feature. They are recorded, and the next daemon adopts them.
|
|
487
|
+
// Listeners ARE detached, so a stopped registry is fully inert — it must
|
|
488
|
+
// not keep writing jobs.json out from under whoever adopts next.
|
|
489
|
+
for (const child of liveChildren.values()) {
|
|
490
|
+
child.removeAllListeners('exit');
|
|
491
|
+
child.removeAllListeners('error');
|
|
492
|
+
}
|
|
493
|
+
liveChildren.clear();
|
|
494
|
+
},
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
//# sourceMappingURL=jobs.js.map
|