omp-conductor 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +732 -0
- package/package.json +40 -0
- package/skills/conductor-onboarding/SKILL.md +626 -0
- package/src/briefs/orchestrator.md +213 -0
- package/src/briefs/worker.md +146 -0
- package/src/cli.ts +179 -0
- package/src/config.ts +446 -0
- package/src/daemon.ts +689 -0
- package/src/escalate.ts +265 -0
- package/src/lifecycle.ts +367 -0
- package/src/omp.ts +273 -0
- package/src/orchestrator-tick.ts +432 -0
- package/src/orchestrator.ts +267 -0
- package/src/plugin.ts +605 -0
- package/src/routing.ts +160 -0
- package/src/setup.ts +644 -0
- package/src/store.ts +263 -0
- package/src/tracker/github.ts +160 -0
- package/src/types.ts +250 -0
- package/src/worker.ts +292 -0
- package/src/worktree.ts +303 -0
package/src/escalate.ts
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Escalation: the one path by which a stuck run reaches someone who can unstick
|
|
3
|
+
* it. Tier 1 is the orchestrator session's problem, tier 2 is the human's, and
|
|
4
|
+
* an issue comment is what is left when neither transport is reachable.
|
|
5
|
+
*
|
|
6
|
+
* Two rules shape everything here:
|
|
7
|
+
*
|
|
8
|
+
* 1. An escalation is never silently dropped. Either a transport delivered it,
|
|
9
|
+
* or `escalate()` throws so the dispatcher learns nobody is reachable. A
|
|
10
|
+
* swallowed escalation looks exactly like a healthy fleet. Tier 1 is where
|
|
11
|
+
* that is hardest: the orchestrator *accepts* an injection minutes before
|
|
12
|
+
* its turn settles, so the dedup marker waits for the settlement and a
|
|
13
|
+
* late failure falls back to an issue comment for that same escalation.
|
|
14
|
+
* 2. An escalation is never repeated. The dispatcher re-notices the same
|
|
15
|
+
* unroutable issue on every poll, so the store's notification ledger — not
|
|
16
|
+
* the loop — is what stops a human being paged every five minutes.
|
|
17
|
+
*
|
|
18
|
+
* The bot token is read at send time and never travels further than the request
|
|
19
|
+
* URL: it is redacted out of every error string this module can produce, since
|
|
20
|
+
* those strings end up in daemon logs and, on the fallback path, in a public
|
|
21
|
+
* issue comment.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { readFileSync } from "node:fs";
|
|
25
|
+
import { homedir } from "node:os";
|
|
26
|
+
import { join } from "node:path";
|
|
27
|
+
|
|
28
|
+
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
29
|
+
import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
|
|
30
|
+
|
|
31
|
+
/** Telegram rejects `sendMessage` over 4096 chars; leave room for the marker. */
|
|
32
|
+
const TELEGRAM_TEXT_LIMIT = 4000;
|
|
33
|
+
|
|
34
|
+
export interface Escalator {
|
|
35
|
+
escalate(e: Escalation): Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The human-facing text, shared by both transports so a Telegram ping and its
|
|
40
|
+
* issue-comment fallback are the same message — a human comparing the two
|
|
41
|
+
* should never have to wonder whether they describe the same event.
|
|
42
|
+
*
|
|
43
|
+
* Deliberately plain: Telegram is called without `parse_mode`, because an issue
|
|
44
|
+
* title containing `_` or `*` would otherwise make Telegram reject the whole
|
|
45
|
+
* send, turning a cosmetic problem into a lost escalation. Plain text is also
|
|
46
|
+
* valid Markdown, so the same string renders fine as an issue comment.
|
|
47
|
+
*/
|
|
48
|
+
export function formatEscalation(e: Escalation, project: string): string {
|
|
49
|
+
const lines = [
|
|
50
|
+
`omp-conductor · tier ${e.tier} escalation`,
|
|
51
|
+
`project: ${project}`,
|
|
52
|
+
`issue: #${e.issue}`,
|
|
53
|
+
`summary: ${e.summary}`,
|
|
54
|
+
];
|
|
55
|
+
if (e.detail) lines.push(`detail: ${e.detail}`);
|
|
56
|
+
if (e.runId) lines.push(`run: ${e.runId}`);
|
|
57
|
+
return lines.join("\n");
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* `orchestrator` is the tier-1 transport when one is running: a tier-1
|
|
62
|
+
* escalation is the orchestrator's problem, not the human's, and an injected
|
|
63
|
+
* prompt is the only form of it that can actually change anything. It stays
|
|
64
|
+
* optional so a daemon whose orchestrator failed to start — and the unit suite —
|
|
65
|
+
* still escalate, just to an issue comment.
|
|
66
|
+
*/
|
|
67
|
+
export function createEscalator(
|
|
68
|
+
p: ProjectConfig,
|
|
69
|
+
tracker: Tracker,
|
|
70
|
+
store: Store,
|
|
71
|
+
orchestrator?: OrchestratorHandle,
|
|
72
|
+
): Escalator {
|
|
73
|
+
return {
|
|
74
|
+
async escalate(e: Escalation): Promise<void> {
|
|
75
|
+
// Stable across daemon restarts: same project, issue, tier and summary is
|
|
76
|
+
// the same event, however many times the loop rediscovers it.
|
|
77
|
+
const key = `${p.name}:${e.issue}:${e.tier}:${e.summary}`;
|
|
78
|
+
if (store.wasNotified(key)) return;
|
|
79
|
+
|
|
80
|
+
const text = formatEscalation(e, p.name);
|
|
81
|
+
const chatId = p.escalation.telegramChatId;
|
|
82
|
+
|
|
83
|
+
if (e.tier === 1 && orchestrator) {
|
|
84
|
+
// Resolves on acceptance, not on an answer, so this does not park the
|
|
85
|
+
// tick behind a model. A rejection means the injection was never taken
|
|
86
|
+
// at all: `undefined` falls through to the human-facing path below
|
|
87
|
+
// rather than losing the escalation — a tier-1 event that reached
|
|
88
|
+
// nobody is indistinguishable from a healthy fleet.
|
|
89
|
+
const receipt = await orchestrator.deliver(e, p.name).catch(() => undefined);
|
|
90
|
+
if (receipt) {
|
|
91
|
+
/**
|
|
92
|
+
* This escalation's own late failure, routed to this escalation's own
|
|
93
|
+
* fallback. The key stays unmarked unless something actually lands:
|
|
94
|
+
* an unmarked key means the next tick re-escalates, which is the
|
|
95
|
+
* whole difference between a late escalation and a lost one. Marking
|
|
96
|
+
* on acceptance is what used to drop it — "notified" written over an
|
|
97
|
+
* event no human ever read, and nothing left to retry it.
|
|
98
|
+
*/
|
|
99
|
+
const onTurnFailed = async (cause: unknown): Promise<void> => {
|
|
100
|
+
if (!p.escalation.fallbackToIssueComment) {
|
|
101
|
+
warn(
|
|
102
|
+
`tier 1 escalation on issue #${e.issue} was accepted by the orchestrator but its ` +
|
|
103
|
+
`turn failed (${errText(cause)}), and no fallback is configured — left unmarked ` +
|
|
104
|
+
`so the next tick retries`,
|
|
105
|
+
);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
try {
|
|
109
|
+
await tracker.comment(e.issue, text);
|
|
110
|
+
store.markNotified(key);
|
|
111
|
+
} catch (err) {
|
|
112
|
+
warn(
|
|
113
|
+
`tier 1 escalation on issue #${e.issue} failed after acceptance ` +
|
|
114
|
+
`(${errText(cause)}) and its issue-comment fallback failed too ` +
|
|
115
|
+
`(${errText(err)}) — left unmarked so the next tick retries`,
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
// Acceptance is not delivery: the orchestrator's turn for *this*
|
|
121
|
+
// injection can still fail minutes from now, so the marker waits for
|
|
122
|
+
// `settled`. Deliberately not awaited — `settled` resolves only when
|
|
123
|
+
// the model has finished answering, which is exactly the wait the
|
|
124
|
+
// dispatcher tick must not take.
|
|
125
|
+
void receipt.settled
|
|
126
|
+
.then(() => {
|
|
127
|
+
store.markNotified(key);
|
|
128
|
+
}, onTurnFailed)
|
|
129
|
+
.catch(() => {
|
|
130
|
+
// `markNotified` is the only thing above that can still throw,
|
|
131
|
+
// and a failing store write must not become an unhandled
|
|
132
|
+
// rejection in a daemon that has nobody left to throw at.
|
|
133
|
+
});
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if (e.tier === 2 && chatId) {
|
|
139
|
+
const token = readTelegramToken();
|
|
140
|
+
if (token) {
|
|
141
|
+
// A send failure throws: `markNotified` stays uncalled so the next
|
|
142
|
+
// poll retries instead of writing the event off as delivered.
|
|
143
|
+
// ponytail: no retry/backoff in here — the dispatcher tick is the
|
|
144
|
+
// retry. Upgrade path is a durable outbox table in the store.
|
|
145
|
+
await sendTelegram(token, chatId, text);
|
|
146
|
+
store.markNotified(key);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Tier 1 lands here with no orchestrator, or with one that would not take
|
|
152
|
+
// the injection; tier 2 lands here when omp-telegram is not installed or
|
|
153
|
+
// the project never configured a chat id.
|
|
154
|
+
if (!p.escalation.fallbackToIssueComment) {
|
|
155
|
+
throw new Error(
|
|
156
|
+
`no escalation transport configured for project "${p.name}": tier ${e.tier} ` +
|
|
157
|
+
`escalation on issue #${e.issue} (${e.summary}) could not be delivered — ` +
|
|
158
|
+
`set escalation.telegramChatId or escalation.fallbackToIssueComment`,
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
await tracker.comment(e.issue, text);
|
|
162
|
+
store.markNotified(key);
|
|
163
|
+
},
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* The tier-1 settlement tail runs after `escalate()` has already returned, so a
|
|
169
|
+
* failure there has no caller left to throw at. stderr is where it can still be
|
|
170
|
+
* seen: the daemon's log *is* its stderr, so these land in `daemon.log` beside
|
|
171
|
+
* every other conductor line.
|
|
172
|
+
*
|
|
173
|
+
* ponytail: duplicates daemon.ts's one-line format rather than sharing a
|
|
174
|
+
* logger. Upgrade path is a `log.ts` both modules import.
|
|
175
|
+
*/
|
|
176
|
+
function warn(msg: string): void {
|
|
177
|
+
process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function errText(e: unknown): string {
|
|
181
|
+
return e instanceof Error ? e.message : String(e);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* omp-telegram owns `<state dir>/.env`; conductor only borrows the token, so a
|
|
186
|
+
* user already running that bot gets tier-2 pings with no extra configuration.
|
|
187
|
+
* Absence is not an error — it just means tier 2 degrades to the fallback.
|
|
188
|
+
*/
|
|
189
|
+
function readTelegramToken(): string | undefined {
|
|
190
|
+
const override = process.env.OMP_TELEGRAM_STATE_DIR?.trim();
|
|
191
|
+
const dir = override ? override : join(homedir(), ".omp", "agent", "telegram");
|
|
192
|
+
let raw: string;
|
|
193
|
+
try {
|
|
194
|
+
raw = readFileSync(join(dir, ".env"), "utf8");
|
|
195
|
+
} catch {
|
|
196
|
+
return undefined;
|
|
197
|
+
}
|
|
198
|
+
for (const line of raw.split("\n")) {
|
|
199
|
+
const trimmed = line.trim();
|
|
200
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
201
|
+
const match = /^(?:export\s+)?TELEGRAM_BOT_TOKEN\s*=\s*(.*)$/.exec(trimmed);
|
|
202
|
+
if (!match) continue;
|
|
203
|
+
let value = (match[1] ?? "").trim();
|
|
204
|
+
const quote = value[0];
|
|
205
|
+
if (value.length >= 2 && (quote === '"' || quote === "'") && value.endsWith(quote)) {
|
|
206
|
+
value = value.slice(1, -1);
|
|
207
|
+
}
|
|
208
|
+
if (value) return value;
|
|
209
|
+
}
|
|
210
|
+
return undefined;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async function sendTelegram(token: string, chatId: string, text: string): Promise<void> {
|
|
214
|
+
const url = `https://api.telegram.org/bot${token}/sendMessage`;
|
|
215
|
+
const body = JSON.stringify({
|
|
216
|
+
chat_id: chatId,
|
|
217
|
+
// ponytail: hard truncation rather than splitting across messages — the
|
|
218
|
+
// tail of a stack trace is rarely the interesting part. Upgrade path is to
|
|
219
|
+
// attach the overflow as a file via sendDocument.
|
|
220
|
+
text:
|
|
221
|
+
text.length > TELEGRAM_TEXT_LIMIT ? `${text.slice(0, TELEGRAM_TEXT_LIMIT)}\n[truncated]` : text,
|
|
222
|
+
disable_web_page_preview: true,
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
let res: Response;
|
|
226
|
+
try {
|
|
227
|
+
res = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body });
|
|
228
|
+
} catch (cause) {
|
|
229
|
+
// The URL embeds the token, and runtimes love to quote the failing URL back
|
|
230
|
+
// at you — redact before this string reaches a log or an issue comment.
|
|
231
|
+
const nested = cause instanceof Error && cause.cause instanceof Error ? `: ${cause.cause.message}` : "";
|
|
232
|
+
const reason = cause instanceof Error ? `${cause.message}${nested}` : String(cause);
|
|
233
|
+
throw new Error(`telegram sendMessage failed: ${redact(reason, token)}`);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const payload = redact(await res.text().catch(() => ""), token).slice(0, 400);
|
|
237
|
+
if (!res.ok) {
|
|
238
|
+
throw new Error(`telegram sendMessage failed: HTTP ${res.status} ${payload}`);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Telegram answers 200 with `{"ok":false}` for plenty of real failures
|
|
242
|
+
// (kicked from the chat, bad chat_id), so the status alone proves nothing.
|
|
243
|
+
let ok = false;
|
|
244
|
+
try {
|
|
245
|
+
const parsed: unknown = JSON.parse(payload);
|
|
246
|
+
ok = typeof parsed === "object" && parsed !== null && "ok" in parsed && parsed.ok === true;
|
|
247
|
+
} catch {
|
|
248
|
+
ok = false;
|
|
249
|
+
}
|
|
250
|
+
if (!ok) {
|
|
251
|
+
throw new Error(`telegram sendMessage rejected: ${payload}`);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Strips the bot token — and its secret half, which some proxies log on its
|
|
257
|
+
* own — out of any string that is about to be thrown, logged or commented.
|
|
258
|
+
*/
|
|
259
|
+
function redact(text: string, token: string): string {
|
|
260
|
+
let out = token ? text.split(token).join("<redacted>") : text;
|
|
261
|
+
const colon = token.indexOf(":");
|
|
262
|
+
const secret = colon >= 0 ? token.slice(colon + 1) : "";
|
|
263
|
+
if (secret.length >= 8) out = out.split(secret).join("<redacted>");
|
|
264
|
+
return out;
|
|
265
|
+
}
|
package/src/lifecycle.ts
ADDED
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lifecycle for the standalone daemon: start it in the background, prove it
|
|
3
|
+
* came up, stop it, and answer "is it running?" honestly.
|
|
4
|
+
*
|
|
5
|
+
* The record under the runtime directory is a hint, never a fact. A pidfile
|
|
6
|
+
* outlives the process that wrote it — a crash, a reboot, an OOM kill all
|
|
7
|
+
* leave one behind — and pids are recycled, so the file's *existence* says
|
|
8
|
+
* nothing. Every read probes the pid before believing the record; a stale
|
|
9
|
+
* entry is cleared rather than honoured, because the two failures that cost an
|
|
10
|
+
* operator real time are `start` refusing against a ghost and `status`
|
|
11
|
+
* reporting a daemon that died hours ago.
|
|
12
|
+
*
|
|
13
|
+
* Deliberately free of every other module in this package: nothing here opens
|
|
14
|
+
* the store, loads the config or talks to `gh`, so `stop` and `status` keep
|
|
15
|
+
* working when the config is the very thing that is broken.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { spawn } from "node:child_process";
|
|
19
|
+
import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
20
|
+
import { homedir } from "node:os";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Mirrors `DEFAULT_PORT` in ./daemon.ts. Duplicated rather than imported so
|
|
25
|
+
* this module stays free of the dispatcher's dependency tree.
|
|
26
|
+
* // ponytail: two constants that must agree. If a third caller ever needs it,
|
|
27
|
+
* // move it into ./types.ts and import it in both places.
|
|
28
|
+
*/
|
|
29
|
+
const DEFAULT_PORT = 8787;
|
|
30
|
+
|
|
31
|
+
/** How long `startDaemon` waits for the first successful `/healthz`. */
|
|
32
|
+
const READY_TIMEOUT_MS = 15_000;
|
|
33
|
+
const READY_POLL_MS = 250;
|
|
34
|
+
|
|
35
|
+
/** `/healthz` is a local, in-memory answer; a slow one means something is wrong. */
|
|
36
|
+
const HEALTH_TIMEOUT_MS = 1_500;
|
|
37
|
+
|
|
38
|
+
/** Default grace period between `SIGTERM` and `SIGKILL`. */
|
|
39
|
+
const STOP_TIMEOUT_MS = 10_000;
|
|
40
|
+
|
|
41
|
+
/** Bytes of `daemon.log` quoted back when the daemon dies during boot. */
|
|
42
|
+
const LOG_TAIL_BYTES = 4_096;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Where the pidfile and the log live. Sits under `~/.omp/run/daemons/` with
|
|
46
|
+
* the other omp daemons rather than beside the config, because this is
|
|
47
|
+
* runtime state that is meaningless after a reboot — the config directory is
|
|
48
|
+
* for things worth keeping.
|
|
49
|
+
*/
|
|
50
|
+
export function runtimeDir(): string {
|
|
51
|
+
const override = process.env["OMP_CONDUCTOR_RUNTIME_DIR"];
|
|
52
|
+
if (override !== undefined && override.length > 0) return override;
|
|
53
|
+
return join(homedir(), ".omp", "run", "daemons", "omp-conductor");
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** What a running daemon was started with, as far as the pidfile knows. */
|
|
57
|
+
export interface DaemonRecord {
|
|
58
|
+
pid: number;
|
|
59
|
+
port: number;
|
|
60
|
+
project?: string;
|
|
61
|
+
startedAt: number;
|
|
62
|
+
logFile: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function recordPath(): string {
|
|
66
|
+
return join(runtimeDir(), "daemon.json");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Creates the runtime directory 0700; only chmods what this call created. */
|
|
70
|
+
function ensureRuntimeDir(): string {
|
|
71
|
+
const dir = runtimeDir();
|
|
72
|
+
const created = mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
73
|
+
// mkdir's mode is masked by umask, so fix up what we made; an existing
|
|
74
|
+
// directory keeps whatever the operator chose for it.
|
|
75
|
+
if (created !== undefined) chmodSync(dir, 0o700);
|
|
76
|
+
return dir;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Reads the pidfile. Every failure — absent, unreadable, truncated mid-write,
|
|
81
|
+
* hand-edited into nonsense — is the same answer: `undefined`. A lifecycle
|
|
82
|
+
* command that throws because a runtime file is garbage is a lifecycle command
|
|
83
|
+
* that cannot clean up after itself.
|
|
84
|
+
*/
|
|
85
|
+
export function readRecord(): DaemonRecord | undefined {
|
|
86
|
+
let parsed: unknown;
|
|
87
|
+
try {
|
|
88
|
+
parsed = JSON.parse(readFileSync(recordPath(), "utf8"));
|
|
89
|
+
} catch {
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
if (parsed === null || typeof parsed !== "object") return undefined;
|
|
93
|
+
const r = parsed as Record<string, unknown>;
|
|
94
|
+
|
|
95
|
+
const pid = r["pid"];
|
|
96
|
+
const port = r["port"];
|
|
97
|
+
const startedAt = r["startedAt"];
|
|
98
|
+
const logFile = r["logFile"];
|
|
99
|
+
const project = r["project"];
|
|
100
|
+
|
|
101
|
+
// pid 0 and 1 are never ours: 0 means "this process group" to `kill`, which
|
|
102
|
+
// would make a corrupt file signal the whole group.
|
|
103
|
+
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 1) return undefined;
|
|
104
|
+
if (typeof port !== "number" || !Number.isInteger(port) || port < 1 || port > 65535) return undefined;
|
|
105
|
+
if (typeof startedAt !== "number" || !Number.isFinite(startedAt)) return undefined;
|
|
106
|
+
if (typeof logFile !== "string" || logFile.length === 0) return undefined;
|
|
107
|
+
if (project !== undefined && typeof project !== "string") return undefined;
|
|
108
|
+
|
|
109
|
+
return { pid, port, startedAt, logFile, ...(project === undefined ? {} : { project }) };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Writes the pidfile atomically — temp file, then rename — so a reader never
|
|
114
|
+
* catches a half-written record and concludes the daemon is gone. Mode 0600:
|
|
115
|
+
* it names a process another user has no business signalling.
|
|
116
|
+
*/
|
|
117
|
+
export function writeRecord(r: DaemonRecord): void {
|
|
118
|
+
const dir = ensureRuntimeDir();
|
|
119
|
+
const target = recordPath();
|
|
120
|
+
const tmp = join(dir, `.daemon.json.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`);
|
|
121
|
+
try {
|
|
122
|
+
writeFileSync(tmp, `${JSON.stringify(r, null, 2)}\n`, { mode: 0o600 });
|
|
123
|
+
renameSync(tmp, target);
|
|
124
|
+
} catch (err) {
|
|
125
|
+
rmSync(tmp, { force: true });
|
|
126
|
+
throw err;
|
|
127
|
+
}
|
|
128
|
+
chmodSync(target, 0o600);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Removes the pidfile. Absent is success — this is how staleness is repaired. */
|
|
132
|
+
export function clearRecord(): void {
|
|
133
|
+
rmSync(recordPath(), { force: true });
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Whether a pid names a live process. Signal 0 performs the permission and
|
|
138
|
+
* existence checks without delivering anything: `ESRCH` is dead, `EPERM` is
|
|
139
|
+
* alive but owned by somebody else — still alive, so still a reason not to
|
|
140
|
+
* start a second daemon.
|
|
141
|
+
*/
|
|
142
|
+
export function isAlive(pid: number): boolean {
|
|
143
|
+
if (!Number.isInteger(pid) || pid <= 1) return false;
|
|
144
|
+
try {
|
|
145
|
+
process.kill(pid, 0);
|
|
146
|
+
return true;
|
|
147
|
+
} catch (err) {
|
|
148
|
+
return (err as NodeJS.ErrnoException).code === "EPERM";
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The record, but only when the process it names is actually running. A dead
|
|
154
|
+
* pid clears the file on the way out, so the next `start` is not blocked by a
|
|
155
|
+
* daemon that stopped existing three reboots ago.
|
|
156
|
+
*/
|
|
157
|
+
export function livingDaemon(): DaemonRecord | undefined {
|
|
158
|
+
const rec = readRecord();
|
|
159
|
+
if (rec === undefined) return undefined;
|
|
160
|
+
if (isAlive(rec.pid)) return rec;
|
|
161
|
+
clearRecord();
|
|
162
|
+
return undefined;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Probes the daemon's own health endpoint. Never throws: a refused connection,
|
|
167
|
+
* a DNS-less host, a hung socket and a 500 are all just "not healthy", and the
|
|
168
|
+
* callers of this are the ones responsible for saying so nicely.
|
|
169
|
+
*/
|
|
170
|
+
export async function healthCheck(port: number): Promise<{ ok: boolean; body?: string }> {
|
|
171
|
+
try {
|
|
172
|
+
const res = await fetch(`http://127.0.0.1:${port}/healthz`, {
|
|
173
|
+
signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS),
|
|
174
|
+
});
|
|
175
|
+
const body = (await res.text()).trim();
|
|
176
|
+
return { ok: res.ok, ...(body.length > 0 ? { body } : {}) };
|
|
177
|
+
} catch {
|
|
178
|
+
return { ok: false };
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Starts the daemon in the background and does not return until it answers
|
|
184
|
+
* `/healthz`.
|
|
185
|
+
*
|
|
186
|
+
* Spawning is not starting. A daemon whose config is broken, whose port is
|
|
187
|
+
* taken or whose database is locked exits within a second of being spawned,
|
|
188
|
+
* and a `start` that printed "started" for it hands the operator a lie they
|
|
189
|
+
* only discover when work silently fails to be picked up. So: spawn, watch
|
|
190
|
+
* both the pid and the endpoint, and fail loudly with the log if either says
|
|
191
|
+
* no.
|
|
192
|
+
*/
|
|
193
|
+
export async function startDaemon(o: { port?: number; project?: string } = {}): Promise<DaemonRecord> {
|
|
194
|
+
const running = livingDaemon();
|
|
195
|
+
if (running !== undefined) {
|
|
196
|
+
throw new Error(
|
|
197
|
+
`daemon already running (pid ${running.pid}, port ${running.port}) — ` +
|
|
198
|
+
`use "omp-conductor restart" to replace it, or "omp-conductor stop" first`,
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const port = o.port ?? DEFAULT_PORT;
|
|
203
|
+
const logFile = join(ensureRuntimeDir(), "daemon.log");
|
|
204
|
+
|
|
205
|
+
// Append, never truncate: the previous boot's failure is usually the reason
|
|
206
|
+
// somebody is running `start` again.
|
|
207
|
+
const logFd = openSync(logFile, "a", 0o600);
|
|
208
|
+
let pid: number | undefined;
|
|
209
|
+
try {
|
|
210
|
+
const child = spawn(
|
|
211
|
+
process.execPath,
|
|
212
|
+
[join(import.meta.dir, "cli.ts"), "daemon", "--port", String(port), ...(o.project === undefined ? [] : ["--project", o.project])],
|
|
213
|
+
{
|
|
214
|
+
// The runtime directory is one we know exists and will not be deleted
|
|
215
|
+
// out from under a long-running process; the daemon itself resolves
|
|
216
|
+
// every path from the config, so cwd is only about not holding a
|
|
217
|
+
// stale directory open.
|
|
218
|
+
cwd: runtimeDir(),
|
|
219
|
+
detached: true,
|
|
220
|
+
stdio: ["ignore", logFd, logFd],
|
|
221
|
+
},
|
|
222
|
+
);
|
|
223
|
+
child.once("error", (err) => {
|
|
224
|
+
process.stderr.write(`omp-conductor: daemon process failed: ${err.message}\n`);
|
|
225
|
+
});
|
|
226
|
+
// Detach from the parent's event loop and process group so the daemon
|
|
227
|
+
// outlives the shell that started it.
|
|
228
|
+
child.unref();
|
|
229
|
+
pid = child.pid;
|
|
230
|
+
} finally {
|
|
231
|
+
closeSync(logFd);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (pid === undefined) throw new Error(`could not spawn the daemon; see ${logFile}`);
|
|
235
|
+
|
|
236
|
+
const record: DaemonRecord = {
|
|
237
|
+
pid,
|
|
238
|
+
port,
|
|
239
|
+
startedAt: Date.now(),
|
|
240
|
+
logFile,
|
|
241
|
+
...(o.project === undefined ? {} : { project: o.project }),
|
|
242
|
+
};
|
|
243
|
+
// Written before the wait so a concurrent `status` sees a booting daemon
|
|
244
|
+
// rather than nothing at all.
|
|
245
|
+
writeRecord(record);
|
|
246
|
+
|
|
247
|
+
const deadline = Date.now() + READY_TIMEOUT_MS;
|
|
248
|
+
for (;;) {
|
|
249
|
+
// Liveness first. If the child is gone, a healthy answer on that port came
|
|
250
|
+
// from somebody else's server, and reporting it as ours would be worse
|
|
251
|
+
// than reporting nothing.
|
|
252
|
+
if (!isAlive(pid)) {
|
|
253
|
+
clearRecord();
|
|
254
|
+
throw new Error(`daemon exited during startup${tailLog(logFile)}`);
|
|
255
|
+
}
|
|
256
|
+
const health = await healthCheck(port);
|
|
257
|
+
if (health.ok) return record;
|
|
258
|
+
if (Date.now() >= deadline) break;
|
|
259
|
+
await sleep(READY_POLL_MS);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// Alive but never healthy: a wedged process is not a running daemon, and
|
|
263
|
+
// leaving it behind would block the next `start` for no benefit.
|
|
264
|
+
await terminate(pid, STOP_TIMEOUT_MS);
|
|
265
|
+
clearRecord();
|
|
266
|
+
throw new Error(
|
|
267
|
+
`daemon did not answer http://127.0.0.1:${port}/healthz within ${Math.round(READY_TIMEOUT_MS / 1000)}s${tailLog(logFile)}`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Stops the daemon: `SIGTERM`, wait, then `SIGKILL`.
|
|
273
|
+
*
|
|
274
|
+
* `SIGTERM` asks the loop to finish the tick it is on, and a tick that has a
|
|
275
|
+
* worker in flight can run for the worker's whole wall clock. The grace period
|
|
276
|
+
* is therefore a deadline, not a promise of a clean drain.
|
|
277
|
+
* // ponytail: no way to say "stop when the current worker lands". If that
|
|
278
|
+
* // matters, take a `--timeout` on the CLI verb, or poll `activeRuns` before
|
|
279
|
+
* // escalating.
|
|
280
|
+
*/
|
|
281
|
+
export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<"stopped" | "not-running"> {
|
|
282
|
+
const rec = livingDaemon();
|
|
283
|
+
if (rec === undefined) {
|
|
284
|
+
// `livingDaemon` already cleared a stale file; this covers the unparseable
|
|
285
|
+
// one it refused to read.
|
|
286
|
+
clearRecord();
|
|
287
|
+
return "not-running";
|
|
288
|
+
}
|
|
289
|
+
const gone = await terminate(rec.pid, o.timeoutMs ?? STOP_TIMEOUT_MS);
|
|
290
|
+
if (!gone) {
|
|
291
|
+
// The record stays: something is still holding that pid, and forgetting
|
|
292
|
+
// about it would let the next `start` bind a port that is already taken.
|
|
293
|
+
throw new Error(`daemon pid ${rec.pid} survived SIGTERM and SIGKILL — it may belong to another user`);
|
|
294
|
+
}
|
|
295
|
+
clearRecord();
|
|
296
|
+
return "stopped";
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// ---------------------------------------------------------------------------
|
|
300
|
+
// internals
|
|
301
|
+
// ---------------------------------------------------------------------------
|
|
302
|
+
|
|
303
|
+
function sleep(ms: number): Promise<void> {
|
|
304
|
+
return new Promise<void>((resolve) => {
|
|
305
|
+
const t = setTimeout(resolve, ms);
|
|
306
|
+
t.unref?.();
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** `SIGTERM`, poll until dead, then `SIGKILL`. Returns whether the pid is gone. */
|
|
311
|
+
async function terminate(pid: number, timeoutMs: number): Promise<boolean> {
|
|
312
|
+
if (!isAlive(pid)) return true;
|
|
313
|
+
if (!signal(pid, "SIGTERM")) return !isAlive(pid);
|
|
314
|
+
|
|
315
|
+
const deadline = Date.now() + timeoutMs;
|
|
316
|
+
while (isAlive(pid) && Date.now() < deadline) await sleep(100);
|
|
317
|
+
if (!isAlive(pid)) return true;
|
|
318
|
+
|
|
319
|
+
signal(pid, "SIGKILL");
|
|
320
|
+
// SIGKILL is not instant — the kernel still has to reap it — so give the
|
|
321
|
+
// pid a moment to disappear before anyone reads liveness again.
|
|
322
|
+
const hardDeadline = Date.now() + 2_000;
|
|
323
|
+
while (isAlive(pid) && Date.now() < hardDeadline) await sleep(50);
|
|
324
|
+
return !isAlive(pid);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** Sends a signal, treating "already gone" as success. Returns false if the send failed. */
|
|
328
|
+
function signal(pid: number, sig: NodeJS.Signals): boolean {
|
|
329
|
+
try {
|
|
330
|
+
process.kill(pid, sig);
|
|
331
|
+
return true;
|
|
332
|
+
} catch (err) {
|
|
333
|
+
return (err as NodeJS.ErrnoException).code === "ESRCH";
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* The tail of the daemon log, formatted for an error message. Reads the last
|
|
339
|
+
* few KB by offset rather than the whole file, because the log is append-only
|
|
340
|
+
* across every boot and can be arbitrarily long.
|
|
341
|
+
*/
|
|
342
|
+
function tailLog(path: string): string {
|
|
343
|
+
let text: string;
|
|
344
|
+
let truncated = false;
|
|
345
|
+
try {
|
|
346
|
+
const size = statSync(path).size;
|
|
347
|
+
const want = Math.min(size, LOG_TAIL_BYTES);
|
|
348
|
+
if (want === 0) return ` — nothing was written to ${path}`;
|
|
349
|
+
truncated = want < size;
|
|
350
|
+
const fd = openSync(path, "r");
|
|
351
|
+
try {
|
|
352
|
+
const buf = Buffer.allocUnsafe(want);
|
|
353
|
+
const read = readSync(fd, buf, 0, want, size - want);
|
|
354
|
+
text = buf.subarray(0, read).toString("utf8");
|
|
355
|
+
} finally {
|
|
356
|
+
closeSync(fd);
|
|
357
|
+
}
|
|
358
|
+
} catch {
|
|
359
|
+
return ` — see ${path}`;
|
|
360
|
+
}
|
|
361
|
+
// Reading from an offset lands mid-line; that fragment is an artefact, not
|
|
362
|
+
// output. Reading the whole file does not, so keep line one in that case.
|
|
363
|
+
const all = text.split("\n");
|
|
364
|
+
const lines = (truncated ? all.slice(1) : all).filter((l) => l.trim().length > 0);
|
|
365
|
+
if (lines.length === 0) return ` — see ${path}`;
|
|
366
|
+
return `:\n${lines.slice(-10).join("\n")}\n(full log: ${path})`;
|
|
367
|
+
}
|