@crewhaus/gateway-protocol 0.4.2 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control-codegen.d.ts +125 -0
- package/dist/control-codegen.js +135 -0
- package/dist/control.d.ts +329 -0
- package/dist/control.js +704 -0
- package/dist/index.d.ts +10 -10
- package/package.json +11 -2
package/dist/control.js
ADDED
|
@@ -0,0 +1,704 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `crewhaus.control.v1` — the daemon control plane every daemon-shape bundle
|
|
3
|
+
* serves.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS EXISTS. A compiled daemon's schedulers are in-process
|
|
6
|
+
* (`setInterval` for `heartbeat:`, `armSchedule` for `schedule:`), so from
|
|
7
|
+
* the outside there is no way to ask "when does the next heartbeat fire?" and
|
|
8
|
+
* no way to make one fire now: the phase of a heartbeat is knowable ONLY
|
|
9
|
+
* inside the process that armed it. A supervisor that wants to drive — not
|
|
10
|
+
* just watch — a fleet of daemons needs a uniform, signal-free surface. This
|
|
11
|
+
* module is that surface, written ONCE and consumed by every daemon-emitting
|
|
12
|
+
* target (channel, managed, batch, crew, voice) so the five shapes can never
|
|
13
|
+
* drift apart.
|
|
14
|
+
*
|
|
15
|
+
* SHAPE OF THE CONTRACT.
|
|
16
|
+
* - A DEDICATED control port, separate from any public webhook/gateway
|
|
17
|
+
* port. Default bind `127.0.0.1`; the port comes from
|
|
18
|
+
* `CREWHAUS_CONTROL_PORT` (`0` asks the kernel for an ephemeral port,
|
|
19
|
+
* which is then reported on stdout). Unset ⇒ no control socket at all,
|
|
20
|
+
* so upgrading a bundle never opens a listener nobody asked for.
|
|
21
|
+
* Exposing it on a PaaS is an explicit opt-in: set the bind + token as
|
|
22
|
+
* provider secrets at deploy time.
|
|
23
|
+
* - Bearer auth against `CREWHAUS_CONTROL_TOKEN`, or a token minted at boot
|
|
24
|
+
* into `<cwd>/.crewhaus/run/control-token` (0600) so a local manager can
|
|
25
|
+
* read it off disk. Compared in constant time; never logged, never
|
|
26
|
+
* echoed, never written to an audit payload.
|
|
27
|
+
* - `GET /control/v1/healthz` → `{ok, name, target}`
|
|
28
|
+
* - `GET /control/v1/status` → counters + per-lane timers + channels +
|
|
29
|
+
* pending approvals
|
|
30
|
+
* - `POST /control/v1/wake` → one synthetic tick down the IDENTICAL code
|
|
31
|
+
* path as the timer fire
|
|
32
|
+
* - `POST /control/v1/drain` → stop intake, finish in-flight work, exit 0
|
|
33
|
+
* - Every call appends a `gateway_request` record to the harness's
|
|
34
|
+
* hash-chained audit log when one is wired.
|
|
35
|
+
*
|
|
36
|
+
* Separately — and INDEPENDENT of whether the control port is bound — a bare,
|
|
37
|
+
* unauthenticated `GET /healthz` is served on the daemon's PUBLIC port when it
|
|
38
|
+
* has one ({@link ControlPlane.publicGate}). Deployment scaffolds declare that
|
|
39
|
+
* health check but no daemon served it; the liveness answer carries no state,
|
|
40
|
+
* so closing that gap never exposes control.
|
|
41
|
+
*
|
|
42
|
+
* TESTABILITY. `fetch` is a plain `Request → Response` function, so the whole
|
|
43
|
+
* router (auth, wake, 409-while-in-flight, drain) is exercisable with no
|
|
44
|
+
* socket, no timers and no daemon. `start()` is the only part that touches
|
|
45
|
+
* `Bun.serve`.
|
|
46
|
+
*/
|
|
47
|
+
import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
|
48
|
+
import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
49
|
+
import { join } from "node:path";
|
|
50
|
+
import { openEventLog } from "@crewhaus/event-log";
|
|
51
|
+
/** Wire identifier for this control protocol. Bumping it is a breaking change. */
|
|
52
|
+
export const CONTROL_PROTOCOL = "crewhaus.control.v1";
|
|
53
|
+
/** Every control route lives under this prefix. */
|
|
54
|
+
export const CONTROL_PATH_PREFIX = "/control/v1";
|
|
55
|
+
/** Harness-local ops directory. All daemon-written control state lives here. */
|
|
56
|
+
export const CONTROL_RUN_DIR = ".crewhaus/run";
|
|
57
|
+
/** File the boot-minted bearer token lands in, mode 0600. */
|
|
58
|
+
export const CONTROL_TOKEN_FILENAME = "control-token";
|
|
59
|
+
export const CONTROL_BIND_ENV = "CREWHAUS_CONTROL_BIND";
|
|
60
|
+
export const CONTROL_PORT_ENV = "CREWHAUS_CONTROL_PORT";
|
|
61
|
+
export const CONTROL_TOKEN_ENV = "CREWHAUS_CONTROL_TOKEN";
|
|
62
|
+
/** Loopback-only unless the operator explicitly widens it. */
|
|
63
|
+
export const DEFAULT_CONTROL_BIND = "127.0.0.1";
|
|
64
|
+
/**
|
|
65
|
+
* Longest free-text a control-plane field (`reason`, `by`) may carry into a
|
|
66
|
+
* daemon's prose log, and the default cap {@link sanitizeControlText} applies.
|
|
67
|
+
*/
|
|
68
|
+
export const CONTROL_TEXT_MAX = 200;
|
|
69
|
+
/**
|
|
70
|
+
* Budget (ms) for a best-effort housekeeping step run during a drain, and the
|
|
71
|
+
* env var that widens it. `0` skips the step entirely.
|
|
72
|
+
*/
|
|
73
|
+
export const DRAIN_SWEEP_BUDGET_ENV = "CREWHAUS_DRAIN_SWEEP_MS";
|
|
74
|
+
export const DEFAULT_DRAIN_SWEEP_MS = 5_000;
|
|
75
|
+
/** `Retry-After` (seconds) on the 503 a draining daemon answers intake with. */
|
|
76
|
+
export const DRAIN_RETRY_AFTER_SECONDS = 15;
|
|
77
|
+
/**
|
|
78
|
+
* Flatten untrusted text into ONE printable log-safe line.
|
|
79
|
+
*
|
|
80
|
+
* WHY THIS EXISTS. A daemon's stdout/stderr is not just for humans: the
|
|
81
|
+
* manager PARSES it — the `[control] crewhaus.control.v1 listening on
|
|
82
|
+
* http://host:port` announcement is the only way it learns a kernel-assigned
|
|
83
|
+
* control port. Every prose line a daemon prints that interpolates text the
|
|
84
|
+
* daemon did not author is therefore a log-injection surface: an operator's
|
|
85
|
+
* `reason`, and — strictly worse, because it needs no operator at all — the
|
|
86
|
+
* AGENT's own turn output, which a channel message can steer. A single
|
|
87
|
+
* newline in either would let that text START a line, and a forged
|
|
88
|
+
* announcement line is enough to repoint the manager's control calls (bearer
|
|
89
|
+
* included) at an attacker-chosen port.
|
|
90
|
+
*
|
|
91
|
+
* So: every control character (C0, DEL + C1) and every Unicode line separator
|
|
92
|
+
* is replaced with a space, whitespace runs collapse, and the result is capped
|
|
93
|
+
* — one line, bounded, no matter what went in. Anchoring the manager's own
|
|
94
|
+
* announcement regex to a line start is the other half of the same fix; this
|
|
95
|
+
* half is what makes the anchor hold, because it guarantees untrusted text can
|
|
96
|
+
* never begin a line.
|
|
97
|
+
*/
|
|
98
|
+
export function sanitizeControlText(value, maxLen = CONTROL_TEXT_MAX) {
|
|
99
|
+
const flattened = value
|
|
100
|
+
.replace(/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu, " ")
|
|
101
|
+
.replace(/\s+/g, " ")
|
|
102
|
+
.trim();
|
|
103
|
+
return flattened.length > maxLen ? `${flattened.slice(0, maxLen)}…` : flattened;
|
|
104
|
+
}
|
|
105
|
+
function resolveDrainSweepBudgetMs(explicit, env) {
|
|
106
|
+
// An UNSET or empty env var is "not configured", not "skip" — `Number("")`
|
|
107
|
+
// is 0, and a stray `CREWHAUS_DRAIN_SWEEP_MS=` must not silently disable
|
|
108
|
+
// housekeeping.
|
|
109
|
+
const raw = env[DRAIN_SWEEP_BUDGET_ENV];
|
|
110
|
+
const candidate = explicit ?? (raw === undefined || raw === "" ? Number.NaN : Number(raw));
|
|
111
|
+
if (!Number.isFinite(candidate) || candidate < 0)
|
|
112
|
+
return DEFAULT_DRAIN_SWEEP_MS;
|
|
113
|
+
return candidate;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Run a BEST-EFFORT housekeeping step during a drain, under its own budget.
|
|
117
|
+
*
|
|
118
|
+
* WHY THIS EXISTS. A drain's contract is "stop intake, finish in-flight work,
|
|
119
|
+
* exit 0", and a supervisor holds it to a deadline. A janitor sweep is not
|
|
120
|
+
* in-flight work — it is housekeeping the next boot repeats — so letting it
|
|
121
|
+
* sit inside that deadline spends the operator's whole drain budget on a step
|
|
122
|
+
* nothing depends on, and the turn the drain existed to finish gets SIGTERM'd
|
|
123
|
+
* anyway. Emitted drain steps therefore close their listeners FIRST and run
|
|
124
|
+
* the sweep through here, last and time-boxed: a slow or wedged sweep can
|
|
125
|
+
* cost at most `budgetMs`, and its failure is reported, never thrown.
|
|
126
|
+
*/
|
|
127
|
+
export async function runDrainSweep(step, opts = {}) {
|
|
128
|
+
const budgetMs = resolveDrainSweepBudgetMs(opts.budgetMs, opts.env ?? process.env);
|
|
129
|
+
if (budgetMs === 0) {
|
|
130
|
+
opts.onOutcome?.("skipped");
|
|
131
|
+
return "skipped";
|
|
132
|
+
}
|
|
133
|
+
let timer;
|
|
134
|
+
let detail;
|
|
135
|
+
try {
|
|
136
|
+
const ran = (async () => {
|
|
137
|
+
try {
|
|
138
|
+
await step();
|
|
139
|
+
return "done";
|
|
140
|
+
}
|
|
141
|
+
catch (err) {
|
|
142
|
+
detail = err.message;
|
|
143
|
+
return "failed";
|
|
144
|
+
}
|
|
145
|
+
})();
|
|
146
|
+
const deadline = new Promise((resolve) => {
|
|
147
|
+
timer = setTimeout(() => resolve("timeout"), budgetMs);
|
|
148
|
+
// A finished sweep must not leave a live timer holding the loop open —
|
|
149
|
+
// the drain exits explicitly, but this keeps the seam usable elsewhere.
|
|
150
|
+
timer.unref?.();
|
|
151
|
+
});
|
|
152
|
+
const outcome = await Promise.race([ran, deadline]);
|
|
153
|
+
opts.onOutcome?.(outcome, detail);
|
|
154
|
+
return outcome;
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
if (timer !== undefined)
|
|
158
|
+
clearTimeout(timer);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/** Session ids must satisfy `@crewhaus/session-store`'s grammar or the very
|
|
162
|
+
* first store/event-log write of the tick throws. Minted exactly as the
|
|
163
|
+
* store's own `generateId` does. */
|
|
164
|
+
function mintSessionId() {
|
|
165
|
+
return `sess_${randomBytes(8).toString("hex")}`;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Constant-time string compare. Both sides are hashed first so the comparison
|
|
169
|
+
* is over fixed-length buffers — `timingSafeEqual` throws on length mismatch,
|
|
170
|
+
* and branching on length would itself leak the token's length.
|
|
171
|
+
*/
|
|
172
|
+
export function constantTimeEquals(a, b) {
|
|
173
|
+
const ha = createHash("sha256").update(a, "utf8").digest();
|
|
174
|
+
const hb = createHash("sha256").update(b, "utf8").digest();
|
|
175
|
+
return timingSafeEqual(ha, hb);
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Resolve the control bearer. `CREWHAUS_CONTROL_TOKEN` wins; otherwise a fresh
|
|
179
|
+
* 32-byte token is minted into `<cwd>/.crewhaus/run/control-token` at 0600.
|
|
180
|
+
*
|
|
181
|
+
* Minting FRESH each boot is deliberate: a token left behind by a dead daemon
|
|
182
|
+
* must not authenticate against its replacement, and the manager reads the
|
|
183
|
+
* file after it spawns the process, so there is nothing to preserve.
|
|
184
|
+
*/
|
|
185
|
+
export function resolveControlToken(opts) {
|
|
186
|
+
const env = opts.env ?? process.env;
|
|
187
|
+
const fromEnv = env[CONTROL_TOKEN_ENV];
|
|
188
|
+
if (fromEnv !== undefined && fromEnv !== "")
|
|
189
|
+
return { token: fromEnv, source: "env" };
|
|
190
|
+
const dir = join(opts.cwd, CONTROL_RUN_DIR);
|
|
191
|
+
const path = join(dir, CONTROL_TOKEN_FILENAME);
|
|
192
|
+
const token = randomBytes(32).toString("hex");
|
|
193
|
+
mkdirSync(dir, { recursive: true });
|
|
194
|
+
writeFileSync(path, `${token}\n`, { mode: 0o600 });
|
|
195
|
+
// `writeFileSync`'s mode only applies when the file is CREATED; an existing
|
|
196
|
+
// file from a previous boot keeps its old mode, so re-assert it.
|
|
197
|
+
chmodSync(path, 0o600);
|
|
198
|
+
return { token, source: "file", path };
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Append the operator-poke marker to the tick's session log.
|
|
202
|
+
*
|
|
203
|
+
* It is written as a `user_message` carrying `synthetic: true` — the
|
|
204
|
+
* established convention for runtime-injected turns. Every turn-deriving
|
|
205
|
+
* reader in the stack (feedback distill, the eval-judge transcript digest, the
|
|
206
|
+
* session summarizer, the advise rules) already skips `synthetic: true` user
|
|
207
|
+
* messages, so an operator poke can never inflate a turn count or land in a
|
|
208
|
+
* training set as if a human had typed it. The `control` sub-object is what
|
|
209
|
+
* lets evals and watch-me positively IDENTIFY the poke and tell it apart from
|
|
210
|
+
* an organic wake.
|
|
211
|
+
*/
|
|
212
|
+
export async function recordSyntheticWake(args) {
|
|
213
|
+
const log = await openEventLog(args.sessionId, args.sessionRootDir !== undefined ? { rootDir: args.sessionRootDir } : {});
|
|
214
|
+
try {
|
|
215
|
+
await log.append({
|
|
216
|
+
kind: "user_message",
|
|
217
|
+
payload: {
|
|
218
|
+
content: `[${CONTROL_PROTOCOL} wake] lane=${args.lane} reason=${args.reason}`,
|
|
219
|
+
synthetic: true,
|
|
220
|
+
control: {
|
|
221
|
+
protocol: CONTROL_PROTOCOL,
|
|
222
|
+
lane: args.lane,
|
|
223
|
+
synthetic: true,
|
|
224
|
+
reason: args.reason,
|
|
225
|
+
by: args.by,
|
|
226
|
+
},
|
|
227
|
+
},
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
finally {
|
|
231
|
+
await log.close();
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/** Default filename `@crewhaus/session-store`'s approval store writes. */
|
|
235
|
+
export const APPROVALS_FILENAME = "approvals.jsonl";
|
|
236
|
+
/**
|
|
237
|
+
* Count parked approvals WITHOUT calling `PendingApprovalStore.list()`.
|
|
238
|
+
*
|
|
239
|
+
* `list()` compacts the backing file as a side-effect (it drops expired and
|
|
240
|
+
* superseded lines), exactly like `SessionStore.list()`'s TTL eviction. A
|
|
241
|
+
* status endpoint is a read: polling it must never rewrite an operator's
|
|
242
|
+
* approvals ledger. So this folds the JSONL itself — last-wins by `id`, the
|
|
243
|
+
* same upsert rule `persist` documents — and counts the records still awaiting
|
|
244
|
+
* a human. A missing file, a torn tail line, or an unreadable record counts as
|
|
245
|
+
* nothing rather than failing the whole status call.
|
|
246
|
+
*/
|
|
247
|
+
export function countPendingApprovals(filePath) {
|
|
248
|
+
let raw;
|
|
249
|
+
try {
|
|
250
|
+
raw = readFileSync(filePath, "utf-8");
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
return 0;
|
|
254
|
+
}
|
|
255
|
+
const latest = new Map();
|
|
256
|
+
for (const line of raw.split("\n")) {
|
|
257
|
+
if (line === "")
|
|
258
|
+
continue;
|
|
259
|
+
let parsed;
|
|
260
|
+
try {
|
|
261
|
+
parsed = JSON.parse(line);
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
continue; // torn tail line — a reader never aborts on one
|
|
265
|
+
}
|
|
266
|
+
if (typeof parsed.id !== "string")
|
|
267
|
+
continue;
|
|
268
|
+
latest.set(parsed.id, parsed);
|
|
269
|
+
}
|
|
270
|
+
let pending = 0;
|
|
271
|
+
for (const record of latest.values()) {
|
|
272
|
+
if (record.consumedAt !== undefined)
|
|
273
|
+
continue;
|
|
274
|
+
if (record.decision === undefined || record.decision === "pending")
|
|
275
|
+
pending += 1;
|
|
276
|
+
}
|
|
277
|
+
return pending;
|
|
278
|
+
}
|
|
279
|
+
function jsonResponse(body, status, headers) {
|
|
280
|
+
return new Response(JSON.stringify(body), {
|
|
281
|
+
status,
|
|
282
|
+
headers: { "content-type": "application/json", ...(headers ?? {}) },
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
export function createControlPlane(opts) {
|
|
286
|
+
const env = opts.env ?? process.env;
|
|
287
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
288
|
+
const now = opts.now ?? Date.now;
|
|
289
|
+
const stdout = opts.stdout ?? ((line) => process.stdout.write(line));
|
|
290
|
+
const stderr = opts.stderr ?? ((line) => process.stderr.write(line));
|
|
291
|
+
const exit = opts.exit ?? ((code) => process.exit(code));
|
|
292
|
+
const drainSettleMs = opts.drainSettleMs ?? 50;
|
|
293
|
+
const startedAt = new Date(now()).toISOString();
|
|
294
|
+
const counters = {
|
|
295
|
+
turns: 0,
|
|
296
|
+
heartbeatTicks: 0,
|
|
297
|
+
scheduleWakes: 0,
|
|
298
|
+
janitorRuns: 0,
|
|
299
|
+
};
|
|
300
|
+
const lanes = new Map();
|
|
301
|
+
const extraTimers = [];
|
|
302
|
+
const drainSteps = [];
|
|
303
|
+
let draining = false;
|
|
304
|
+
let server;
|
|
305
|
+
let resolved;
|
|
306
|
+
function laneCounterKey(lane) {
|
|
307
|
+
return lane === "heartbeat" ? "heartbeatTicks" : "scheduleWakes";
|
|
308
|
+
}
|
|
309
|
+
function makeLane(laneOpts) {
|
|
310
|
+
const armedAtMs = now();
|
|
311
|
+
const ownsSession = laneOpts.ownsSession !== false;
|
|
312
|
+
let busy = false;
|
|
313
|
+
let inFlight = Promise.resolve();
|
|
314
|
+
let lastStartMs;
|
|
315
|
+
let lastFiredAt;
|
|
316
|
+
let lastOutcome;
|
|
317
|
+
/**
|
|
318
|
+
* Claim the lane and run one tick. The claim (`busy = true`) is made
|
|
319
|
+
* SYNCHRONOUSLY, before any await: two wakes arriving in the same tick of
|
|
320
|
+
* the event loop must not both pass the guard, and anything awaited ahead
|
|
321
|
+
* of the claim — writing the marker, say — is a yield point where exactly
|
|
322
|
+
* that would happen.
|
|
323
|
+
*/
|
|
324
|
+
function start(synthetic) {
|
|
325
|
+
if (busy)
|
|
326
|
+
return { accepted: false, sessionId: "", done: inFlight };
|
|
327
|
+
busy = true;
|
|
328
|
+
const sessionId = mintSessionId();
|
|
329
|
+
lastStartMs = now();
|
|
330
|
+
lastFiredAt = new Date(lastStartMs).toISOString();
|
|
331
|
+
counters[laneCounterKey(laneOpts.lane)] += 1;
|
|
332
|
+
const done = (async () => {
|
|
333
|
+
// The marker is a SESSION artifact, so it is written only by a lane
|
|
334
|
+
// that owns the session — see `ownsSession`. A lane that does not gets
|
|
335
|
+
// no orphan `.jsonl`; its evidence is the `gateway_request` record.
|
|
336
|
+
if (synthetic !== undefined && ownsSession) {
|
|
337
|
+
// Written BEFORE the turn so a tick that dies mid-flight is still
|
|
338
|
+
// attributable to the operator who poked it.
|
|
339
|
+
try {
|
|
340
|
+
await recordSyntheticWake({
|
|
341
|
+
sessionId,
|
|
342
|
+
lane: laneOpts.lane,
|
|
343
|
+
reason: synthetic.reason,
|
|
344
|
+
by: synthetic.by,
|
|
345
|
+
...(opts.sessionRootDir !== undefined ? { sessionRootDir: opts.sessionRootDir } : {}),
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
catch (err) {
|
|
349
|
+
stderr(`[control] wake marker not recorded: ${err.message}\n`);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
try {
|
|
353
|
+
await laneOpts.run({
|
|
354
|
+
sessionId,
|
|
355
|
+
...(synthetic !== undefined ? { synthetic } : {}),
|
|
356
|
+
});
|
|
357
|
+
lastOutcome = "ok";
|
|
358
|
+
}
|
|
359
|
+
catch (err) {
|
|
360
|
+
lastOutcome = "error";
|
|
361
|
+
stderr(`[control] ${laneOpts.lane} tick failed: ${err.message}\n`);
|
|
362
|
+
}
|
|
363
|
+
finally {
|
|
364
|
+
busy = false;
|
|
365
|
+
}
|
|
366
|
+
})();
|
|
367
|
+
inFlight = done;
|
|
368
|
+
return { accepted: true, sessionId, done };
|
|
369
|
+
}
|
|
370
|
+
const handle = {
|
|
371
|
+
lane: laneOpts.lane,
|
|
372
|
+
busy: () => busy,
|
|
373
|
+
async tick() {
|
|
374
|
+
await start().done;
|
|
375
|
+
},
|
|
376
|
+
async wake(args) {
|
|
377
|
+
// Sanitized HERE as well as at the router, because this handle is
|
|
378
|
+
// public API: a target that pokes a lane directly must not be able to
|
|
379
|
+
// push a newline into the daemon's own prose log either.
|
|
380
|
+
const started = start({
|
|
381
|
+
reason: sanitizeControlText(args.reason),
|
|
382
|
+
by: sanitizeControlText(args.by),
|
|
383
|
+
});
|
|
384
|
+
// Deliberately NOT awaiting `started.done`: the operator gets a 202
|
|
385
|
+
// and the session id, and the tick runs on its own.
|
|
386
|
+
void started.done;
|
|
387
|
+
return {
|
|
388
|
+
accepted: started.accepted,
|
|
389
|
+
...(started.accepted && ownsSession ? { sessionId: started.sessionId } : {}),
|
|
390
|
+
};
|
|
391
|
+
},
|
|
392
|
+
async settled() {
|
|
393
|
+
while (busy)
|
|
394
|
+
await inFlight;
|
|
395
|
+
},
|
|
396
|
+
report() {
|
|
397
|
+
const projected = laneOpts.nextDueAt !== undefined
|
|
398
|
+
? laneOpts.nextDueAt()
|
|
399
|
+
: laneOpts.everyMs !== undefined
|
|
400
|
+
? new Date((lastStartMs ?? armedAtMs) + laneOpts.everyMs).toISOString()
|
|
401
|
+
: undefined;
|
|
402
|
+
return {
|
|
403
|
+
lane: laneOpts.lane,
|
|
404
|
+
cadence: laneOpts.cadence,
|
|
405
|
+
...(lastFiredAt !== undefined ? { lastFiredAt } : {}),
|
|
406
|
+
...(lastOutcome !== undefined ? { lastOutcome } : {}),
|
|
407
|
+
...(projected !== undefined ? { nextDueAt: projected } : {}),
|
|
408
|
+
};
|
|
409
|
+
},
|
|
410
|
+
};
|
|
411
|
+
return handle;
|
|
412
|
+
}
|
|
413
|
+
function token() {
|
|
414
|
+
if (resolved === undefined)
|
|
415
|
+
resolved = resolveControlToken({ cwd, env });
|
|
416
|
+
return resolved.token;
|
|
417
|
+
}
|
|
418
|
+
function authorized(req) {
|
|
419
|
+
const header = req.headers.get("authorization") ?? "";
|
|
420
|
+
if (!header.startsWith("Bearer "))
|
|
421
|
+
return false;
|
|
422
|
+
return constantTimeEquals(header.slice(7), token());
|
|
423
|
+
}
|
|
424
|
+
async function audit(payload) {
|
|
425
|
+
if (opts.audit === undefined)
|
|
426
|
+
return;
|
|
427
|
+
try {
|
|
428
|
+
await opts.audit({ kind: "gateway_request", payload });
|
|
429
|
+
}
|
|
430
|
+
catch (err) {
|
|
431
|
+
// An unwritable audit log must never take the daemon's control plane
|
|
432
|
+
// down; the failure is reported and the call proceeds.
|
|
433
|
+
stderr(`[control] audit append failed: ${err.message}\n`);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
async function statusBody() {
|
|
437
|
+
let pendingApprovals = 0;
|
|
438
|
+
if (opts.pendingApprovals !== undefined) {
|
|
439
|
+
try {
|
|
440
|
+
pendingApprovals = await opts.pendingApprovals();
|
|
441
|
+
}
|
|
442
|
+
catch {
|
|
443
|
+
pendingApprovals = 0;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
const timers = [
|
|
447
|
+
...[...lanes.values()].map((l) => l.report()),
|
|
448
|
+
...extraTimers.map((t) => t()),
|
|
449
|
+
];
|
|
450
|
+
return {
|
|
451
|
+
protocol: CONTROL_PROTOCOL,
|
|
452
|
+
name: opts.name,
|
|
453
|
+
target: opts.target,
|
|
454
|
+
pid: opts.pid ?? process.pid,
|
|
455
|
+
startedAt,
|
|
456
|
+
draining,
|
|
457
|
+
counters: { ...counters },
|
|
458
|
+
timers,
|
|
459
|
+
channels: opts.channels !== undefined ? [...opts.channels()] : [],
|
|
460
|
+
pendingApprovals,
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
function beginDrain() {
|
|
464
|
+
draining = true;
|
|
465
|
+
setTimeout(() => {
|
|
466
|
+
void (async () => {
|
|
467
|
+
try {
|
|
468
|
+
// Order matters. Intake is already refused (the `draining` flag was
|
|
469
|
+
// set synchronously above: `publicGate` sheds and `/wake` 409s), so
|
|
470
|
+
// this waits out the work that was already accepted, lets each
|
|
471
|
+
// registered step flush — the steps are also what CANCEL the timers,
|
|
472
|
+
// so a tick could have started while the first wait ran — and then
|
|
473
|
+
// waits once more for anything that slipped through.
|
|
474
|
+
await Promise.all([...lanes.values()].map((l) => l.settled()));
|
|
475
|
+
for (const step of drainSteps)
|
|
476
|
+
await step();
|
|
477
|
+
await Promise.all([...lanes.values()].map((l) => l.settled()));
|
|
478
|
+
}
|
|
479
|
+
catch (err) {
|
|
480
|
+
stderr(`[control] drain error: ${err.message}\n`);
|
|
481
|
+
}
|
|
482
|
+
finally {
|
|
483
|
+
stdout("[control] drained — exiting 0\n");
|
|
484
|
+
exit(0);
|
|
485
|
+
}
|
|
486
|
+
})();
|
|
487
|
+
}, drainSettleMs);
|
|
488
|
+
}
|
|
489
|
+
async function handleWake(req) {
|
|
490
|
+
let body;
|
|
491
|
+
try {
|
|
492
|
+
body = await req.json();
|
|
493
|
+
}
|
|
494
|
+
catch {
|
|
495
|
+
body = undefined;
|
|
496
|
+
}
|
|
497
|
+
const parsed = (body ?? {});
|
|
498
|
+
const lane = parsed.lane;
|
|
499
|
+
if (lane !== "heartbeat" && lane !== "schedule") {
|
|
500
|
+
return {
|
|
501
|
+
res: jsonResponse({ error: 'wake requires {"lane": "heartbeat" | "schedule"}', code: "bad_request" }, 400),
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
if (draining) {
|
|
505
|
+
// A poke during a drain would start work the drain is about to abandon.
|
|
506
|
+
return {
|
|
507
|
+
lane,
|
|
508
|
+
res: jsonResponse({ error: "daemon is draining — no new ticks accepted", code: "draining", lane }, 409),
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
const handle = lanes.get(lane);
|
|
512
|
+
if (handle === undefined) {
|
|
513
|
+
return {
|
|
514
|
+
lane,
|
|
515
|
+
res: jsonResponse({ error: `lane "${lane}" is not armed in this bundle`, code: "lane_not_armed", lane }, 404),
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
// THE control-plane trust boundary for operator free-text. `reason`/`by`
|
|
519
|
+
// are echoed into the daemon's prose stdout, which the manager PARSES for
|
|
520
|
+
// the control-port announcement — so they are flattened to one bounded
|
|
521
|
+
// line here, before they can reach a log line, a marker record or the 202.
|
|
522
|
+
const cleanReason = typeof parsed.reason === "string" ? sanitizeControlText(parsed.reason) : "";
|
|
523
|
+
const cleanBy = typeof parsed.by === "string" ? sanitizeControlText(parsed.by) : "";
|
|
524
|
+
const reason = cleanReason !== "" ? cleanReason : "operator wake";
|
|
525
|
+
const by = cleanBy !== "" ? cleanBy : CONTROL_PROTOCOL;
|
|
526
|
+
const outcome = await handle.wake({ reason, by });
|
|
527
|
+
if (!outcome.accepted) {
|
|
528
|
+
// Ticks never overlap themselves — armSchedule's re-arm-after-resolve
|
|
529
|
+
// rule applies to operator pokes too.
|
|
530
|
+
return {
|
|
531
|
+
lane,
|
|
532
|
+
reason,
|
|
533
|
+
by,
|
|
534
|
+
res: jsonResponse({ error: `a ${lane} tick is already in flight`, code: "tick_in_flight", lane }, 409),
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
return {
|
|
538
|
+
lane,
|
|
539
|
+
reason,
|
|
540
|
+
by,
|
|
541
|
+
...(outcome.sessionId !== undefined ? { sessionId: outcome.sessionId } : {}),
|
|
542
|
+
res: jsonResponse({
|
|
543
|
+
// Present ONLY when the lane threads it into a real session. Omitted
|
|
544
|
+
// rather than dangling on the fan-out/producer shapes.
|
|
545
|
+
...(outcome.sessionId !== undefined ? { sessionId: outcome.sessionId } : {}),
|
|
546
|
+
lane,
|
|
547
|
+
reason,
|
|
548
|
+
synthetic: true,
|
|
549
|
+
}, 202),
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
async function fetchControl(req) {
|
|
553
|
+
const path = new URL(req.url).pathname;
|
|
554
|
+
if (!path.startsWith(CONTROL_PATH_PREFIX)) {
|
|
555
|
+
return jsonResponse({ error: "not found", code: "not_found" }, 404);
|
|
556
|
+
}
|
|
557
|
+
const route = path.slice(CONTROL_PATH_PREFIX.length);
|
|
558
|
+
if (!authorized(req)) {
|
|
559
|
+
// The rejected attempt is evidenced too — an unauthenticated poke at the
|
|
560
|
+
// control plane is exactly what an operator wants to see later. The
|
|
561
|
+
// presented credential is never part of the record.
|
|
562
|
+
await audit({
|
|
563
|
+
protocol: CONTROL_PROTOCOL,
|
|
564
|
+
route: path,
|
|
565
|
+
method: req.method,
|
|
566
|
+
authorized: false,
|
|
567
|
+
status: 401,
|
|
568
|
+
});
|
|
569
|
+
return jsonResponse({ error: "unauthorized", code: "unauthorized" }, 401, {
|
|
570
|
+
"www-authenticate": `Bearer realm="${CONTROL_PROTOCOL}"`,
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
if (req.method === "GET" && route === "/healthz") {
|
|
574
|
+
await audit({
|
|
575
|
+
protocol: CONTROL_PROTOCOL,
|
|
576
|
+
route: path,
|
|
577
|
+
method: "GET",
|
|
578
|
+
authorized: true,
|
|
579
|
+
status: 200,
|
|
580
|
+
});
|
|
581
|
+
return jsonResponse({ ok: true, name: opts.name, target: opts.target }, 200);
|
|
582
|
+
}
|
|
583
|
+
if (req.method === "GET" && route === "/status") {
|
|
584
|
+
const body = await statusBody();
|
|
585
|
+
await audit({
|
|
586
|
+
protocol: CONTROL_PROTOCOL,
|
|
587
|
+
route: path,
|
|
588
|
+
method: "GET",
|
|
589
|
+
authorized: true,
|
|
590
|
+
status: 200,
|
|
591
|
+
});
|
|
592
|
+
return jsonResponse(body, 200);
|
|
593
|
+
}
|
|
594
|
+
if (req.method === "POST" && route === "/wake") {
|
|
595
|
+
const outcome = await handleWake(req);
|
|
596
|
+
await audit({
|
|
597
|
+
protocol: CONTROL_PROTOCOL,
|
|
598
|
+
route: path,
|
|
599
|
+
method: "POST",
|
|
600
|
+
authorized: true,
|
|
601
|
+
status: outcome.res.status,
|
|
602
|
+
...(outcome.lane !== undefined ? { lane: outcome.lane } : {}),
|
|
603
|
+
...(outcome.sessionId !== undefined ? { sessionId: outcome.sessionId } : {}),
|
|
604
|
+
// The poke's provenance rides the hash-chained audit log — the record
|
|
605
|
+
// that IS covered by the harness's retention policy — so a lane with
|
|
606
|
+
// no session of its own is still evidenced.
|
|
607
|
+
...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
|
|
608
|
+
...(outcome.by !== undefined ? { by: outcome.by } : {}),
|
|
609
|
+
});
|
|
610
|
+
return outcome.res;
|
|
611
|
+
}
|
|
612
|
+
if (req.method === "POST" && route === "/drain") {
|
|
613
|
+
const already = draining;
|
|
614
|
+
if (!already)
|
|
615
|
+
beginDrain();
|
|
616
|
+
await audit({
|
|
617
|
+
protocol: CONTROL_PROTOCOL,
|
|
618
|
+
route: path,
|
|
619
|
+
method: "POST",
|
|
620
|
+
authorized: true,
|
|
621
|
+
status: 202,
|
|
622
|
+
alreadyDraining: already,
|
|
623
|
+
});
|
|
624
|
+
return jsonResponse({ draining: true, alreadyDraining: already }, 202);
|
|
625
|
+
}
|
|
626
|
+
await audit({
|
|
627
|
+
protocol: CONTROL_PROTOCOL,
|
|
628
|
+
route: path,
|
|
629
|
+
method: req.method,
|
|
630
|
+
authorized: true,
|
|
631
|
+
status: 404,
|
|
632
|
+
});
|
|
633
|
+
return jsonResponse({ error: `no such control route: ${path}`, code: "not_found" }, 404);
|
|
634
|
+
}
|
|
635
|
+
return {
|
|
636
|
+
counters,
|
|
637
|
+
lane(laneOpts) {
|
|
638
|
+
const handle = makeLane(laneOpts);
|
|
639
|
+
lanes.set(laneOpts.lane, handle);
|
|
640
|
+
return handle;
|
|
641
|
+
},
|
|
642
|
+
timer(report) {
|
|
643
|
+
extraTimers.push(report);
|
|
644
|
+
},
|
|
645
|
+
onDrain(step) {
|
|
646
|
+
drainSteps.push(step);
|
|
647
|
+
},
|
|
648
|
+
draining: () => draining,
|
|
649
|
+
fetch: fetchControl,
|
|
650
|
+
async start() {
|
|
651
|
+
const raw = env[CONTROL_PORT_ENV];
|
|
652
|
+
if (raw === undefined || raw === "")
|
|
653
|
+
return undefined;
|
|
654
|
+
const port = Number(raw);
|
|
655
|
+
if (!Number.isInteger(port) || port < 0 || port > 65_535) {
|
|
656
|
+
stderr(`[control] ${CONTROL_PORT_ENV}="${raw}" is not a valid port — control.v1 not served\n`);
|
|
657
|
+
return undefined;
|
|
658
|
+
}
|
|
659
|
+
const bind = env[CONTROL_BIND_ENV] ?? DEFAULT_CONTROL_BIND;
|
|
660
|
+
// Resolving the token here is what MINTS `.crewhaus/run/control-token`
|
|
661
|
+
// — a daemon with no control port never writes one. Reuse an already
|
|
662
|
+
// resolved token rather than minting a second: a fresh mint would
|
|
663
|
+
// silently invalidate one a caller had already been handed.
|
|
664
|
+
const tok = resolved ?? resolveControlToken({ cwd, env });
|
|
665
|
+
resolved = tok;
|
|
666
|
+
const handle = Bun.serve({ port, hostname: bind, fetch: fetchControl });
|
|
667
|
+
server = handle;
|
|
668
|
+
const bound = handle.port ?? port;
|
|
669
|
+
const url = `http://${bind}:${bound}`;
|
|
670
|
+
// The PORT is reported (never the token) so a supervisor that passed
|
|
671
|
+
// port 0 learns the ephemeral port from the log pump.
|
|
672
|
+
stdout(`[control] ${CONTROL_PROTOCOL} listening on ${url} (token: ${tok.source === "env" ? CONTROL_TOKEN_ENV : `${CONTROL_RUN_DIR}/${CONTROL_TOKEN_FILENAME}`})\n`);
|
|
673
|
+
return { port: bound, url };
|
|
674
|
+
},
|
|
675
|
+
async stop() {
|
|
676
|
+
if (server === undefined)
|
|
677
|
+
return;
|
|
678
|
+
server.stop(true);
|
|
679
|
+
server = undefined;
|
|
680
|
+
},
|
|
681
|
+
publicGate(req) {
|
|
682
|
+
const path = new URL(req.url).pathname;
|
|
683
|
+
// Liveness first: a health check must still answer while draining, or a
|
|
684
|
+
// PaaS reaps the process before it finishes its in-flight work. No state
|
|
685
|
+
// is disclosed — this route is unauthenticated by design.
|
|
686
|
+
if (req.method === "GET" && path === "/healthz") {
|
|
687
|
+
return jsonResponse({ ok: true }, 200);
|
|
688
|
+
}
|
|
689
|
+
if (draining) {
|
|
690
|
+
return jsonResponse({ error: "draining", code: "draining" }, 503, {
|
|
691
|
+
"retry-after": String(DRAIN_RETRY_AFTER_SECONDS),
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
return undefined;
|
|
695
|
+
},
|
|
696
|
+
tokenSource() {
|
|
697
|
+
if (resolved === undefined)
|
|
698
|
+
return undefined;
|
|
699
|
+
return resolved.path !== undefined
|
|
700
|
+
? { source: resolved.source, path: resolved.path }
|
|
701
|
+
: { source: resolved.source };
|
|
702
|
+
},
|
|
703
|
+
};
|
|
704
|
+
}
|