@codedology/code-forge 0.15.2 → 0.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/README.md +5 -1
- package/docs/README.md +2 -0
- package/docs/reference/blocks.json +154 -1
- package/docs/reference/config.md +27 -14
- package/package.json +1 -1
- package/schema/code-forge.schema.json +57 -2
- package/schema/outcome.schema.json +45 -0
- package/schema/pipeline.schema.json +30 -0
- package/scripts/first-use.mjs +8 -0
- package/scripts/gen-config-doc.mjs +1 -1
- package/skill/references/code.md +15 -0
- package/skill/references/ledger.md +13 -1
- package/skill/scripts/forge +1 -1
- package/src/cli/block.mjs +146 -30
- package/src/cli/context.mjs +17 -7
- package/src/cli/gates.mjs +76 -10
- package/src/cli/help.mjs +4 -0
- package/src/cli/outcome.mjs +87 -0
- package/src/cli/pipeline.mjs +61 -0
- package/src/cli/proof.mjs +10 -7
- package/src/cli/report.mjs +53 -6
- package/src/cli/review-file.mjs +168 -38
- package/src/cli/run.mjs +3 -2
- package/src/cli/sweep.mjs +28 -0
- package/src/cli/validate.mjs +6 -0
- package/src/config/validate.mjs +83 -1
- package/src/context/pack.mjs +53 -3
- package/src/doctor/local.mjs +22 -4
- package/src/engines/builders/codex.mjs +27 -7
- package/src/engines/builders/grok.mjs +11 -5
- package/src/engines/codex-home.mjs +18 -15
- package/src/engines/sentinel.mjs +11 -0
- package/src/gates/detect.mjs +6 -3
- package/src/gates/run.mjs +295 -27
- package/src/ledger/export.mjs +1 -1
- package/src/ledger/report.mjs +143 -5
- package/src/outcome/outcome.mjs +108 -0
- package/src/pipeline/pipeline.mjs +112 -0
- package/src/proof/red-green.mjs +41 -13
- package/src/proof/runners.mjs +28 -4
- package/src/proof/worktree.mjs +25 -15
- package/src/review/agent-waiver.mjs +18 -4
- package/src/review/engine.mjs +25 -5
- package/src/review/gate-check.mjs +39 -1
- package/src/review/lenses/full.md +1 -0
- package/src/review/lenses/quick.md +1 -0
- package/src/review/packet.mjs +34 -6
- package/src/review/related.mjs +112 -0
- package/src/review/reopen.mjs +4 -3
- package/src/review/session-retry.mjs +59 -0
- package/src/session/spawn.mjs +54 -3
- package/src/state/block.mjs +101 -34
- package/src/state/config-snapshot.mjs +13 -3
- package/src/state/run.mjs +3 -2
- package/src/util/exec.mjs +48 -31
- package/src/util/reaper.mjs +238 -8
- package/src/worker/engine.mjs +7 -3
- package/src/worker/loop.mjs +67 -14
- package/src/worker/queue.mjs +106 -23
- package/src/worker/watch.mjs +2 -2
package/src/util/reaper.mjs
CHANGED
|
@@ -27,7 +27,8 @@
|
|
|
27
27
|
*/
|
|
28
28
|
|
|
29
29
|
import { execFileSync } from 'node:child_process';
|
|
30
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
30
|
+
import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
31
|
+
import os from 'node:os';
|
|
31
32
|
import path from 'node:path';
|
|
32
33
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
33
34
|
|
|
@@ -186,6 +187,220 @@ export function groupMembers(pgid) {
|
|
|
186
187
|
return members;
|
|
187
188
|
}
|
|
188
189
|
|
|
190
|
+
/** @typedef {{uid:number,pid:number,ppid:number,state:string,rssMb:number,start_time:string,command:string}} ProcessInfo */
|
|
191
|
+
|
|
192
|
+
/** Parse C-locale ps output; malformed rows supply no authority to signal.
|
|
193
|
+
* @param {string} text @returns {ProcessInfo[]}
|
|
194
|
+
*/
|
|
195
|
+
export function parseProcesses(text) {
|
|
196
|
+
return text.split('\n').flatMap((line) => {
|
|
197
|
+
const m = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\d+)\s+(\S+\s+\S+\s+\d+\s+\d+:\d+:\d+\s+\d{4})\s+(.+?)\s*$/.exec(line);
|
|
198
|
+
return m ? [{ uid: +m[1], pid: +m[2], ppid: +m[3], state: m[4], rssMb: +m[5] / 1024,
|
|
199
|
+
start_time: m[6].replace(/\s+/g, ' '), command: m[7] }] : [];
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** @returns {ProcessInfo[]} */
|
|
204
|
+
export function readProcesses() {
|
|
205
|
+
try {
|
|
206
|
+
return parseProcesses(execFileSync(existsSync('/bin/ps') ? '/bin/ps' : 'ps',
|
|
207
|
+
['-A', '-ww', '-o', 'uid=,pid=,ppid=,stat=,rss=,lstart=,command='], {
|
|
208
|
+
encoding: 'utf8', env: { ...process.env, LC_ALL: 'C', LANG: 'C' },
|
|
209
|
+
timeout: 5000, maxBuffer: 16 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'],
|
|
210
|
+
}));
|
|
211
|
+
} catch { return []; }
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Current process, ancestors and zombies are never signal targets.
|
|
215
|
+
* @param {ProcessInfo[]} rows @returns {Set<number>}
|
|
216
|
+
*/
|
|
217
|
+
function protectedPids(rows) {
|
|
218
|
+
const blocked = new Set([1, process.pid, process.ppid]);
|
|
219
|
+
const byPid = new Map(rows.map((p) => [p.pid, p]));
|
|
220
|
+
let pid = process.ppid;
|
|
221
|
+
const visited = new Set();
|
|
222
|
+
while (pid > 1 && !visited.has(pid)) {
|
|
223
|
+
visited.add(pid);
|
|
224
|
+
blocked.add(pid);
|
|
225
|
+
pid = byPid.get(pid)?.ppid ?? 1;
|
|
226
|
+
}
|
|
227
|
+
for (const p of rows) if (p.state.startsWith('Z')) blocked.add(p.pid);
|
|
228
|
+
return blocked;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** One synchronous identity/uid/state check immediately followed by a bare-pid signal.
|
|
232
|
+
* @param {number} pid @param {string} started @param {NodeJS.Signals} signal
|
|
233
|
+
* @returns {boolean}
|
|
234
|
+
*/
|
|
235
|
+
function signalMember(pid, started, signal) {
|
|
236
|
+
if (!started || started === UNKNOWN_START_TIME || !process.getuid) return false;
|
|
237
|
+
const rows = readProcesses();
|
|
238
|
+
const p = rows.find((r) => r.pid === pid);
|
|
239
|
+
if (!p || p.uid !== process.getuid() || protectedPids(rows).has(pid) || p.start_time !== started) return false;
|
|
240
|
+
try { process.kill(pid, signal); return true; } catch { return false; }
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Stop a captured set, checking every member separately at both signals.
|
|
244
|
+
* @param {Map<number,string>} members @param {number} [graceMs]
|
|
245
|
+
* @returns {Promise<Map<number, 'SIGTERM'|'SIGKILL'>>}
|
|
246
|
+
*/
|
|
247
|
+
export async function stopMembers(members, graceMs = DEFAULT_GRACE_MS) {
|
|
248
|
+
const sent = new Map();
|
|
249
|
+
for (const [pid, started] of members) if (signalMember(pid, started, 'SIGTERM')) sent.set(pid, 'SIGTERM');
|
|
250
|
+
const deadline = Date.now() + graceMs;
|
|
251
|
+
while (sent.size && Date.now() < deadline) {
|
|
252
|
+
const live = readProcesses();
|
|
253
|
+
if (![...sent.keys()].some((pid) => live.some((p) => p.pid === pid && !p.state.startsWith('Z') && p.start_time === members.get(pid)))) break;
|
|
254
|
+
await sleep(POLL_MS);
|
|
255
|
+
}
|
|
256
|
+
for (const pid of sent.keys()) if (signalMember(pid, members.get(pid), 'SIGKILL')) sent.set(pid, 'SIGKILL');
|
|
257
|
+
// Delivery is not exit: wait for SIGKILL to take effect before exec can return to its caller.
|
|
258
|
+
const killedDeadline = Date.now() + Math.max(graceMs, 1000);
|
|
259
|
+
while (sent.size && Date.now() < killedDeadline) {
|
|
260
|
+
await sleep(POLL_MS);
|
|
261
|
+
const live = readProcesses();
|
|
262
|
+
if (![...sent.keys()].some((pid) => live.some((p) => p.pid === pid && !p.state.startsWith('Z') && p.start_time === members.get(pid)))) break;
|
|
263
|
+
}
|
|
264
|
+
return sent;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Read only ownership markers, never retain/print the rest of a child's environment.
|
|
268
|
+
* @param {ProcessInfo} p @returns {Record<string,string>|null}
|
|
269
|
+
*/
|
|
270
|
+
export function readProcessEnv(p) {
|
|
271
|
+
try {
|
|
272
|
+
let text;
|
|
273
|
+
if (process.platform === 'linux') text = readFileSync(`/proc/${p.pid}/environ`, 'utf8');
|
|
274
|
+
else if (process.platform === 'darwin') {
|
|
275
|
+
const out = execFileSync('/bin/ps', ['-E', '-ww', '-o', 'pid=,ppid=,lstart=,command=', '-p', String(p.pid)], {
|
|
276
|
+
encoding: 'utf8', timeout: 5000, maxBuffer: 8 * 1024 * 1024,
|
|
277
|
+
env: { ...process.env, LC_ALL: 'C', LANG: 'C' }, stdio: ['ignore', 'pipe', 'ignore'],
|
|
278
|
+
});
|
|
279
|
+
const m = /^\s*(\d+)\s+(\d+)\s+(\S+\s+\S+\s+\d+\s+\d+:\d+:\d+\s+\d{4})\s+/.exec(out);
|
|
280
|
+
if (!m || +m[1] !== p.pid || +m[2] !== p.ppid || m[3].replace(/\s+/g, ' ') !== p.start_time) return null;
|
|
281
|
+
text = out.slice(m[0].length);
|
|
282
|
+
// ps appends the environment to argv. An ownership-looking argument is not an env
|
|
283
|
+
// marker: strip the independently observed command before parsing the environment.
|
|
284
|
+
if (!text.startsWith(`${p.command} `)) return null;
|
|
285
|
+
text = text.slice(p.command.length + 1);
|
|
286
|
+
} else return null;
|
|
287
|
+
/** @type {Record<string,string>} */
|
|
288
|
+
const markers = {};
|
|
289
|
+
for (const name of ['HOME', 'CODE_FORGE_OWNER_PID', 'CODE_FORGE_RUN_ID', 'CODE_FORGE_OWNER_START_TIME']) {
|
|
290
|
+
const match = process.platform === 'linux'
|
|
291
|
+
? text.split('\0').find((entry) => entry.startsWith(`${name}=`))?.slice(name.length + 1)
|
|
292
|
+
: new RegExp(`(?:^|\\s)${name}=(.*?)(?=\\s+[A-Za-z_][A-Za-z0-9_]*=|$)`).exec(text)?.[1];
|
|
293
|
+
if (match !== undefined) markers[name] = match;
|
|
294
|
+
}
|
|
295
|
+
return markers;
|
|
296
|
+
} catch { return null; }
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Inspect and optionally stop proven code-forge processes. Defaults are read-only.
|
|
301
|
+
* `psText`/readers are test seams; production always probes the OS and existing record reader.
|
|
302
|
+
* @param {{apply?:boolean, root?:string, runId?:string, orphansOnly?:boolean, zombies?:boolean,
|
|
303
|
+
* graceMs?:number, psText?:string, uid?:number,
|
|
304
|
+
* readEnv?:(p:ProcessInfo)=>Record<string,string>|null,
|
|
305
|
+
* readRun?:(id:string)=>Promise<Record<string,any>>}} [opts]
|
|
306
|
+
*/
|
|
307
|
+
export async function sweepProcesses(opts = {}) {
|
|
308
|
+
const rows = opts.psText === undefined ? readProcesses() : parseProcesses(opts.psText);
|
|
309
|
+
const uid = opts.uid ?? process.getuid?.();
|
|
310
|
+
const own = rows.filter((p) => p.uid === uid);
|
|
311
|
+
const blocked = protectedPids(rows);
|
|
312
|
+
const envFor = opts.readEnv ?? readProcessEnv;
|
|
313
|
+
const loadRun = opts.readRun ?? (await import('../state/run.mjs')).readRun;
|
|
314
|
+
const tmp = await import('./tmp.mjs');
|
|
315
|
+
/** @type {(ProcessInfo & {kind:'worker'|'orphan'|'registry',runId?:string,action?:string})[]} */
|
|
316
|
+
const items = [];
|
|
317
|
+
/** @type {{ppid:number,command:string,count:number,pids:number[]}[]} */
|
|
318
|
+
const zombies = [];
|
|
319
|
+
if (opts.zombies !== false && !opts.orphansOnly) {
|
|
320
|
+
for (const p of own.filter((p) => p.state.startsWith('Z'))) {
|
|
321
|
+
let group = zombies.find((g) => g.ppid === p.ppid);
|
|
322
|
+
if (!group) { group = { ppid: p.ppid, command: rows.find((r) => r.pid === p.ppid)?.command ?? '(unreadable)', count: 0, pids: [] }; zombies.push(group); }
|
|
323
|
+
group.count++; group.pids.push(p.pid);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
for (const p of own) {
|
|
327
|
+
if (p.pid <= 1 || blocked.has(p.pid)) continue;
|
|
328
|
+
// Exactly node + a code-forge entry point + worker, never an argv substring in a shell/-e.
|
|
329
|
+
const worker = /^(?:\S*\/)?node(?:js)?\s+(?:\S*\/)?code-forge\.mjs\s+worker\s+--run(?:=|\s+)([a-z0-9][a-z0-9-]{0,63})(?:\s|$)/.exec(p.command);
|
|
330
|
+
if (worker && !opts.orphansOnly) {
|
|
331
|
+
// Run records only authorize worker cleanup inside their own code-forge home.
|
|
332
|
+
// Unreadable environments and workers from another HOME supply no such authority.
|
|
333
|
+
if (envFor(p)?.HOME !== os.homedir()) continue;
|
|
334
|
+
try {
|
|
335
|
+
const record = await loadRun(worker[1]);
|
|
336
|
+
if (typeof record.status === 'string' && !['open', 'running', 'active'].includes(record.status)) items.push({ ...p, kind: 'worker', runId: worker[1] });
|
|
337
|
+
} catch (err) {
|
|
338
|
+
if (err?.code === 'no-run') items.push({ ...p, kind: 'worker', runId: worker[1] });
|
|
339
|
+
}
|
|
340
|
+
// The run record is authoritative for workers, including when it cannot be read.
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
if (p.ppid !== 1 || items.some((item) => item.pid === p.pid)) continue;
|
|
344
|
+
const env = envFor(p);
|
|
345
|
+
const owner = env?.CODE_FORGE_OWNER_PID;
|
|
346
|
+
if (!owner || !/^[1-9]\d*$/.test(owner) || +owner <= 1 || (opts.runId && env.CODE_FORGE_RUN_ID !== opts.runId)) continue;
|
|
347
|
+
const liveOwner = rows.find((r) => r.pid === +owner);
|
|
348
|
+
const recorded = env.CODE_FORGE_OWNER_START_TIME;
|
|
349
|
+
// A missing row alone is not proof of death when ps was incomplete.
|
|
350
|
+
const dead = !liveOwner ? !isAlive(+owner) : liveOwner.state.startsWith('Z')
|
|
351
|
+
|| !!recorded && recorded !== UNKNOWN_START_TIME && liveOwner.start_time !== recorded;
|
|
352
|
+
if (dead) items.push({ ...p, kind: 'orphan', ...(env.CODE_FORGE_RUN_ID ? { runId: env.CODE_FORGE_RUN_ID } : {}) });
|
|
353
|
+
}
|
|
354
|
+
const root = tmp.tmpBase(opts.root);
|
|
355
|
+
const registryRoots = [];
|
|
356
|
+
if (!opts.orphansOnly && tmp.untrustedReason(root) === null) {
|
|
357
|
+
for (const dir of readdirSync(root, { withFileTypes: true })) {
|
|
358
|
+
const full = path.join(root, dir.name);
|
|
359
|
+
if (!dir.isDirectory() || tmp.untrustedReason(full) !== null) continue;
|
|
360
|
+
const state = tmp.ownerState(full);
|
|
361
|
+
let reap = state === 'dead';
|
|
362
|
+
if (state === 'missing') {
|
|
363
|
+
// Mirror sweepRoots' ten-minute ownerless grace, including every registry mtime.
|
|
364
|
+
try {
|
|
365
|
+
const pids = path.join(full, 'pids');
|
|
366
|
+
const files = existsSync(pids) ? [pids, ...readdirSync(pids).map((name) => path.join(pids, name))] : [];
|
|
367
|
+
reap = [full, ...files].every((file) => Date.now() - lstatSync(file).mtimeMs > 600_000);
|
|
368
|
+
} catch { reap = false; }
|
|
369
|
+
}
|
|
370
|
+
if (!reap) continue;
|
|
371
|
+
registryRoots.push(full);
|
|
372
|
+
for (const entry of listEntries(path.join(full, 'pids'))) {
|
|
373
|
+
const p = own.find((p) => p.pid === entry.pid && p.start_time === entry.start_time);
|
|
374
|
+
if (!p || blocked.has(p.pid)) continue;
|
|
375
|
+
const members = new Map([[p.pid, p.start_time], ...groupMembers(p.pid)]);
|
|
376
|
+
for (const [pid, started] of members) {
|
|
377
|
+
const member = own.find((p) => p.pid === pid && p.start_time === started);
|
|
378
|
+
if (member && !blocked.has(pid) && !items.some((i) => i.pid === pid)) items.push({ ...member, kind: 'registry' });
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
// Memory across all code-forge trees, deduplicated. Unmarked descendants are children too.
|
|
384
|
+
const forge = new Set(own.filter((p) => /^(?:\S*\/)?node(?:js)?\s+(?:\S*\/)?code-forge\.mjs(?:\s|$)/.test(p.command) || p.pid === process.pid).map((p) => p.pid));
|
|
385
|
+
const tree = new Set(forge);
|
|
386
|
+
let size;
|
|
387
|
+
do { size = tree.size; for (const p of own) if (tree.has(p.ppid)) tree.add(p.pid); } while (tree.size !== size);
|
|
388
|
+
const sum = (/** @type {ProcessInfo[]} */ ps) => ps.reduce((n, p) => n + p.rssMb, 0);
|
|
389
|
+
const memory = { codeForgeMb: sum(own.filter((p) => forge.has(p.pid))), childrenMb: sum(own.filter((p) => tree.has(p.pid) && !forge.has(p.pid))), orphansMb: sum(items.filter((p) => p.kind === 'orphan')) };
|
|
390
|
+
if (opts.apply) {
|
|
391
|
+
for (const item of items.filter((p) => p.kind !== 'registry')) {
|
|
392
|
+
const stopped = await stopMembers(new Map([[item.pid, item.start_time]]), opts.graceMs);
|
|
393
|
+
item.action = stopped.has(item.pid) ? 'stopped' : 'skipped';
|
|
394
|
+
}
|
|
395
|
+
if (!opts.orphansOnly) await tmp.sweepRoots({ root, graceMs: opts.graceMs });
|
|
396
|
+
for (const item of items.filter((p) => p.kind === 'registry')) {
|
|
397
|
+
const started = readStartTime(item.pid);
|
|
398
|
+
item.action = !isAlive(item.pid) || (started !== null && started !== item.start_time) ? 'stopped' : 'skipped';
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
return { apply: opts.apply === true, items, registryRoots, zombies, totalRssMb: sum(items), memory };
|
|
402
|
+
}
|
|
403
|
+
|
|
189
404
|
/**
|
|
190
405
|
* Send `signal` to `pid`'s group — only after a SYNCHRONOUS start-time match, with no await
|
|
191
406
|
* between the check and the kill. Falls back to the bare pid only when the group does not exist
|
|
@@ -196,6 +411,19 @@ export function groupMembers(pgid) {
|
|
|
196
411
|
* @returns {'sent'|'mismatch'|'eperm'|'gone'}
|
|
197
412
|
*/
|
|
198
413
|
function signalIfOurs(pid, recorded, signal) {
|
|
414
|
+
const rows = readProcesses();
|
|
415
|
+
const blocked = protectedPids(rows);
|
|
416
|
+
const leader = rows.find((p) => p.pid === pid);
|
|
417
|
+
if (!process.getuid || !leader || leader.uid !== process.getuid() || blocked.has(pid)) return 'eperm';
|
|
418
|
+
const members = groupMembers(pid);
|
|
419
|
+
const safeGroup = [...members].every(([member, started]) => rows.some((p) => p.pid === member
|
|
420
|
+
&& p.uid === process.getuid() && !blocked.has(member) && p.start_time === started));
|
|
421
|
+
if (!safeGroup) {
|
|
422
|
+
if (readStartTime(pid) !== recorded) return 'mismatch';
|
|
423
|
+
const sent = signalMember(pid, recorded, signal);
|
|
424
|
+
for (const [member, started] of members) signalMember(member, started, signal);
|
|
425
|
+
return sent ? 'sent' : 'eperm';
|
|
426
|
+
}
|
|
199
427
|
if (readStartTime(pid) !== recorded) return 'mismatch';
|
|
200
428
|
try {
|
|
201
429
|
process.kill(-pid, signal);
|
|
@@ -234,10 +462,16 @@ export async function sweep(dir, opts = {}) {
|
|
|
234
462
|
const { graceMs = DEFAULT_GRACE_MS } = opts;
|
|
235
463
|
/** @type {SweepResult} */
|
|
236
464
|
const result = { killed: [], mismatched: [], unknown: [], stale: [] };
|
|
465
|
+
const rows = readProcesses();
|
|
466
|
+
const blocked = protectedPids(rows);
|
|
237
467
|
|
|
238
468
|
for (const entry of listEntries(dir)) {
|
|
239
469
|
const { pid } = entry;
|
|
240
|
-
if (
|
|
470
|
+
if (blocked.has(pid)) continue;
|
|
471
|
+
if (isAlive(pid) && !rows.some((p) => p.pid === pid && p.uid === process.getuid?.())) {
|
|
472
|
+
result.unknown.push(pid);
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
241
475
|
|
|
242
476
|
if (!isAlive(pid)) {
|
|
243
477
|
unregisterPid(dir, pid);
|
|
@@ -278,6 +512,7 @@ export async function sweep(dir, opts = {}) {
|
|
|
278
512
|
continue;
|
|
279
513
|
}
|
|
280
514
|
await waitForDeath(pid, graceMs);
|
|
515
|
+
for (const [member, started] of members) signalMemberIfOurs(member, started);
|
|
281
516
|
result.killed.push({ pid, signal: 'SIGKILL' });
|
|
282
517
|
}
|
|
283
518
|
unregisterPid(dir, pid);
|
|
@@ -291,10 +526,5 @@ export async function sweep(dir, opts = {}) {
|
|
|
291
526
|
* @param {number} pid @param {string} started
|
|
292
527
|
*/
|
|
293
528
|
function signalMemberIfOurs(pid, started) {
|
|
294
|
-
|
|
295
|
-
try {
|
|
296
|
-
process.kill(pid, 'SIGKILL');
|
|
297
|
-
} catch {
|
|
298
|
-
// gone between the check and the kill
|
|
299
|
-
}
|
|
529
|
+
signalMember(pid, started, 'SIGKILL');
|
|
300
530
|
}
|
package/src/worker/engine.mjs
CHANGED
|
@@ -109,7 +109,7 @@ import { verifyRow } from '../state/signer.mjs';
|
|
|
109
109
|
import { redact } from '../util/redact.mjs';
|
|
110
110
|
import { withLock } from '../util/file-locks.mjs';
|
|
111
111
|
import { blockAnchors, blockRungUsed, loadState, RESTORED_EVENT, saveState } from './review-state.mjs';
|
|
112
|
-
import { assertAnswers, enqueueHash, isDone, writeResult } from './queue.mjs';
|
|
112
|
+
import { assertAnswers, enqueueHash, isDone, reviewExclusionFor, writeResult } from './queue.mjs';
|
|
113
113
|
import { assertTicketId, contentHash, readRegularFileNoFollow, WorkerError } from './ticket.mjs';
|
|
114
114
|
|
|
115
115
|
/**
|
|
@@ -345,6 +345,9 @@ export function packetDirFor(runRootDir, id) {
|
|
|
345
345
|
* @returns {Promise<ReviewOutcome>}
|
|
346
346
|
*/
|
|
347
347
|
export async function reviewTicket(ticket, ctx) {
|
|
348
|
+
const excluded = (/** @type {string} */ file) => reviewExclusionFor(ctx.repoRoot, ctx.runId, file, ctx.cfg);
|
|
349
|
+
const rule = excluded(ticket.file) ?? (ticket.kind === 'answers' ? excluded(ticket.by_file) : null);
|
|
350
|
+
if (rule !== null) return { status: 'excluded', reason: `review.exclude ${rule}`, approved: false, engine: 'adaptive', sessions: [] };
|
|
348
351
|
const workDir = packetDirFor(ctx.runRootDir, ticket.ticket);
|
|
349
352
|
if (workDir === null) return { status: 'unavailable', reason: 'ticket-id-invalid', approved: false, engine: 'adaptive', sessions: [] };
|
|
350
353
|
let entry;
|
|
@@ -390,7 +393,7 @@ export async function reviewTicket(ticket, ctx) {
|
|
|
390
393
|
try {
|
|
391
394
|
// Read the listing directly: blockPeers' [] fallback would hide a git failure and
|
|
392
395
|
// still authorize explicit tests. Context and peer diffs must fail soft together.
|
|
393
|
-
const peers = entry.base === null || entry.owned.length === 0 ? [] : (await computeFileSet({ cwd: contentRoot, base: entry.base })).all.filter((f) => f !== ticket.file && ownsFile(entry.owned, f, entry.ownedKinds));
|
|
396
|
+
const peers = entry.base === null || entry.owned.length === 0 ? [] : (await computeFileSet({ cwd: contentRoot, base: entry.base })).all.filter((f) => f !== ticket.file && ownsFile(entry.owned, f, entry.ownedKinds) && excluded(f) === null);
|
|
394
397
|
readDiffs = peerDiffReader({ repoRoot: contentRoot, base: entry.base, file: ticket.file, peers });
|
|
395
398
|
const changedFiles = [...peers, ticket.file].filter((f) => ownsFile(entry.owned, f, entry.ownedKinds));
|
|
396
399
|
return { peers, changedFiles, ownedFiles: [...new Set([...changedFiles, ...entry.testFiles])], testFiles: entry.testFiles };
|
|
@@ -444,7 +447,7 @@ async function lightRound(ticket, ctx, { entry, workDir, factsExcerpt, rulesDige
|
|
|
444
447
|
const worker = { pid: process.pid, started_at: await processStartTime(process.pid) };
|
|
445
448
|
return withLock(`light:${ctx.runId}\0${ticket.block}`, async () => {
|
|
446
449
|
const root = ctx.contentRoot ?? ctx.repoRoot;
|
|
447
|
-
const files = (await computeFileSet({ cwd: root, base: entry.base ?? 'HEAD' })).all.filter((file) => ownsFile(entry.owned, file, entry.ownedKinds)).sort();
|
|
450
|
+
const files = (await computeFileSet({ cwd: root, base: entry.base ?? 'HEAD' })).all.filter((file) => ownsFile(entry.owned, file, entry.ownedKinds) && reviewExclusionFor(ctx.repoRoot, ctx.runId, file, ctx.cfg) === null).sort();
|
|
448
451
|
const hashes = files.map((file) => ({ file, content_hash: contentHash(root, file) }));
|
|
449
452
|
if (!hashes.some((f) => f.file === ticket.file && f.content_hash === ticket.content_hash)) return { status: 'stale', approved: false, engine: 'adaptive', sessions: [] };
|
|
450
453
|
const signature = JSON.stringify([entry.base, hashes]);
|
|
@@ -473,6 +476,7 @@ async function lightRound(ticket, ctx, { entry, workDir, factsExcerpt, rulesDige
|
|
|
473
476
|
if (approved) await writeRow({ event: 'review.approved', file, content_hash });
|
|
474
477
|
}
|
|
475
478
|
const queued = enqueueHash({ repoRoot: ctx.repoRoot, run: ctx.runId, block: ticket.block, file, content_hash, ...(ticket.tree ? { tree: ticket.tree } : {}) });
|
|
479
|
+
if (queued.status === 'excluded') continue;
|
|
476
480
|
const open = approved ? [] : own;
|
|
477
481
|
await writeRow({ event: 'review.result', ticket: queued.ticket, file, content_hash, status: outcome.status, approved, open, worker });
|
|
478
482
|
writeResult({ repoRoot: ctx.repoRoot, runId: ctx.runId, ticket: queued.ticket, result: { ...outcome, block: ticket.block, file, content_hash, open, worker }, key });
|
package/src/worker/loop.mjs
CHANGED
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
*/
|
|
37
37
|
|
|
38
38
|
import { spawn as nodeSpawn } from 'node:child_process';
|
|
39
|
-
import { existsSync, statSync } from 'node:fs';
|
|
39
|
+
import { existsSync, statSync, watch } from 'node:fs';
|
|
40
40
|
import os from 'node:os';
|
|
41
41
|
import path from 'node:path';
|
|
42
42
|
import { fileURLToPath } from 'node:url';
|
|
@@ -53,11 +53,12 @@ import { loadKey, signRow } from '../state/signer.mjs';
|
|
|
53
53
|
import { logWarning } from '../util/error-log.mjs';
|
|
54
54
|
import { withLock, tryLock } from '../util/file-locks.mjs';
|
|
55
55
|
import { redact, writeSafe } from '../util/redact.mjs';
|
|
56
|
+
import { sweepProcesses } from '../util/reaper.mjs';
|
|
56
57
|
import { runRoot, setRunRoot } from '../util/tmp.mjs';
|
|
57
58
|
import { openFindings } from '../review/gate-check.mjs';
|
|
58
59
|
import { countLines, RouteError, routeTicket } from '../review/route.mjs';
|
|
59
60
|
import { canonicalPath, contentRootFor, reviewTicket } from './engine.mjs';
|
|
60
|
-
import { announceWorker, beatWorker, HEARTBEAT_MS, isDone, liveWorker, pendingTickets, readTicket, retractWorker, workerRegistry, writeResult } from './queue.mjs';
|
|
61
|
+
import { announceWorker, beatWorker, HEARTBEAT_MS, isDone, liveWorker, pendingTickets, queueDir, readTicket, retractWorker, workerRegistry, writeResult, writeTicketTimings } from './queue.mjs';
|
|
61
62
|
import { DELETED_HASH, hashBytes, readRegularFileNoFollow, repoRootOf, WorkerError } from './ticket.mjs';
|
|
62
63
|
import { watchTick } from './watch.mjs';
|
|
63
64
|
|
|
@@ -236,6 +237,7 @@ export function recordedVerdict(result) {
|
|
|
236
237
|
* @property {typeof reviewTicket} [review] - the engine hook (default `./engine.mjs`).
|
|
237
238
|
* @property {typeof spawnSession} [spawn]
|
|
238
239
|
* @property {typeof readRun} [readRun] - the run-record read behind the per-ticket config (a test seam).
|
|
240
|
+
* @property {(ms: number) => Promise<unknown>} [sleep] - idle scheduler wait (a test seam).
|
|
239
241
|
* @property {{beforeLock?: () => void, afterCount?: (count: number) => Promise<unknown>}} [hooks] - deterministic boot races.
|
|
240
242
|
* @property {typeof readTicket} [readTicket] - the ticket read (a test seam).
|
|
241
243
|
* @property {typeof import('../util/exec.mjs').exec} [watchExec] - watch git exec seam.
|
|
@@ -287,6 +289,7 @@ export async function createWorker(opts, deps = {}) {
|
|
|
287
289
|
let wake = null;
|
|
288
290
|
/** @type {(() => void) | null} wakes the pool's wait in `drain` (stop) */
|
|
289
291
|
let poolWake = null;
|
|
292
|
+
let activity = 0;
|
|
290
293
|
|
|
291
294
|
/** @param {Record<string, any>} row */
|
|
292
295
|
const ledger = (row) => writeRow(signRow({ run: runId, ...row }, key));
|
|
@@ -407,14 +410,17 @@ export async function createWorker(opts, deps = {}) {
|
|
|
407
410
|
|
|
408
411
|
/**
|
|
409
412
|
* @param {string} id
|
|
413
|
+
* @param {{ticket: import('./queue.mjs').Ticket | null, queued_at: string | null, started_at: string}} [attempt]
|
|
410
414
|
* @returns {Promise<Record<string, any> | null>} the signed result, or null when abandoned.
|
|
411
415
|
*/
|
|
412
|
-
async function processTicket(id) {
|
|
416
|
+
async function processTicket(id, attempt = { ticket: null, queued_at: null, started_at: new Date().toISOString() }) {
|
|
413
417
|
/** @type {Record<string, any>} */
|
|
414
418
|
let result;
|
|
415
419
|
let ticket = null;
|
|
416
420
|
try {
|
|
417
421
|
ticket = loadTicket(repoRoot, id);
|
|
422
|
+
attempt.ticket = ticket;
|
|
423
|
+
attempt.queued_at = ticket.queued_at ?? ticket.enqueued_at ?? null;
|
|
418
424
|
} catch (err) {
|
|
419
425
|
result = { event: 'review.result', status: 'refused', reason: err instanceof WorkerError ? err.code : 'bad-ticket', approved: false };
|
|
420
426
|
}
|
|
@@ -423,13 +429,17 @@ export async function createWorker(opts, deps = {}) {
|
|
|
423
429
|
// read, advanced and saved by one ticket at a time; the later one waits here. The key
|
|
424
430
|
// carries the run so two workers in one process (tests) never share a lock.
|
|
425
431
|
const t = ticket;
|
|
426
|
-
result = await withLock(`${runId}\0${fileLockKey(t.block, t.file)}`, () =>
|
|
432
|
+
result = await withLock(`${runId}\0${fileLockKey(t.block, t.file)}`, () => {
|
|
433
|
+
attempt.started_at = new Date().toISOString();
|
|
434
|
+
writeTicketTimings(repoRoot, t, { queued_at: attempt.queued_at, started_at: attempt.started_at, finished_at: null, duration_ms: null });
|
|
435
|
+
return reviewOne(id, t);
|
|
436
|
+
});
|
|
427
437
|
}
|
|
428
438
|
// Stopping: a ticket whose session our own stop killed ends `unavailable` — leave it queued
|
|
429
439
|
// (no done marker) for the next worker instead of recording the kill as a failure. A ticket
|
|
430
440
|
// that reached a real outcome while the pool drained is recorded as usual.
|
|
431
441
|
if (stopping && result.status === 'unavailable') return null;
|
|
432
|
-
return complete(id, result);
|
|
442
|
+
return complete(id, result, attempt);
|
|
433
443
|
}
|
|
434
444
|
|
|
435
445
|
/**
|
|
@@ -485,10 +495,14 @@ export async function createWorker(opts, deps = {}) {
|
|
|
485
495
|
* Sign and write the result + done marker, then the signed ledger row. A ledger failure does not
|
|
486
496
|
* undo the result: the done marker is what the queue reads.
|
|
487
497
|
* @param {string} id @param {Record<string, any>} outcome - recorded through {@link recordedVerdict}.
|
|
498
|
+
* @param {{ticket: import('./queue.mjs').Ticket | null, queued_at: string | null, started_at: string}} attempt
|
|
488
499
|
*/
|
|
489
|
-
async function complete(id, outcome) {
|
|
500
|
+
async function complete(id, outcome, attempt) {
|
|
501
|
+
const finished_at = new Date().toISOString();
|
|
502
|
+
const timings = { queued_at: attempt.queued_at, started_at: attempt.started_at, finished_at, duration_ms: Math.max(0, Date.parse(finished_at) - Date.parse(attempt.started_at)) };
|
|
503
|
+
if (attempt.ticket) writeTicketTimings(repoRoot, attempt.ticket, timings);
|
|
490
504
|
/** @type {Record<string, any>} */
|
|
491
|
-
const result = { ...recordedVerdict(outcome), worker: identity };
|
|
505
|
+
const result = { ...(attempt.ticket ? { block: attempt.ticket.block, file: attempt.ticket.file, content_hash: attempt.ticket.content_hash } : {}), ...recordedVerdict(outcome), ...timings, worker: identity };
|
|
492
506
|
const signed = writeResult({ repoRoot, runId, ticket: id, result, key });
|
|
493
507
|
try {
|
|
494
508
|
await ledger({
|
|
@@ -499,6 +513,7 @@ export async function createWorker(opts, deps = {}) {
|
|
|
499
513
|
file: result.file ?? null,
|
|
500
514
|
content_hash: result.content_hash ?? null,
|
|
501
515
|
status: result.status,
|
|
516
|
+
...timings,
|
|
502
517
|
// B52: the gate reads the verdict from the ledger, never from the result file
|
|
503
518
|
approved: result.approved,
|
|
504
519
|
open: result.open,
|
|
@@ -525,13 +540,14 @@ export async function createWorker(opts, deps = {}) {
|
|
|
525
540
|
* @param {string} id @returns {Promise<number>} 1 when completed, else 0.
|
|
526
541
|
*/
|
|
527
542
|
async function runOne(id) {
|
|
543
|
+
const attempt = { ticket: /** @type {import('./queue.mjs').Ticket | null} */ (null), queued_at: /** @type {string | null} */ (null), started_at: new Date().toISOString() };
|
|
528
544
|
try {
|
|
529
|
-
return (await processTicket(id)) !== null ? 1 : 0;
|
|
545
|
+
return (await processTicket(id, attempt)) !== null ? 1 : 0;
|
|
530
546
|
} catch (err) {
|
|
531
547
|
// B90: a failure a step names (StepError) keeps its step and original text, redacted
|
|
532
548
|
const detail = err instanceof StepError ? { detail: redact(err.message) } : {};
|
|
533
549
|
try {
|
|
534
|
-
await complete(id, { event: 'review.result', status: 'unavailable', reason: 'worker-error', ...detail, approved: false });
|
|
550
|
+
await complete(id, { event: 'review.result', status: 'unavailable', reason: 'worker-error', ...detail, approved: false }, attempt);
|
|
535
551
|
return 1;
|
|
536
552
|
} catch {
|
|
537
553
|
poisoned.add(id);
|
|
@@ -557,7 +573,7 @@ export async function createWorker(opts, deps = {}) {
|
|
|
557
573
|
|
|
558
574
|
/**
|
|
559
575
|
* The block and (block, file) key of each ticket read so far, by id. A written ticket never
|
|
560
|
-
* changes
|
|
576
|
+
* changes its identity as lifecycle metadata advances, so scheduling reads it once across wake-ups.
|
|
561
577
|
* Dropped once the ticket is done.
|
|
562
578
|
* @type {Map<string, {key: string, block: string}>}
|
|
563
579
|
*/
|
|
@@ -643,6 +659,7 @@ export async function createWorker(opts, deps = {}) {
|
|
|
643
659
|
try { await logWarning({ warning: 'watch_tick_failed', message: text }); } catch { /* a log failure never stops dispatch */ }
|
|
644
660
|
}
|
|
645
661
|
const pending = pendingTickets(repoRoot).filter((id) => !poisoned.has(id) && !started.has(id));
|
|
662
|
+
if (pending.length > 0 || running.size > 0) activity += 1;
|
|
646
663
|
if (pending.length > 0 && limit === null) ({ limit, fairShare } = await poolSettings());
|
|
647
664
|
// Cache even unreadable hints for this tick: ranking never reads a ticket twice.
|
|
648
665
|
/** @type {Map<string, ReturnType<typeof fileKeyOf>>} */
|
|
@@ -700,6 +717,13 @@ export async function createWorker(opts, deps = {}) {
|
|
|
700
717
|
return done;
|
|
701
718
|
}
|
|
702
719
|
|
|
720
|
+
/** Process inspection belongs to the worker lifecycle, outside ticket dispatch and draining. */
|
|
721
|
+
async function cleanOrphans() {
|
|
722
|
+
try {
|
|
723
|
+
await sweepProcesses({ apply: true, orphansOnly: true, runId, zombies: false });
|
|
724
|
+
} catch { /* process hygiene must never fail the worker */ }
|
|
725
|
+
}
|
|
726
|
+
|
|
703
727
|
/**
|
|
704
728
|
* Resolve when one of `slots` settles, `pollMs` passes, or `stop()` is called.
|
|
705
729
|
* @param {Array<Promise<void>>} slots
|
|
@@ -748,24 +772,51 @@ export async function createWorker(opts, deps = {}) {
|
|
|
748
772
|
/** Serve the queue until `stop()`. */
|
|
749
773
|
async function run() {
|
|
750
774
|
announce();
|
|
775
|
+
let interval = pollMs;
|
|
776
|
+
let polled = false;
|
|
777
|
+
/** @type {import('node:fs').FSWatcher | null} */
|
|
778
|
+
let queueWatch = null;
|
|
751
779
|
try {
|
|
780
|
+
// Enqueue publishes a ticket by rename. Wake even during the longest idle back-off;
|
|
781
|
+
// heartbeat and temporary files do not wake an idle worker.
|
|
782
|
+
try {
|
|
783
|
+
queueWatch = watch(queueDir(repoRoot), (_event, file) => {
|
|
784
|
+
if (!file || !/^[a-f0-9]{24}\.json$/.test(String(file))) return;
|
|
785
|
+
activity += 1;
|
|
786
|
+
wake?.();
|
|
787
|
+
poolWake?.();
|
|
788
|
+
});
|
|
789
|
+
queueWatch.on('error', () => {
|
|
790
|
+
queueWatch?.close();
|
|
791
|
+
queueWatch = null;
|
|
792
|
+
wake?.();
|
|
793
|
+
});
|
|
794
|
+
} catch { /* without filesystem notifications, keep polling at the fast interval */ }
|
|
752
795
|
while (!stopping) {
|
|
796
|
+
const before = activity;
|
|
753
797
|
try {
|
|
754
|
-
await drain();
|
|
798
|
+
if (await drain() > 0) await cleanOrphans();
|
|
755
799
|
} catch {
|
|
756
800
|
// the queue directory itself failed to read: try again on the next poll
|
|
757
801
|
}
|
|
758
802
|
if (stopping) break;
|
|
759
|
-
|
|
760
|
-
|
|
803
|
+
const unchanged = before === activity;
|
|
804
|
+
interval = polled && unchanged ? Math.min(5000, interval * 2) : pollMs;
|
|
805
|
+
if (queueWatch === null && !deps.sleep) interval = pollMs;
|
|
806
|
+
polled = true;
|
|
807
|
+
await new Promise((resolve, reject) => {
|
|
808
|
+
let timer;
|
|
761
809
|
wake = () => {
|
|
762
810
|
clearTimeout(timer);
|
|
763
811
|
resolve(undefined);
|
|
764
812
|
};
|
|
813
|
+
if (deps.sleep) deps.sleep(interval).then(resolve, reject);
|
|
814
|
+
else timer = setTimeout(resolve, interval);
|
|
765
815
|
});
|
|
766
816
|
wake = null;
|
|
767
817
|
}
|
|
768
818
|
} finally {
|
|
819
|
+
queueWatch?.close();
|
|
769
820
|
release();
|
|
770
821
|
}
|
|
771
822
|
}
|
|
@@ -774,7 +825,9 @@ export async function createWorker(opts, deps = {}) {
|
|
|
774
825
|
async function once() {
|
|
775
826
|
announce();
|
|
776
827
|
try {
|
|
777
|
-
|
|
828
|
+
const done = await drain();
|
|
829
|
+
if (done > 0) await cleanOrphans();
|
|
830
|
+
return done;
|
|
778
831
|
} finally {
|
|
779
832
|
stop();
|
|
780
833
|
release();
|