claude-code-session-manager 0.25.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{TiptapBody-_BRc_wPu.js → TiptapBody-BsNr6F0B.js} +1 -1
- package/dist/assets/index-DU1pkhIQ.js +3533 -0
- package/dist/assets/index-Dwb94Uxm.css +32 -0
- package/dist/index.html +2 -2
- package/package.json +1 -1
- package/src/main/__tests__/dod-drain-hook.test.cjs +197 -0
- package/src/main/__tests__/dod-report.test.cjs +304 -0
- package/src/main/hives.cjs +96 -36
- package/src/main/lib/definitionOfDone.cjs +231 -2
- package/src/main/lib/dodDrainHook.cjs +81 -0
- package/src/main/scheduler.cjs +85 -22
- package/src/preload/api.d.ts +8 -7
- package/dist/assets/index-CK5Ob11w.js +0 -3525
- package/dist/assets/index-Cu9X6oyA.css +0 -32
|
@@ -11,7 +11,7 @@ const crypto = require('node:crypto');
|
|
|
11
11
|
const fs = require('node:fs');
|
|
12
12
|
const os = require('node:os');
|
|
13
13
|
const path = require('node:path');
|
|
14
|
-
const { spawn } = require('node:child_process');
|
|
14
|
+
const { spawn, spawnSync } = require('node:child_process');
|
|
15
15
|
const { splitFrontmatter } = require('./prdFrontmatter.cjs');
|
|
16
16
|
|
|
17
17
|
// Regex identifying meta/dod slugs that must NOT influence the batchKey.
|
|
@@ -313,4 +313,233 @@ async function reverifyBatch(jobs, { timeoutMs = 60_000, batchTimeoutMs = 600_00
|
|
|
313
313
|
return results;
|
|
314
314
|
}
|
|
315
315
|
|
|
316
|
-
|
|
316
|
+
// ─── Risk heuristics ──────────────────────────────────────────────────────────
|
|
317
|
+
// Conservative keyword/path matches — false positives are fine (they only add
|
|
318
|
+
// a "review recommended" line); the cost of a miss on a money path is higher.
|
|
319
|
+
const RISK_HEURISTICS = [
|
|
320
|
+
{ surface: 'money-path', re: /money|trade|order|position|price|payment/i },
|
|
321
|
+
{ surface: 'auth', re: /auth|token|credential|secret/i },
|
|
322
|
+
{ surface: 'migration', re: /migration|schema|alembic|\.sql/i },
|
|
323
|
+
];
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Extract a named heading section from a PRD body string.
|
|
327
|
+
* Returns the lines under the heading until the next sibling/parent heading.
|
|
328
|
+
* Complexity: O(L) where L = number of lines in body (bounded PRD document).
|
|
329
|
+
*
|
|
330
|
+
* @param {string} body
|
|
331
|
+
* @param {string} headingText Case-insensitive heading to find.
|
|
332
|
+
* @returns {string}
|
|
333
|
+
*/
|
|
334
|
+
function _extractSection(body, headingText) {
|
|
335
|
+
const escapedHeading = headingText.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
336
|
+
const headingRe = new RegExp(`^#+\\s+${escapedHeading}`, 'i');
|
|
337
|
+
const lines = body.split('\n');
|
|
338
|
+
let inSection = false;
|
|
339
|
+
let headingLevel = 0;
|
|
340
|
+
const sectionLines = [];
|
|
341
|
+
|
|
342
|
+
for (const line of lines) {
|
|
343
|
+
if (/^#+\s/.test(line)) {
|
|
344
|
+
const level = line.match(/^(#+)/)[1].length;
|
|
345
|
+
if (headingRe.test(line)) {
|
|
346
|
+
inSection = true;
|
|
347
|
+
headingLevel = level;
|
|
348
|
+
} else if (inSection && level <= headingLevel) {
|
|
349
|
+
inSection = false;
|
|
350
|
+
}
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
if (inSection) sectionLines.push(line);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
return sectionLines.join('\n');
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Inspect the files each job touched and flag risk surfaces.
|
|
361
|
+
*
|
|
362
|
+
* For each job:
|
|
363
|
+
* 1. If job.landedCommit is set, run `git show --name-only` (bounded by
|
|
364
|
+
* gitTimeoutMs) to obtain changed file paths.
|
|
365
|
+
* 2. Otherwise fall back to the text of the PRD's `# Implementation notes`
|
|
366
|
+
* section which authors are expected to list the files they plan to touch.
|
|
367
|
+
* Then check the candidate text against RISK_HEURISTICS (keyword/path match).
|
|
368
|
+
*
|
|
369
|
+
* Complexity: O(n * H) where n = jobs.length, H = RISK_HEURISTICS.length (3,
|
|
370
|
+
* a constant). Not user-scaled data — the scheduler queue is bounded.
|
|
371
|
+
*
|
|
372
|
+
* @param {Array<{ slug: string, cwd: string, landedCommit?: string }>} jobs
|
|
373
|
+
* @param {{ prdsDir?: string, gitTimeoutMs?: number }} opts
|
|
374
|
+
* @returns {Array<{ slug: string, surfaces: string[] }>} Only jobs with ≥1 hit.
|
|
375
|
+
*/
|
|
376
|
+
function flagRiskySurfaces(jobs, { prdsDir, gitTimeoutMs = 10_000 } = {}) {
|
|
377
|
+
const resolvedPrdsDir = prdsDir ?? PRDS_DIR;
|
|
378
|
+
const results = [];
|
|
379
|
+
|
|
380
|
+
for (const job of jobs) {
|
|
381
|
+
let candidateText = '';
|
|
382
|
+
|
|
383
|
+
// 1. Try git commit — bounded by gitTimeoutMs.
|
|
384
|
+
if (job.landedCommit && job.cwd) {
|
|
385
|
+
try {
|
|
386
|
+
const r = spawnSync(
|
|
387
|
+
'git',
|
|
388
|
+
['-C', job.cwd, 'show', '--name-only', '--format=', job.landedCommit],
|
|
389
|
+
{ timeout: gitTimeoutMs, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }
|
|
390
|
+
);
|
|
391
|
+
if (r.status === 0 && r.stdout) candidateText = r.stdout;
|
|
392
|
+
} catch { /* fall through to PRD body */ }
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// 2. Fallback: PRD # Implementation notes section.
|
|
396
|
+
// Guard: slug must contain only safe chars to prevent path traversal via a
|
|
397
|
+
// corrupted queue.json entry (e.g. slug: "../../.ssh/id_rsa").
|
|
398
|
+
if (!candidateText && job.slug && /^[\w-]+$/.test(job.slug)) {
|
|
399
|
+
try {
|
|
400
|
+
const raw = fs.readFileSync(path.join(resolvedPrdsDir, `${job.slug}.md`), 'utf8');
|
|
401
|
+
const body = splitFrontmatter(raw).body;
|
|
402
|
+
candidateText = _extractSection(body, 'Implementation notes');
|
|
403
|
+
} catch { /* no candidate text — skip */ }
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
if (!candidateText) continue;
|
|
407
|
+
|
|
408
|
+
// 3. Apply risk heuristics (O(H), constant).
|
|
409
|
+
const surfaces = RISK_HEURISTICS
|
|
410
|
+
.filter(h => h.re.test(candidateText))
|
|
411
|
+
.map(h => h.surface);
|
|
412
|
+
|
|
413
|
+
if (surfaces.length > 0) results.push({ slug: job.slug, surfaces });
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
return results;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// ─── Atomic write helper ───────────────────────────────────────────────────────
|
|
420
|
+
// Re-implements the tmp+rename recipe from config.cjs writeTextAtomic (sync
|
|
421
|
+
// variant), avoiding an import of Electron IPC code in a pure-node context.
|
|
422
|
+
// Cross-ref: src/main/config.cjs writeJsonSync (same pattern).
|
|
423
|
+
function _writeFileAtomic(absPath, text) {
|
|
424
|
+
const tmp = `${absPath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
425
|
+
try {
|
|
426
|
+
fs.writeFileSync(tmp, text, 'utf8');
|
|
427
|
+
fs.renameSync(tmp, absPath);
|
|
428
|
+
} catch (err) {
|
|
429
|
+
try { fs.unlinkSync(tmp); } catch { /* tmp never created or already gone */ }
|
|
430
|
+
throw err;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
const STATUS_EMOJI = { pass: '✅', fail: '❌', unverifiable: '⚠️' };
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* Write a definition-of-done report for a completed batch.
|
|
438
|
+
*
|
|
439
|
+
* The report contains:
|
|
440
|
+
* (a) a per-PRD AC table (slug · pass/fail/unverifiable),
|
|
441
|
+
* (b) a risk-flag summary,
|
|
442
|
+
* (c) a "needs human attention" list with a one-line recommendation per entry.
|
|
443
|
+
*
|
|
444
|
+
* The report is written atomically (tmp + rename) to avoid partial reads.
|
|
445
|
+
* Each call mints a fresh timestamped directory under runsDir, so two calls
|
|
446
|
+
* with the same key produce two separate files; use reportExists() to gate
|
|
447
|
+
* before calling.
|
|
448
|
+
*
|
|
449
|
+
* @param {string} key Output of batchKey().
|
|
450
|
+
* @param {{
|
|
451
|
+
* acResults: Array<{ slug: string, status: string, code: number|null, ms: number }>,
|
|
452
|
+
* riskFlags: Array<{ slug: string, surfaces: string[] }>,
|
|
453
|
+
* runsDir?: string,
|
|
454
|
+
* }} opts
|
|
455
|
+
* @returns {string} Absolute path of the written report.
|
|
456
|
+
*/
|
|
457
|
+
function writeReport(key, { acResults = [], riskFlags = [], runsDir } = {}) {
|
|
458
|
+
if (!/^[0-9a-f]+$/.test(key)) throw new Error(`invalid batchKey: ${key}`);
|
|
459
|
+
|
|
460
|
+
const resolvedRunsDir = runsDir ?? RUNS_DIR;
|
|
461
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
462
|
+
const dir = path.join(resolvedRunsDir, ts);
|
|
463
|
+
const reportPath = path.join(dir, `definition-of-done-${key}.md`);
|
|
464
|
+
|
|
465
|
+
// ── AC table ────────────────────────────────────────────────────────────────
|
|
466
|
+
const acRows = acResults.map(r => {
|
|
467
|
+
const emoji = STATUS_EMOJI[r.status] ?? '';
|
|
468
|
+
return `| ${r.slug} | ${emoji} ${r.status} |`;
|
|
469
|
+
});
|
|
470
|
+
const acTable = [
|
|
471
|
+
'| Slug | Status |',
|
|
472
|
+
'|------|--------|',
|
|
473
|
+
...acRows,
|
|
474
|
+
].join('\n');
|
|
475
|
+
|
|
476
|
+
// ── Risk flags table ─────────────────────────────────────────────────────────
|
|
477
|
+
let riskSection;
|
|
478
|
+
if (riskFlags.length === 0) {
|
|
479
|
+
riskSection = '_No risk surfaces detected._';
|
|
480
|
+
} else {
|
|
481
|
+
riskSection = [
|
|
482
|
+
'| Slug | Surfaces |',
|
|
483
|
+
'|------|----------|',
|
|
484
|
+
...riskFlags.map(r => `| ${r.slug} | ${r.surfaces.join(', ')} |`),
|
|
485
|
+
].join('\n');
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// ── Needs human attention ────────────────────────────────────────────────────
|
|
489
|
+
const attentionLines = [];
|
|
490
|
+
for (const r of acResults) {
|
|
491
|
+
if (r.status === 'fail') {
|
|
492
|
+
attentionLines.push(`- **${r.slug}** — AC failed; run \`/code-review\` on ${r.slug}`);
|
|
493
|
+
} else if (r.status === 'unverifiable') {
|
|
494
|
+
attentionLines.push(`- **${r.slug}** — unverifiable AC; manual inspection recommended for ${r.slug}`);
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
for (const r of riskFlags) {
|
|
498
|
+
attentionLines.push(
|
|
499
|
+
`- **${r.slug}** — touches ${r.surfaces.join(', ')}; run \`/code-review\` on ${r.slug} — ${r.surfaces.join('/')} path`
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
const attentionSection = attentionLines.length > 0
|
|
503
|
+
? attentionLines.join('\n')
|
|
504
|
+
: '_None — all checks passed and no risk surfaces detected._';
|
|
505
|
+
|
|
506
|
+
// ── Assemble report ──────────────────────────────────────────────────────────
|
|
507
|
+
const report = [
|
|
508
|
+
`# Definition of Done — batch ${key}`,
|
|
509
|
+
'',
|
|
510
|
+
`Generated: ${new Date().toISOString()}`,
|
|
511
|
+
'',
|
|
512
|
+
'> **Note:** This report flags surfaces for human review.',
|
|
513
|
+
'> Deep LLM code-review is **recommended, not auto-run** — the gate detects',
|
|
514
|
+
'> and flags; humans or `/code-review` do the deep pass.',
|
|
515
|
+
'',
|
|
516
|
+
'## AC Results',
|
|
517
|
+
'',
|
|
518
|
+
acTable,
|
|
519
|
+
'',
|
|
520
|
+
'## Risk Flags',
|
|
521
|
+
'',
|
|
522
|
+
riskSection,
|
|
523
|
+
'',
|
|
524
|
+
'## Needs Human Attention',
|
|
525
|
+
'',
|
|
526
|
+
attentionSection,
|
|
527
|
+
'',
|
|
528
|
+
].join('\n');
|
|
529
|
+
|
|
530
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
531
|
+
_writeFileAtomic(reportPath, report);
|
|
532
|
+
|
|
533
|
+
return reportPath;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
module.exports = {
|
|
537
|
+
batchKey,
|
|
538
|
+
reportPathFor,
|
|
539
|
+
reportExists,
|
|
540
|
+
extractAcCommand,
|
|
541
|
+
reverifyAc,
|
|
542
|
+
reverifyBatch,
|
|
543
|
+
flagRiskySurfaces,
|
|
544
|
+
writeReport,
|
|
545
|
+
};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* dodDrainHook.cjs — definition-of-done gate that fires at queue drain.
|
|
5
|
+
*
|
|
6
|
+
* Extracted so the scheduler can call it fire-and-forget (non-blocking)
|
|
7
|
+
* while tests drive it directly with await.
|
|
8
|
+
*
|
|
9
|
+
* Kill-switch: SM_DOD_DISABLE=1 (mirrors SM_SUPERVISOR_DISABLE convention).
|
|
10
|
+
* Loop-safe: batchKey excludes dod/meta slugs; reportExists() + _inFlight
|
|
11
|
+
* dedup make re-drains over the same completed set a single fs-stat no-op,
|
|
12
|
+
* even when tickQueue fires twice in quick succession before the first
|
|
13
|
+
* reverifyBatch completes.
|
|
14
|
+
* Non-blocking: callers should fire-and-forget via .catch(); this function
|
|
15
|
+
* never throws to the caller (errors are logged instead).
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const {
|
|
19
|
+
batchKey,
|
|
20
|
+
reportExists,
|
|
21
|
+
reverifyBatch,
|
|
22
|
+
flagRiskySurfaces,
|
|
23
|
+
writeReport,
|
|
24
|
+
} = require('./definitionOfDone.cjs');
|
|
25
|
+
|
|
26
|
+
// Track keys whose DoD pass is currently in-flight.
|
|
27
|
+
// Prevents concurrent tickQueue drains from spawning duplicate reverifyBatch
|
|
28
|
+
// runs for the same completed set before the first one has written its report.
|
|
29
|
+
const _inFlight = new Set();
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Run the definition-of-done gate when the scheduler queue drains.
|
|
33
|
+
*
|
|
34
|
+
* Guards (in order, fast-fail):
|
|
35
|
+
* 1. SM_DOD_DISABLE=1 → skip
|
|
36
|
+
* 2. state.paused (covers rate-limit) → defer, no report
|
|
37
|
+
* 3. cancelToken.cancelled → skip
|
|
38
|
+
* 4. No completed jobs → nothing to verify
|
|
39
|
+
* 5. reportExists(key) → already done for this batch, no-op
|
|
40
|
+
* 6. _inFlight.has(key) → concurrent drain for same batch, no-op
|
|
41
|
+
* 7. cancelToken.cancelled re-checked after reverifyBatch (up to 600s)
|
|
42
|
+
*
|
|
43
|
+
* Complexity: O(n) over completed jobs for batchKey + reverifyBatch;
|
|
44
|
+
* reportExists is O(d) where d = number of run subdirectories (bounded).
|
|
45
|
+
*
|
|
46
|
+
* @param {{ paused: object|null, jobs: Array }} state Scheduler queue state.
|
|
47
|
+
* @param {{
|
|
48
|
+
* cancelToken?: { cancelled: boolean },
|
|
49
|
+
* prdsDir?: string,
|
|
50
|
+
* runsDir?: string,
|
|
51
|
+
* }} opts
|
|
52
|
+
* @returns {Promise<void>}
|
|
53
|
+
*/
|
|
54
|
+
async function runDefinitionOfDoneOnDrain(state, opts = {}) {
|
|
55
|
+
const { cancelToken = {}, prdsDir, runsDir } = opts;
|
|
56
|
+
|
|
57
|
+
if (process.env.SM_DOD_DISABLE === '1') return;
|
|
58
|
+
if (state.paused) return;
|
|
59
|
+
if (cancelToken.cancelled) return;
|
|
60
|
+
|
|
61
|
+
const completedJobs = (state.jobs || []).filter(j => j.status === 'completed');
|
|
62
|
+
if (completedJobs.length === 0) return;
|
|
63
|
+
|
|
64
|
+
const key = batchKey(completedJobs);
|
|
65
|
+
|
|
66
|
+
if (reportExists(key, runsDir)) return;
|
|
67
|
+
if (_inFlight.has(key)) return;
|
|
68
|
+
|
|
69
|
+
_inFlight.add(key);
|
|
70
|
+
try {
|
|
71
|
+
const acResults = await reverifyBatch(completedJobs, { prdsDir });
|
|
72
|
+
// Re-check after the slow await — scheduler may have stopped mid-flight.
|
|
73
|
+
if (cancelToken.cancelled) return;
|
|
74
|
+
const riskFlags = flagRiskySurfaces(completedJobs, { prdsDir });
|
|
75
|
+
writeReport(key, { acResults, riskFlags, runsDir });
|
|
76
|
+
} finally {
|
|
77
|
+
_inFlight.delete(key);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = { runDefinitionOfDoneOnDrain };
|
package/src/main/scheduler.cjs
CHANGED
|
@@ -65,6 +65,7 @@ const {
|
|
|
65
65
|
MAX_JOB_DURATION_MS,
|
|
66
66
|
} = require('./lib/schedulerConfig.cjs');
|
|
67
67
|
const { pickForProject, pickNextBatch, DEFAULT_PROJECT_CWD } = require('./lib/schedulerBatch.cjs');
|
|
68
|
+
const { runDefinitionOfDoneOnDrain } = require('./lib/dodDrainHook.cjs');
|
|
68
69
|
|
|
69
70
|
const MAX_INVESTIGATION_DURATION_MS = 30 * 60_000;
|
|
70
71
|
|
|
@@ -220,9 +221,27 @@ const ENV_CAP = process.env.SM_SCHEDULER_MAX_CONCURRENCY
|
|
|
220
221
|
? Math.max(1, Math.min(20, parseInt(process.env.SM_SCHEDULER_MAX_CONCURRENCY, 10) || 3))
|
|
221
222
|
: null;
|
|
222
223
|
|
|
223
|
-
// Each headless claude -p
|
|
224
|
-
// per running+pending slot
|
|
225
|
-
|
|
224
|
+
// Each headless claude -p job can shell out to tsc/vite/pytest and grow well
|
|
225
|
+
// past 1 GB at peak; reserve 2.5 GB per running+pending slot. Raised from 1.5 GB
|
|
226
|
+
// after the 2026-06-16 OOM: 3 concurrent cross-project jobs + their build
|
|
227
|
+
// subprocesses pushed a 24 GB host ~10 GB into swap and the OOM killer took
|
|
228
|
+
// Electron — every pty got SIGHUP (code=0 signal=1).
|
|
229
|
+
const MIN_FREE_MB_PER_JOB = 2500;
|
|
230
|
+
|
|
231
|
+
// Absolute headroom kept free for the Electron host (main + renderer + GPU) and
|
|
232
|
+
// the OS — NEVER lent to jobs. Without it the gate green-lights a job whenever
|
|
233
|
+
// MemAvailable is just above per-job need, starving the very process that owns
|
|
234
|
+
// the ptys; that's the one the OOM killer then reaps. Subtracted from
|
|
235
|
+
// MemAvailable before the per-job gate runs. See availableForJobs().
|
|
236
|
+
const RESERVED_HOST_MB = 3000;
|
|
237
|
+
|
|
238
|
+
// oom_score_adj applied to each spawned claude -p job (range -1000..1000;
|
|
239
|
+
// Electron inherits the default 0). A positive bias makes the kernel OOM killer
|
|
240
|
+
// prefer a disposable, restartable job over Electron — whose death SIGHUPs every
|
|
241
|
+
// pty and drops the sleep inhibitor. The job's build subprocesses (tsc/vite)
|
|
242
|
+
// inherit it, so the actual memory hogs are the preferred victims. The gate caps
|
|
243
|
+
// how many START; this decides who dies if a spike slips through anyway.
|
|
244
|
+
const OOM_SCORE_ADJ_JOB = 500;
|
|
226
245
|
|
|
227
246
|
const DEFAULT_CONFIG = {
|
|
228
247
|
offsetMinutes: 15,
|
|
@@ -276,6 +295,33 @@ function memoryLimitedBatchSize(availableMb, minPerJob, runningCount, batchLen)
|
|
|
276
295
|
return allowed;
|
|
277
296
|
}
|
|
278
297
|
|
|
298
|
+
/**
|
|
299
|
+
* Memory available to LAUNCH jobs with: MemAvailable minus the host reserve,
|
|
300
|
+
* floored at 0. Keeping the host reserve out of the job budget is what stops the
|
|
301
|
+
* gate from green-lighting a job into an OOM that kills Electron. Fails open
|
|
302
|
+
* (Infinity) on non-Linux where MemAvailable is unknown. Exported for tests.
|
|
303
|
+
*/
|
|
304
|
+
function availableForJobs(availableMb, reservedHostMb) {
|
|
305
|
+
if (availableMb === Infinity) return Infinity;
|
|
306
|
+
return Math.max(0, availableMb - reservedHostMb);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Bias a spawned job's oom_score_adj up so the kernel OOM killer sacrifices the
|
|
311
|
+
* (restartable) job before Electron. Raising a child's OWN score is privilege-
|
|
312
|
+
* free; lowering Electron's would need CAP_SYS_RESOURCE. Linux-only, best-effort
|
|
313
|
+
* — the job may have already exited (write ENOENTs), which is fine. See the
|
|
314
|
+
* 2026-06-16 OOM-kills-Electron incident.
|
|
315
|
+
*/
|
|
316
|
+
function biasJobOomScore(pid) {
|
|
317
|
+
if (process.platform !== 'linux' || !pid) return;
|
|
318
|
+
try {
|
|
319
|
+
fs.writeFileSync(`/proc/${pid}/oom_score_adj`, String(OOM_SCORE_ADJ_JOB));
|
|
320
|
+
} catch {
|
|
321
|
+
/* job already exited, or /proc unavailable — best-effort hardening only */
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
279
325
|
// ---------- fs helpers ----------
|
|
280
326
|
|
|
281
327
|
/**
|
|
@@ -1037,6 +1083,8 @@ async function executeJob(job, runDir, defaultCwd, onPid) {
|
|
|
1037
1083
|
|
|
1038
1084
|
if (child) {
|
|
1039
1085
|
safeLog(`[scheduler] spawned pid=${child.pid} sessionId=${sessionId} (process group)\n\n`);
|
|
1086
|
+
// Make this job the OOM killer's preferred victim over Electron.
|
|
1087
|
+
biasJobOomScore(child.pid);
|
|
1040
1088
|
// Fire-and-forget pid persistence — best effort.
|
|
1041
1089
|
if (onPid) onPid(child.pid, sessionId, cwd).catch(() => {});
|
|
1042
1090
|
}
|
|
@@ -1413,19 +1461,22 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
|
|
|
1413
1461
|
await broadcast();
|
|
1414
1462
|
|
|
1415
1463
|
if (actuallyFailed && failedJobSnapshot) {
|
|
1416
|
-
// Transient-failure detector
|
|
1417
|
-
//
|
|
1418
|
-
//
|
|
1419
|
-
//
|
|
1420
|
-
//
|
|
1421
|
-
//
|
|
1422
|
-
//
|
|
1423
|
-
//
|
|
1424
|
-
//
|
|
1425
|
-
//
|
|
1426
|
-
//
|
|
1464
|
+
// Transient-failure detector. A 143/137 exit is ALWAYS a signal kill — the
|
|
1465
|
+
// agent never self-exits with those — so the only question is WHO killed it.
|
|
1466
|
+
// The scheduler's own intentional kills before the idle watchdog are the
|
|
1467
|
+
// result-tail post-success kill (which maps back to exit 0, so it never
|
|
1468
|
+
// reaches here) and the rare supervisor kill-agent. The idle-output
|
|
1469
|
+
// watchdog only fires at IDLE_OUTPUT_KILL_MS (20 min) of stalled output,
|
|
1470
|
+
// and the deadman at 4 h. THEREFORE any 143/137 with a run shorter than the
|
|
1471
|
+
// idle threshold is an EXTERNAL/transient kill — an app restart (incl. our
|
|
1472
|
+
// own self-restart on publish/HMR — see feedback 2026-06-15-01), an
|
|
1473
|
+
// OOM-kill, or a manual kill — not a real failure. Re-queue it (bounded)
|
|
1474
|
+
// instead of marking it failed + spawning a spurious fix-plan. (Was 45 s,
|
|
1475
|
+
// which wrongly hard-failed externally-killed jobs like 115 SIGTERM'd at
|
|
1476
|
+
// 67 s.) Genuine watchdog kills (idle ≥20 min, deadman 4 h) run longer than
|
|
1477
|
+
// the threshold and still fall through to investigation.
|
|
1427
1478
|
const ec = failedJobSnapshot.exitCode;
|
|
1428
|
-
const transient = (ec === 143 || ec === 137) && res.durationMs <
|
|
1479
|
+
const transient = (ec === 143 || ec === 137) && res.durationMs < IDLE_OUTPUT_KILL_MS;
|
|
1429
1480
|
const retries = failedJobSnapshot.transientRetries ?? 0;
|
|
1430
1481
|
if (transient && retries < 2) {
|
|
1431
1482
|
console.log(`[scheduler] transient failure (exit=${ec} dur=${res.durationMs}ms) — auto-retry ${retries + 1}/2 for ${job.slug}`);
|
|
@@ -1469,20 +1520,31 @@ function tickQueue() {
|
|
|
1469
1520
|
await reconcile(state);
|
|
1470
1521
|
const cap = ENV_CAP ?? state.config.concurrencyCap;
|
|
1471
1522
|
const batch = pickNextBatch(state.jobs, runningSet, cap);
|
|
1472
|
-
if (batch.length === 0)
|
|
1523
|
+
if (batch.length === 0) {
|
|
1524
|
+
// Queue drained — run the definition-of-done gate fire-and-forget.
|
|
1525
|
+
// Non-blocking: does not hold the mutate lock; errors are logged, not thrown.
|
|
1526
|
+
runDefinitionOfDoneOnDrain(state, { cancelToken }).catch((err) => {
|
|
1527
|
+
console.log(`[scheduler] dod-drain: ${err?.message ?? String(err)}`);
|
|
1528
|
+
});
|
|
1529
|
+
return;
|
|
1530
|
+
}
|
|
1473
1531
|
|
|
1474
1532
|
const availableMb = getAvailableMemMb();
|
|
1475
|
-
|
|
1533
|
+
// Reserve a fixed slice for the Electron host before the per-job gate, so a
|
|
1534
|
+
// job is never started into the host's own headroom (that path OOM-kills
|
|
1535
|
+
// Electron and SIGHUPs every pty — 2026-06-16 incident).
|
|
1536
|
+
const jobBudgetMb = availableForJobs(availableMb, RESERVED_HOST_MB);
|
|
1537
|
+
const allowed = memoryLimitedBatchSize(jobBudgetMb, MIN_FREE_MB_PER_JOB, runningSet.size, batch.length);
|
|
1476
1538
|
if (allowed === 0) {
|
|
1477
|
-
const threshold = MIN_FREE_MB_PER_JOB * (runningSet.size + 1);
|
|
1478
|
-
console.log(`[scheduler] memory gate: available=${availableMb} MB < threshold=${threshold} MB — deferring ${batch.length} job(s)`);
|
|
1539
|
+
const threshold = RESERVED_HOST_MB + MIN_FREE_MB_PER_JOB * (runningSet.size + 1);
|
|
1540
|
+
console.log(`[scheduler] memory gate: available=${availableMb} MB < threshold=${threshold} MB (host reserve ${RESERVED_HOST_MB} + ${MIN_FREE_MB_PER_JOB}/job × ${runningSet.size + 1}) — deferring ${batch.length} job(s)`);
|
|
1479
1541
|
lastMemGate = { availableMb, threshold, deferred: true, at: new Date().toISOString() };
|
|
1480
1542
|
return;
|
|
1481
1543
|
}
|
|
1482
1544
|
const gatedBatch = batch.slice(0, allowed);
|
|
1483
1545
|
if (gatedBatch.length < batch.length) {
|
|
1484
|
-
console.log(`[scheduler] memory gate: available=${availableMb} MB — clamped batch ${batch.length} → ${gatedBatch.length}`);
|
|
1485
|
-
lastMemGate = { availableMb, threshold: MIN_FREE_MB_PER_JOB * (runningSet.size + gatedBatch.length), deferred: false, clamped: true, at: new Date().toISOString() };
|
|
1546
|
+
console.log(`[scheduler] memory gate: available=${availableMb} MB — clamped batch ${batch.length} → ${gatedBatch.length} (host reserve ${RESERVED_HOST_MB} + ${MIN_FREE_MB_PER_JOB}/job)`);
|
|
1547
|
+
lastMemGate = { availableMb, threshold: RESERVED_HOST_MB + MIN_FREE_MB_PER_JOB * (runningSet.size + gatedBatch.length), deferred: false, clamped: true, at: new Date().toISOString() };
|
|
1486
1548
|
} else {
|
|
1487
1549
|
// Ungated full batch: clear stale gate snapshot so status doesn't show
|
|
1488
1550
|
// a stale deferral from a previous tick.
|
|
@@ -2153,6 +2215,7 @@ async function init() {
|
|
|
2153
2215
|
for (const j of s.jobs) counts[j.status] = (counts[j.status] || 0) + 1;
|
|
2154
2216
|
appendHeartbeat({
|
|
2155
2217
|
ts: Date.now(),
|
|
2218
|
+
pid: process.pid,
|
|
2156
2219
|
counts,
|
|
2157
2220
|
paused: s.paused ? { reason: s.paused.reason, resumeAt: s.paused.resumeAt } : null,
|
|
2158
2221
|
nextReset: cachedNextReset,
|
|
@@ -2290,4 +2353,4 @@ const remote = {
|
|
|
2290
2353
|
},
|
|
2291
2354
|
};
|
|
2292
2355
|
|
|
2293
|
-
module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, reverifyNeedsReview, isRescanCandidate };
|
|
2356
|
+
module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate };
|
package/src/preload/api.d.ts
CHANGED
|
@@ -533,16 +533,17 @@ export interface RepoAnalyzeResult {
|
|
|
533
533
|
}
|
|
534
534
|
export interface RepoAnalyzeError { ok: false; error: string }
|
|
535
535
|
|
|
536
|
-
export interface
|
|
537
|
-
export interface
|
|
536
|
+
export interface RecipeStep { agentName: string; note?: string }
|
|
537
|
+
export interface Recipe {
|
|
538
538
|
slug: string;
|
|
539
539
|
name: string;
|
|
540
540
|
description: string;
|
|
541
|
-
|
|
542
|
-
|
|
541
|
+
brief?: string;
|
|
542
|
+
steps: RecipeStep[];
|
|
543
543
|
}
|
|
544
|
-
|
|
545
|
-
export interface
|
|
544
|
+
// Channel keeps legacy "hives" name; concept is now "recipe".
|
|
545
|
+
export interface HiveListResult { hives: Recipe[]; error: string | null }
|
|
546
|
+
export interface HiveGetResult { hive: Recipe | null; error: string | null }
|
|
546
547
|
export interface HiveMutationResult { ok: boolean; error: string | null }
|
|
547
548
|
|
|
548
549
|
export interface WatcherInfo {
|
|
@@ -975,7 +976,7 @@ export interface SessionManagerAPI {
|
|
|
975
976
|
hives: {
|
|
976
977
|
list: () => Promise<HiveListResult>;
|
|
977
978
|
get: (slug: string) => Promise<HiveGetResult>;
|
|
978
|
-
save: (slug: string, hive:
|
|
979
|
+
save: (slug: string, hive: Recipe) => Promise<HiveMutationResult>;
|
|
979
980
|
delete: (slug: string) => Promise<HiveMutationResult>;
|
|
980
981
|
};
|
|
981
982
|
history: {
|