@claude-flow/cli 3.26.1 → 3.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,110 @@
1
+ /**
2
+ * #2661 — Global AI launch budget (emergency cost fuse).
3
+ *
4
+ * Every autonomous `claude --print` launch across ALL ruflo daemons in ALL
5
+ * worktrees/workspaces owned by the current user must pass through this
6
+ * user-global budget before a process is created. Without it, N worktree
7
+ * daemons each schedule their own AI workers and aggregate launch volume
8
+ * scales linearly with worktree count — enough to exhaust a user's Claude
9
+ * hourly quota silently (13 launches/hour/daemon under the legacy schedule).
10
+ *
11
+ * The ledger lives under the user's home directory (NOT the workspace) so
12
+ * daemons started from different worktrees of the same repository — or from
13
+ * unrelated repositories — all share one budget:
14
+ *
15
+ * ~/.claude-flow/ai-budget.json launch ledger + circuit breaker
16
+ * ~/.claude-flow/ai-budget.lock O_EXCL mutation lock
17
+ * ~/.claude-flow/ai-budget-receipts.jsonl launch/deny/pause receipts
18
+ *
19
+ * Files are owner-only (0700 dir / 0600 files) and symlinks are rejected
20
+ * (invariant 9 of #2661). All checks happen BEFORE process creation and the
21
+ * ledger mutation is atomic under the lock, so two daemons racing for the
22
+ * last hourly slot cannot both win.
23
+ *
24
+ * Default limits (issue #2661 containment):
25
+ * maxConcurrentGlobal 1 at most one autonomous claude child, user-wide
26
+ * maxLaunchesPerHour 2
27
+ * maxLaunchesPerDay 12
28
+ * pauseOnQuotaErrorMinutes 60 circuit breaker on 429/quota responses
29
+ */
30
+ export interface AiBudgetLimits {
31
+ maxConcurrentGlobal: number;
32
+ maxLaunchesPerHour: number;
33
+ maxLaunchesPerDay: number;
34
+ pauseOnQuotaErrorMinutes: number;
35
+ }
36
+ export declare const DEFAULT_AI_BUDGET_LIMITS: AiBudgetLimits;
37
+ export interface AiBudgetRequest {
38
+ workerType: string;
39
+ model: string;
40
+ /** Worktree/workspace root requesting the launch (recorded for receipts only). */
41
+ workspace: string;
42
+ }
43
+ export interface AiBudgetPermit {
44
+ allowed: boolean;
45
+ permitId?: string;
46
+ reason?: string;
47
+ }
48
+ /**
49
+ * Heuristic match for Anthropic quota / rate-limit failures. Only ever
50
+ * applied to ERROR output of a FAILED launch (never to successful analysis
51
+ * output, which may legitimately discuss "rate limiting" in the user's code).
52
+ */
53
+ export declare function isQuotaErrorText(text: string | undefined): boolean;
54
+ export declare class GlobalAiBudget {
55
+ private readonly dir;
56
+ private readonly ledgerFile;
57
+ private readonly lockFile;
58
+ private readonly receiptsFile;
59
+ private readonly limits;
60
+ constructor(options?: {
61
+ baseDir?: string;
62
+ limits?: Partial<AiBudgetLimits>;
63
+ });
64
+ getLimits(): AiBudgetLimits;
65
+ /**
66
+ * Atomically reserve one launch slot. Denials carry a machine-readable
67
+ * reason and are receipted. The reservation is counted as a launch
68
+ * immediately (the hourly/daily invariant is on launches, not completions);
69
+ * `release()` only frees the concurrency slot.
70
+ *
71
+ * Fails CLOSED: if the ledger cannot be read or locked, the launch is
72
+ * denied — an unaccountable launch is exactly what this fuse exists to
73
+ * prevent. `RUFLO_AI_BUDGET_DISABLE=1` is the explicit escape hatch.
74
+ */
75
+ reserve(req: AiBudgetRequest): Promise<AiBudgetPermit>;
76
+ /** Free the concurrency slot held by a permit. Best-effort. */
77
+ release(permitId: string | undefined): Promise<void>;
78
+ /**
79
+ * Open the user-global circuit breaker: a quota/429 response from ANY
80
+ * daemon pauses ALL autonomous Claude launches for the cooldown window.
81
+ */
82
+ recordQuotaError(detail: string): Promise<void>;
83
+ /** Snapshot for `daemon status` / diagnostics. */
84
+ getUsage(): {
85
+ lastHour: number;
86
+ lastDay: number;
87
+ active: number;
88
+ pausedUntil?: number;
89
+ pauseReason?: string;
90
+ /** #2661 — 24h launch counts per worktree/workspace, most active first. */
91
+ byWorkspace: Array<{
92
+ workspace: string;
93
+ launches: number;
94
+ }>;
95
+ };
96
+ private ensureDir;
97
+ private acquireLock;
98
+ /** Read + prune the ledger. Caller must hold the lock for read-modify-write. */
99
+ private readLedger;
100
+ private writeLedger;
101
+ /**
102
+ * Invariant 10: every launch, denial, and pause emits a receipt. Only
103
+ * operational metadata is persisted — never prompts or source content.
104
+ */
105
+ private appendReceipt;
106
+ }
107
+ export declare function getGlobalAiBudget(): GlobalAiBudget;
108
+ /** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
109
+ export declare function resetGlobalAiBudgetForTests(): void;
110
+ //# sourceMappingURL=global-ai-budget.d.ts.map
@@ -0,0 +1,359 @@
1
+ /**
2
+ * #2661 — Global AI launch budget (emergency cost fuse).
3
+ *
4
+ * Every autonomous `claude --print` launch across ALL ruflo daemons in ALL
5
+ * worktrees/workspaces owned by the current user must pass through this
6
+ * user-global budget before a process is created. Without it, N worktree
7
+ * daemons each schedule their own AI workers and aggregate launch volume
8
+ * scales linearly with worktree count — enough to exhaust a user's Claude
9
+ * hourly quota silently (13 launches/hour/daemon under the legacy schedule).
10
+ *
11
+ * The ledger lives under the user's home directory (NOT the workspace) so
12
+ * daemons started from different worktrees of the same repository — or from
13
+ * unrelated repositories — all share one budget:
14
+ *
15
+ * ~/.claude-flow/ai-budget.json launch ledger + circuit breaker
16
+ * ~/.claude-flow/ai-budget.lock O_EXCL mutation lock
17
+ * ~/.claude-flow/ai-budget-receipts.jsonl launch/deny/pause receipts
18
+ *
19
+ * Files are owner-only (0700 dir / 0600 files) and symlinks are rejected
20
+ * (invariant 9 of #2661). All checks happen BEFORE process creation and the
21
+ * ledger mutation is atomic under the lock, so two daemons racing for the
22
+ * last hourly slot cannot both win.
23
+ *
24
+ * Default limits (issue #2661 containment):
25
+ * maxConcurrentGlobal 1 at most one autonomous claude child, user-wide
26
+ * maxLaunchesPerHour 2
27
+ * maxLaunchesPerDay 12
28
+ * pauseOnQuotaErrorMinutes 60 circuit breaker on 429/quota responses
29
+ */
30
+ import * as fs from 'fs';
31
+ import { join } from 'path';
32
+ import { homedir } from 'os';
33
+ export const DEFAULT_AI_BUDGET_LIMITS = {
34
+ maxConcurrentGlobal: 1,
35
+ maxLaunchesPerHour: 2,
36
+ maxLaunchesPerDay: 12,
37
+ pauseOnQuotaErrorMinutes: 60,
38
+ };
39
+ const HOUR_MS = 60 * 60 * 1000;
40
+ const DAY_MS = 24 * HOUR_MS;
41
+ // Active reservations older than this are treated as abandoned (crashed
42
+ // daemon). Must exceed the daemon's 16-min worker timeout with margin.
43
+ const ACTIVE_STALE_MS = 30 * 60 * 1000;
44
+ // A mutation lock older than this belongs to a crashed process — take it over.
45
+ const LOCK_STALE_MS = 10_000;
46
+ const RECEIPTS_MAX_BYTES = 512 * 1024;
47
+ const RECEIPTS_KEEP_LINES = 200;
48
+ /**
49
+ * Heuristic match for Anthropic quota / rate-limit failures. Only ever
50
+ * applied to ERROR output of a FAILED launch (never to successful analysis
51
+ * output, which may legitimately discuss "rate limiting" in the user's code).
52
+ */
53
+ export function isQuotaErrorText(text) {
54
+ if (!text)
55
+ return false;
56
+ return /\b429\b|rate[\s_-]?limit|usage[\s_-]?limit|quota|too many requests|overloaded_error/i.test(text);
57
+ }
58
+ function envPositiveInt(name) {
59
+ const raw = process.env[name];
60
+ if (raw === undefined || raw.trim() === '')
61
+ return undefined;
62
+ const n = Number.parseInt(raw, 10);
63
+ return Number.isFinite(n) && n >= 0 ? n : undefined;
64
+ }
65
+ function isProcessAlive(pid) {
66
+ try {
67
+ process.kill(pid, 0);
68
+ return true;
69
+ }
70
+ catch {
71
+ return false;
72
+ }
73
+ }
74
+ /** Invariant 9: registry files must never be symlinks. */
75
+ function assertNotSymlink(path) {
76
+ try {
77
+ const st = fs.lstatSync(path);
78
+ if (st.isSymbolicLink()) {
79
+ throw new Error(`AI budget file is a symlink (refusing): ${path}`);
80
+ }
81
+ }
82
+ catch (e) {
83
+ if (e.code === 'ENOENT')
84
+ return;
85
+ throw e;
86
+ }
87
+ }
88
+ function delay(ms) {
89
+ return new Promise((r) => setTimeout(r, ms));
90
+ }
91
+ export class GlobalAiBudget {
92
+ dir;
93
+ ledgerFile;
94
+ lockFile;
95
+ receiptsFile;
96
+ limits;
97
+ constructor(options) {
98
+ this.dir = options?.baseDir
99
+ ?? process.env.RUFLO_AI_BUDGET_DIR
100
+ ?? join(homedir(), '.claude-flow');
101
+ this.ledgerFile = join(this.dir, 'ai-budget.json');
102
+ this.lockFile = join(this.dir, 'ai-budget.lock');
103
+ this.receiptsFile = join(this.dir, 'ai-budget-receipts.jsonl');
104
+ this.limits = {
105
+ maxConcurrentGlobal: options?.limits?.maxConcurrentGlobal
106
+ ?? envPositiveInt('RUFLO_AI_MAX_CONCURRENT')
107
+ ?? DEFAULT_AI_BUDGET_LIMITS.maxConcurrentGlobal,
108
+ maxLaunchesPerHour: options?.limits?.maxLaunchesPerHour
109
+ ?? envPositiveInt('RUFLO_AI_MAX_PER_HOUR')
110
+ ?? DEFAULT_AI_BUDGET_LIMITS.maxLaunchesPerHour,
111
+ maxLaunchesPerDay: options?.limits?.maxLaunchesPerDay
112
+ ?? envPositiveInt('RUFLO_AI_MAX_PER_DAY')
113
+ ?? DEFAULT_AI_BUDGET_LIMITS.maxLaunchesPerDay,
114
+ pauseOnQuotaErrorMinutes: options?.limits?.pauseOnQuotaErrorMinutes
115
+ ?? envPositiveInt('RUFLO_AI_QUOTA_PAUSE_MINUTES')
116
+ ?? DEFAULT_AI_BUDGET_LIMITS.pauseOnQuotaErrorMinutes,
117
+ };
118
+ }
119
+ getLimits() {
120
+ return { ...this.limits };
121
+ }
122
+ /**
123
+ * Atomically reserve one launch slot. Denials carry a machine-readable
124
+ * reason and are receipted. The reservation is counted as a launch
125
+ * immediately (the hourly/daily invariant is on launches, not completions);
126
+ * `release()` only frees the concurrency slot.
127
+ *
128
+ * Fails CLOSED: if the ledger cannot be read or locked, the launch is
129
+ * denied — an unaccountable launch is exactly what this fuse exists to
130
+ * prevent. `RUFLO_AI_BUDGET_DISABLE=1` is the explicit escape hatch.
131
+ */
132
+ async reserve(req) {
133
+ if (process.env.RUFLO_AI_BUDGET_DISABLE === '1') {
134
+ return { allowed: true, permitId: `bypass_${Date.now()}_${process.pid}` };
135
+ }
136
+ let unlock = null;
137
+ try {
138
+ unlock = await this.acquireLock();
139
+ const now = Date.now();
140
+ const ledger = this.readLedger(now);
141
+ let reason = null;
142
+ if (ledger.pausedUntil && ledger.pausedUntil > now) {
143
+ reason = `circuit-open until ${new Date(ledger.pausedUntil).toISOString()} (${ledger.pauseReason ?? 'quota error'})`;
144
+ }
145
+ else if (ledger.active.length >= this.limits.maxConcurrentGlobal) {
146
+ reason = `global-concurrency (${ledger.active.length}/${this.limits.maxConcurrentGlobal} active)`;
147
+ }
148
+ else {
149
+ const lastHour = ledger.launches.filter((l) => now - l.at < HOUR_MS).length;
150
+ const lastDay = ledger.launches.length; // already pruned to 24h
151
+ if (lastHour >= this.limits.maxLaunchesPerHour) {
152
+ reason = `hourly-budget (${lastHour}/${this.limits.maxLaunchesPerHour} in last hour)`;
153
+ }
154
+ else if (lastDay >= this.limits.maxLaunchesPerDay) {
155
+ reason = `daily-budget (${lastDay}/${this.limits.maxLaunchesPerDay} in last 24h)`;
156
+ }
157
+ }
158
+ if (reason) {
159
+ this.appendReceipt({ event: 'deny', at: now, reason, ...req });
160
+ return { allowed: false, reason };
161
+ }
162
+ const permitId = `permit_${now}_${process.pid}_${Math.random().toString(36).slice(2, 8)}`;
163
+ ledger.launches.push({ at: now, pid: process.pid, workerType: req.workerType, model: req.model, workspace: req.workspace });
164
+ ledger.active.push({ permitId, at: now, pid: process.pid, workerType: req.workerType });
165
+ this.writeLedger(ledger);
166
+ this.appendReceipt({ event: 'launch', at: now, permitId, ...req });
167
+ return { allowed: true, permitId };
168
+ }
169
+ catch (e) {
170
+ // Fail closed — see docstring.
171
+ const reason = `budget-ledger-error: ${e instanceof Error ? e.message : String(e)}`;
172
+ try {
173
+ this.appendReceipt({ event: 'deny', at: Date.now(), reason, ...req });
174
+ }
175
+ catch { /* best-effort */ }
176
+ return { allowed: false, reason };
177
+ }
178
+ finally {
179
+ unlock?.();
180
+ }
181
+ }
182
+ /** Free the concurrency slot held by a permit. Best-effort. */
183
+ async release(permitId) {
184
+ if (!permitId || permitId.startsWith('bypass_'))
185
+ return;
186
+ let unlock = null;
187
+ try {
188
+ unlock = await this.acquireLock();
189
+ const ledger = this.readLedger(Date.now());
190
+ const before = ledger.active.length;
191
+ ledger.active = ledger.active.filter((a) => a.permitId !== permitId);
192
+ if (ledger.active.length !== before)
193
+ this.writeLedger(ledger);
194
+ }
195
+ catch {
196
+ // Abandoned reservations expire via ACTIVE_STALE_MS pruning.
197
+ }
198
+ finally {
199
+ unlock?.();
200
+ }
201
+ }
202
+ /**
203
+ * Open the user-global circuit breaker: a quota/429 response from ANY
204
+ * daemon pauses ALL autonomous Claude launches for the cooldown window.
205
+ */
206
+ async recordQuotaError(detail) {
207
+ let unlock = null;
208
+ try {
209
+ unlock = await this.acquireLock();
210
+ const now = Date.now();
211
+ const ledger = this.readLedger(now);
212
+ ledger.pausedUntil = now + this.limits.pauseOnQuotaErrorMinutes * 60 * 1000;
213
+ ledger.pauseReason = detail.slice(0, 200);
214
+ this.writeLedger(ledger);
215
+ this.appendReceipt({ event: 'quota-pause', at: now, until: ledger.pausedUntil, detail: ledger.pauseReason });
216
+ }
217
+ catch {
218
+ // best-effort — the hourly budget still bounds retries
219
+ }
220
+ finally {
221
+ unlock?.();
222
+ }
223
+ }
224
+ /** Snapshot for `daemon status` / diagnostics. */
225
+ getUsage() {
226
+ try {
227
+ const now = Date.now();
228
+ const ledger = this.readLedger(now);
229
+ const byWs = new Map();
230
+ for (const l of ledger.launches) {
231
+ byWs.set(l.workspace, (byWs.get(l.workspace) ?? 0) + 1);
232
+ }
233
+ return {
234
+ lastHour: ledger.launches.filter((l) => now - l.at < HOUR_MS).length,
235
+ lastDay: ledger.launches.length,
236
+ active: ledger.active.length,
237
+ pausedUntil: ledger.pausedUntil && ledger.pausedUntil > now ? ledger.pausedUntil : undefined,
238
+ pauseReason: ledger.pausedUntil && ledger.pausedUntil > now ? ledger.pauseReason : undefined,
239
+ byWorkspace: Array.from(byWs.entries())
240
+ .map(([workspace, launches]) => ({ workspace, launches }))
241
+ .sort((a, b) => b.launches - a.launches),
242
+ };
243
+ }
244
+ catch {
245
+ return { lastHour: 0, lastDay: 0, active: 0, byWorkspace: [] };
246
+ }
247
+ }
248
+ // -- internals ----------------------------------------------------------
249
+ ensureDir() {
250
+ if (!fs.existsSync(this.dir)) {
251
+ fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
252
+ }
253
+ }
254
+ async acquireLock() {
255
+ this.ensureDir();
256
+ const deadline = Date.now() + 2000;
257
+ for (;;) {
258
+ try {
259
+ const fd = fs.openSync(this.lockFile, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, 0o600);
260
+ fs.writeSync(fd, String(process.pid));
261
+ fs.closeSync(fd);
262
+ return () => {
263
+ try {
264
+ fs.unlinkSync(this.lockFile);
265
+ }
266
+ catch { /* already gone */ }
267
+ };
268
+ }
269
+ catch (e) {
270
+ if (e.code !== 'EEXIST')
271
+ throw e;
272
+ // Stale lock from a crashed process — take over.
273
+ try {
274
+ const st = fs.lstatSync(this.lockFile);
275
+ if (Date.now() - st.mtimeMs > LOCK_STALE_MS) {
276
+ fs.unlinkSync(this.lockFile);
277
+ continue;
278
+ }
279
+ }
280
+ catch { /* raced — retry */ }
281
+ if (Date.now() > deadline) {
282
+ throw new Error('timed out acquiring ai-budget lock');
283
+ }
284
+ await delay(25);
285
+ }
286
+ }
287
+ }
288
+ /** Read + prune the ledger. Caller must hold the lock for read-modify-write. */
289
+ readLedger(now) {
290
+ assertNotSymlink(this.ledgerFile);
291
+ let ledger = { version: 1, launches: [], active: [] };
292
+ if (fs.existsSync(this.ledgerFile)) {
293
+ try {
294
+ const raw = JSON.parse(fs.readFileSync(this.ledgerFile, 'utf-8'));
295
+ if (raw && typeof raw === 'object') {
296
+ ledger = {
297
+ version: 1,
298
+ launches: Array.isArray(raw.launches) ? raw.launches.filter((l) => typeof l?.at === 'number') : [],
299
+ active: Array.isArray(raw.active) ? raw.active.filter((a) => typeof a?.at === 'number') : [],
300
+ pausedUntil: typeof raw.pausedUntil === 'number' ? raw.pausedUntil : undefined,
301
+ pauseReason: typeof raw.pauseReason === 'string' ? raw.pauseReason : undefined,
302
+ };
303
+ }
304
+ }
305
+ catch {
306
+ // Corrupt ledger: start fresh rather than blocking forever. The next
307
+ // write re-establishes it; worst case one over-budget launch.
308
+ }
309
+ }
310
+ ledger.launches = ledger.launches.filter((l) => now - l.at < DAY_MS);
311
+ // Drop reservations whose process died (tolerating PID reuse via the
312
+ // staleness cutoff) or that outlived any plausible worker run.
313
+ ledger.active = ledger.active.filter((a) => now - a.at < ACTIVE_STALE_MS && isProcessAlive(a.pid));
314
+ if (ledger.pausedUntil && ledger.pausedUntil <= now) {
315
+ ledger.pausedUntil = undefined;
316
+ ledger.pauseReason = undefined;
317
+ }
318
+ return ledger;
319
+ }
320
+ writeLedger(ledger) {
321
+ this.ensureDir();
322
+ assertNotSymlink(this.ledgerFile);
323
+ const tmp = `${this.ledgerFile}.tmp`;
324
+ fs.writeFileSync(tmp, JSON.stringify(ledger), { mode: 0o600 });
325
+ fs.renameSync(tmp, this.ledgerFile);
326
+ }
327
+ /**
328
+ * Invariant 10: every launch, denial, and pause emits a receipt. Only
329
+ * operational metadata is persisted — never prompts or source content.
330
+ */
331
+ appendReceipt(receipt) {
332
+ try {
333
+ this.ensureDir();
334
+ assertNotSymlink(this.receiptsFile);
335
+ fs.appendFileSync(this.receiptsFile, JSON.stringify(receipt) + '\n', { mode: 0o600 });
336
+ const st = fs.statSync(this.receiptsFile);
337
+ if (st.size > RECEIPTS_MAX_BYTES) {
338
+ const lines = fs.readFileSync(this.receiptsFile, 'utf-8').split('\n');
339
+ fs.writeFileSync(this.receiptsFile, lines.slice(-RECEIPTS_KEEP_LINES).join('\n'), { mode: 0o600 });
340
+ }
341
+ }
342
+ catch {
343
+ // Receipts are best-effort; never block a decision on them.
344
+ }
345
+ }
346
+ }
347
+ // Singleton — one budget per process, shared by all executors.
348
+ let budgetInstance = null;
349
+ export function getGlobalAiBudget() {
350
+ if (!budgetInstance) {
351
+ budgetInstance = new GlobalAiBudget();
352
+ }
353
+ return budgetInstance;
354
+ }
355
+ /** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
356
+ export function resetGlobalAiBudgetForTests() {
357
+ budgetInstance = null;
358
+ }
359
+ //# sourceMappingURL=global-ai-budget.js.map
@@ -131,6 +131,13 @@ export interface HeadlessExecutionResult {
131
131
  error?: string;
132
132
  /** Execution ID for tracking */
133
133
  executionId: string;
134
+ /**
135
+ * #2661 — true when the launch was skipped because the same job
136
+ * (repositoryId + HEAD + worker + config) succeeded within the freshness
137
+ * window, e.g. in a sibling worktree. No model call happened; consumers
138
+ * must not overwrite persisted metrics with this result.
139
+ */
140
+ dedupSkipped?: boolean;
134
141
  }
135
142
  /**
136
143
  * Pool status information
@@ -246,6 +253,13 @@ export declare class HeadlessWorkerExecutor extends EventEmitter {
246
253
  * Get number of active executions
247
254
  */
248
255
  getActiveCount(): number;
256
+ /**
257
+ * #2661 — signal a pool entry's whole process group, not just the head.
258
+ * Children are spawned `detached: true` on POSIX precisely so their MCP
259
+ * bridge grandchildren can be reaped with `kill(-pid)`; a head-only kill
260
+ * orphans them (#2098B).
261
+ */
262
+ private killEntryTree;
249
263
  /**
250
264
  * Cancel a running execution
251
265
  */
@@ -22,6 +22,9 @@ import { spawn, execSync } from 'child_process';
22
22
  import { EventEmitter } from 'events';
23
23
  import { existsSync, readFileSync, readdirSync, mkdirSync, writeFileSync } from 'fs';
24
24
  import { join } from 'path';
25
+ import { getGlobalAiBudget, isQuotaErrorText } from './global-ai-budget.js';
26
+ import { resolveGitWorkspaceIdentity } from './git-workspace-identity.js';
27
+ import { getAiJobDedupRegistry, computeAiJobKey, hashWorkerConfig } from './ai-job-dedup.js';
25
28
  // ============================================
26
29
  // Constants
27
30
  // ============================================
@@ -517,6 +520,25 @@ export class HeadlessWorkerExecutor extends EventEmitter {
517
520
  getActiveCount() {
518
521
  return this.processPool.size;
519
522
  }
523
+ /**
524
+ * #2661 — signal a pool entry's whole process group, not just the head.
525
+ * Children are spawned `detached: true` on POSIX precisely so their MCP
526
+ * bridge grandchildren can be reaped with `kill(-pid)`; a head-only kill
527
+ * orphans them (#2098B).
528
+ */
529
+ killEntryTree(proc, signal) {
530
+ if (process.platform !== 'win32' && typeof proc.pid === 'number') {
531
+ try {
532
+ process.kill(-proc.pid, signal);
533
+ return;
534
+ }
535
+ catch { /* fall through */ }
536
+ }
537
+ try {
538
+ proc.kill(signal);
539
+ }
540
+ catch { /* already dead */ }
541
+ }
520
542
  /**
521
543
  * Cancel a running execution
522
544
  */
@@ -526,7 +548,7 @@ export class HeadlessWorkerExecutor extends EventEmitter {
526
548
  return false;
527
549
  }
528
550
  clearTimeout(entry.timeout);
529
- entry.process.kill('SIGTERM');
551
+ this.killEntryTree(entry.process, 'SIGTERM');
530
552
  this.processPool.delete(executionId);
531
553
  this.emit('cancelled', { executionId });
532
554
  // Process next in queue
@@ -542,12 +564,12 @@ export class HeadlessWorkerExecutor extends EventEmitter {
542
564
  const entries = Array.from(this.processPool.entries());
543
565
  for (const [executionId, entry] of entries) {
544
566
  clearTimeout(entry.timeout);
545
- entry.process.kill('SIGTERM');
567
+ this.killEntryTree(entry.process, 'SIGTERM');
546
568
  // SIGKILL fallback after 5s to prevent orphan processes (#1395 Bug 6)
547
569
  setTimeout(() => {
548
570
  try {
549
571
  if (!entry.process.killed)
550
- entry.process.kill('SIGKILL');
572
+ this.killEntryTree(entry.process, 'SIGKILL');
551
573
  }
552
574
  catch { /* already dead */ }
553
575
  }, 5000).unref();
@@ -612,6 +634,63 @@ export class HeadlessWorkerExecutor extends EventEmitter {
612
634
  const headless = { ...baseConfig.headless, ...configOverrides };
613
635
  const startTime = Date.now();
614
636
  const executionId = `${workerType}_${startTime}_${Math.random().toString(36).slice(2, 8)}`;
637
+ // #2661 invariant 5 — cross-worktree job dedup. Worktrees of one
638
+ // repository at the same HEAD would otherwise run identical analyses
639
+ // once per worktree. jobKey = sha256(repositoryId, HEAD, worker,
640
+ // configHash); a success within the freshness window (the worker's own
641
+ // interval, floor 10 min) skips the launch entirely — no budget spend,
642
+ // no process. HEAD moves → new key → the job runs again.
643
+ const identity = resolveGitWorkspaceIdentity(this.projectRoot);
644
+ const jobKey = computeAiJobKey({
645
+ repositoryId: identity.repositoryId,
646
+ head: identity.head,
647
+ workerType,
648
+ configHash: hashWorkerConfig(headless),
649
+ });
650
+ const dedup = getAiJobDedupRegistry();
651
+ const envWindowSecs = Number.parseInt(process.env.RUFLO_AI_DEDUP_WINDOW_SECS || '', 10);
652
+ const freshnessMs = Number.isFinite(envWindowSecs) && envWindowSecs >= 0
653
+ ? envWindowSecs * 1000
654
+ : Math.max(baseConfig.intervalMs || 0, 10 * 60 * 1000);
655
+ const freshness = dedup.isFresh(jobKey, freshnessMs);
656
+ if (freshness.fresh) {
657
+ const skipped = {
658
+ success: true,
659
+ dedupSkipped: true,
660
+ output: '',
661
+ parsedOutput: undefined,
662
+ durationMs: 0,
663
+ model: 'none',
664
+ sandboxMode: headless.sandbox,
665
+ workerType,
666
+ timestamp: new Date(),
667
+ executionId,
668
+ };
669
+ this.logExecution(executionId, 'result', `dedup-skip: job ${jobKey.slice(0, 12)} succeeded ${Math.round((Date.now() - (freshness.lastRunAt ?? Date.now())) / 1000)}s ago (repo ${identity.repositoryId.slice(0, 12)}, head ${identity.head.slice(0, 12) || 'n/a'})`);
670
+ this.emit('dedup:skipped', { executionId, workerType, jobKey, lastRunAt: freshness.lastRunAt });
671
+ this.processQueue();
672
+ return skipped;
673
+ }
674
+ // #2661 — every autonomous launch must reserve a slot in the USER-GLOBAL
675
+ // AI budget before any process is created. This is the hard invariant
676
+ // that bounds aggregate launches across all worktree daemons: per-daemon
677
+ // maxConcurrent limits multiply with worktree count, the global budget
678
+ // does not. Denials return an error result (with a receipted reason)
679
+ // instead of queueing, so denied work never piles up into a retry storm.
680
+ const budget = getGlobalAiBudget();
681
+ const model = headless.model || 'sonnet';
682
+ const permit = await budget.reserve({ workerType, model, workspace: this.projectRoot });
683
+ if (!permit.allowed) {
684
+ const denied = this.createErrorResult(workerType, `Denied by global AI budget: ${permit.reason}`);
685
+ denied.executionId = executionId;
686
+ this.logExecution(executionId, 'error', `budget-denied: ${permit.reason}`);
687
+ // NOTE: deliberately no `emit('error', ...)` here — Node treats
688
+ // unlistened 'error' events as throws, and callers consume the
689
+ // returned error result; `budget:denied` is the observable signal.
690
+ this.emit('budget:denied', { executionId, workerType, reason: permit.reason });
691
+ this.processQueue();
692
+ return denied;
693
+ }
615
694
  this.emit('start', { executionId, workerType, config: headless });
616
695
  try {
617
696
  // Build context from file patterns
@@ -651,6 +730,23 @@ export class HeadlessWorkerExecutor extends EventEmitter {
651
730
  };
652
731
  // Log result
653
732
  this.logExecution(executionId, 'result', JSON.stringify(executionResult, null, 2));
733
+ // #2661 invariant 5 — record the success so sibling worktrees at the
734
+ // same HEAD skip this job for the rest of the freshness window.
735
+ if (result.success) {
736
+ dedup.recordSuccess(jobKey, {
737
+ workerType,
738
+ repositoryId: identity.repositoryId,
739
+ workspace: this.projectRoot,
740
+ });
741
+ }
742
+ // #2661 — a quota/429/usage-limit failure opens the user-global
743
+ // circuit breaker so EVERY daemon stops launching for the cooldown
744
+ // window instead of retrying into an exhausted quota. Only inspected
745
+ // on failure: successful analysis output may legitimately discuss
746
+ // rate limiting in the user's own code.
747
+ if (!result.success && isQuotaErrorText(result.error)) {
748
+ await budget.recordQuotaError(`${workerType}: ${(result.error ?? '').slice(0, 200)}`);
749
+ }
654
750
  this.emit('complete', executionResult);
655
751
  return executionResult;
656
752
  }
@@ -660,10 +756,15 @@ export class HeadlessWorkerExecutor extends EventEmitter {
660
756
  executionResult.executionId = executionId;
661
757
  executionResult.durationMs = Date.now() - startTime;
662
758
  this.logExecution(executionId, 'error', errorMessage);
759
+ if (isQuotaErrorText(errorMessage)) {
760
+ await budget.recordQuotaError(`${workerType}: ${errorMessage.slice(0, 200)}`);
761
+ }
663
762
  this.emit('error', executionResult);
664
763
  return executionResult;
665
764
  }
666
765
  finally {
766
+ // #2661 — free the global concurrency slot (launch counts persist).
767
+ await budget.release(permit.permitId);
667
768
  // Process next in queue
668
769
  this.processQueue();
669
770
  }
@@ -58,6 +58,7 @@ export interface DaemonConfig {
58
58
  };
59
59
  ttlMs: number;
60
60
  idleShutdownMs: number;
61
+ aiWorkersEnabled: boolean;
61
62
  workers: WorkerConfig[];
62
63
  }
63
64
  /**
@@ -237,6 +238,19 @@ export declare class WorkerDaemon extends EventEmitter {
237
238
  * on its own. A no-op when both limits are disabled (0).
238
239
  */
239
240
  private startLifecycleMonitor;
241
+ /**
242
+ * Decide whether the daemon should self-shutdown, and why. Extracted from
243
+ * the lifecycle timer so it is testable without racing a 60s interval or
244
+ * calling process.exit().
245
+ *
246
+ * #2661 (invariant 6, containment form): a removed worktree makes its
247
+ * daemon ineligible within one check interval — the daemon detects that
248
+ * its workspace directory is gone and shuts down instead of continuing to
249
+ * schedule jobs against a deleted tree. The full lease architecture
250
+ * (supervisor-dispatched jobs, heartbeats) is follow-up work; this stops
251
+ * the leak where recreated/removed worktrees leave schedulers behind.
252
+ */
253
+ private lifecycleShutdownReason;
240
254
  /**
241
255
  * Most recent worker start/finish time across all workers (epoch ms), or
242
256
  * null if no worker has ever started. Used for idle-shutdown detection.