@claude-flow/cli 3.26.1 → 3.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/helpers/helpers.manifest.json +2 -2
- package/catalog-manifest.json +2 -2
- package/dist/src/commands/daemon.js +163 -3
- package/dist/src/init/helpers-generator.js +31 -0
- package/dist/src/services/ai-job-dedup.d.ts +61 -0
- package/dist/src/services/ai-job-dedup.js +136 -0
- package/dist/src/services/git-workspace-identity.d.ts +42 -0
- package/dist/src/services/git-workspace-identity.js +99 -0
- package/dist/src/services/global-ai-budget.d.ts +110 -0
- package/dist/src/services/global-ai-budget.js +359 -0
- package/dist/src/services/headless-worker-executor.d.ts +14 -0
- package/dist/src/services/headless-worker-executor.js +104 -3
- package/dist/src/services/worker-daemon.d.ts +14 -0
- package/dist/src/services/worker-daemon.js +81 -17
- package/package.json +1 -1
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #2661 — Global AI launch budget (emergency cost fuse).
|
|
3
|
+
*
|
|
4
|
+
* Every autonomous `claude --print` launch across ALL ruflo daemons in ALL
|
|
5
|
+
* worktrees/workspaces owned by the current user must pass through this
|
|
6
|
+
* user-global budget before a process is created. Without it, N worktree
|
|
7
|
+
* daemons each schedule their own AI workers and aggregate launch volume
|
|
8
|
+
* scales linearly with worktree count — enough to exhaust a user's Claude
|
|
9
|
+
* hourly quota silently (13 launches/hour/daemon under the legacy schedule).
|
|
10
|
+
*
|
|
11
|
+
* The ledger lives under the user's home directory (NOT the workspace) so
|
|
12
|
+
* daemons started from different worktrees of the same repository — or from
|
|
13
|
+
* unrelated repositories — all share one budget:
|
|
14
|
+
*
|
|
15
|
+
* ~/.claude-flow/ai-budget.json launch ledger + circuit breaker
|
|
16
|
+
* ~/.claude-flow/ai-budget.lock O_EXCL mutation lock
|
|
17
|
+
* ~/.claude-flow/ai-budget-receipts.jsonl launch/deny/pause receipts
|
|
18
|
+
*
|
|
19
|
+
* Files are owner-only (0700 dir / 0600 files) and symlinks are rejected
|
|
20
|
+
* (invariant 9 of #2661). All checks happen BEFORE process creation and the
|
|
21
|
+
* ledger mutation is atomic under the lock, so two daemons racing for the
|
|
22
|
+
* last hourly slot cannot both win.
|
|
23
|
+
*
|
|
24
|
+
* Default limits (issue #2661 containment):
|
|
25
|
+
* maxConcurrentGlobal 1 at most one autonomous claude child, user-wide
|
|
26
|
+
* maxLaunchesPerHour 2
|
|
27
|
+
* maxLaunchesPerDay 12
|
|
28
|
+
* pauseOnQuotaErrorMinutes 60 circuit breaker on 429/quota responses
|
|
29
|
+
*/
|
|
30
|
+
export interface AiBudgetLimits {
|
|
31
|
+
maxConcurrentGlobal: number;
|
|
32
|
+
maxLaunchesPerHour: number;
|
|
33
|
+
maxLaunchesPerDay: number;
|
|
34
|
+
pauseOnQuotaErrorMinutes: number;
|
|
35
|
+
}
|
|
36
|
+
export declare const DEFAULT_AI_BUDGET_LIMITS: AiBudgetLimits;
|
|
37
|
+
export interface AiBudgetRequest {
|
|
38
|
+
workerType: string;
|
|
39
|
+
model: string;
|
|
40
|
+
/** Worktree/workspace root requesting the launch (recorded for receipts only). */
|
|
41
|
+
workspace: string;
|
|
42
|
+
}
|
|
43
|
+
export interface AiBudgetPermit {
|
|
44
|
+
allowed: boolean;
|
|
45
|
+
permitId?: string;
|
|
46
|
+
reason?: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Heuristic match for Anthropic quota / rate-limit failures. Only ever
|
|
50
|
+
* applied to ERROR output of a FAILED launch (never to successful analysis
|
|
51
|
+
* output, which may legitimately discuss "rate limiting" in the user's code).
|
|
52
|
+
*/
|
|
53
|
+
export declare function isQuotaErrorText(text: string | undefined): boolean;
|
|
54
|
+
export declare class GlobalAiBudget {
|
|
55
|
+
private readonly dir;
|
|
56
|
+
private readonly ledgerFile;
|
|
57
|
+
private readonly lockFile;
|
|
58
|
+
private readonly receiptsFile;
|
|
59
|
+
private readonly limits;
|
|
60
|
+
constructor(options?: {
|
|
61
|
+
baseDir?: string;
|
|
62
|
+
limits?: Partial<AiBudgetLimits>;
|
|
63
|
+
});
|
|
64
|
+
getLimits(): AiBudgetLimits;
|
|
65
|
+
/**
|
|
66
|
+
* Atomically reserve one launch slot. Denials carry a machine-readable
|
|
67
|
+
* reason and are receipted. The reservation is counted as a launch
|
|
68
|
+
* immediately (the hourly/daily invariant is on launches, not completions);
|
|
69
|
+
* `release()` only frees the concurrency slot.
|
|
70
|
+
*
|
|
71
|
+
* Fails CLOSED: if the ledger cannot be read or locked, the launch is
|
|
72
|
+
* denied — an unaccountable launch is exactly what this fuse exists to
|
|
73
|
+
* prevent. `RUFLO_AI_BUDGET_DISABLE=1` is the explicit escape hatch.
|
|
74
|
+
*/
|
|
75
|
+
reserve(req: AiBudgetRequest): Promise<AiBudgetPermit>;
|
|
76
|
+
/** Free the concurrency slot held by a permit. Best-effort. */
|
|
77
|
+
release(permitId: string | undefined): Promise<void>;
|
|
78
|
+
/**
|
|
79
|
+
* Open the user-global circuit breaker: a quota/429 response from ANY
|
|
80
|
+
* daemon pauses ALL autonomous Claude launches for the cooldown window.
|
|
81
|
+
*/
|
|
82
|
+
recordQuotaError(detail: string): Promise<void>;
|
|
83
|
+
/** Snapshot for `daemon status` / diagnostics. */
|
|
84
|
+
getUsage(): {
|
|
85
|
+
lastHour: number;
|
|
86
|
+
lastDay: number;
|
|
87
|
+
active: number;
|
|
88
|
+
pausedUntil?: number;
|
|
89
|
+
pauseReason?: string;
|
|
90
|
+
/** #2661 — 24h launch counts per worktree/workspace, most active first. */
|
|
91
|
+
byWorkspace: Array<{
|
|
92
|
+
workspace: string;
|
|
93
|
+
launches: number;
|
|
94
|
+
}>;
|
|
95
|
+
};
|
|
96
|
+
private ensureDir;
|
|
97
|
+
private acquireLock;
|
|
98
|
+
/** Read + prune the ledger. Caller must hold the lock for read-modify-write. */
|
|
99
|
+
private readLedger;
|
|
100
|
+
private writeLedger;
|
|
101
|
+
/**
|
|
102
|
+
* Invariant 10: every launch, denial, and pause emits a receipt. Only
|
|
103
|
+
* operational metadata is persisted — never prompts or source content.
|
|
104
|
+
*/
|
|
105
|
+
private appendReceipt;
|
|
106
|
+
}
|
|
107
|
+
export declare function getGlobalAiBudget(): GlobalAiBudget;
|
|
108
|
+
/** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
|
|
109
|
+
export declare function resetGlobalAiBudgetForTests(): void;
|
|
110
|
+
//# sourceMappingURL=global-ai-budget.d.ts.map
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #2661 — Global AI launch budget (emergency cost fuse).
|
|
3
|
+
*
|
|
4
|
+
* Every autonomous `claude --print` launch across ALL ruflo daemons in ALL
|
|
5
|
+
* worktrees/workspaces owned by the current user must pass through this
|
|
6
|
+
* user-global budget before a process is created. Without it, N worktree
|
|
7
|
+
* daemons each schedule their own AI workers and aggregate launch volume
|
|
8
|
+
* scales linearly with worktree count — enough to exhaust a user's Claude
|
|
9
|
+
* hourly quota silently (13 launches/hour/daemon under the legacy schedule).
|
|
10
|
+
*
|
|
11
|
+
* The ledger lives under the user's home directory (NOT the workspace) so
|
|
12
|
+
* daemons started from different worktrees of the same repository — or from
|
|
13
|
+
* unrelated repositories — all share one budget:
|
|
14
|
+
*
|
|
15
|
+
* ~/.claude-flow/ai-budget.json launch ledger + circuit breaker
|
|
16
|
+
* ~/.claude-flow/ai-budget.lock O_EXCL mutation lock
|
|
17
|
+
* ~/.claude-flow/ai-budget-receipts.jsonl launch/deny/pause receipts
|
|
18
|
+
*
|
|
19
|
+
* Files are owner-only (0700 dir / 0600 files) and symlinks are rejected
|
|
20
|
+
* (invariant 9 of #2661). All checks happen BEFORE process creation and the
|
|
21
|
+
* ledger mutation is atomic under the lock, so two daemons racing for the
|
|
22
|
+
* last hourly slot cannot both win.
|
|
23
|
+
*
|
|
24
|
+
* Default limits (issue #2661 containment):
|
|
25
|
+
* maxConcurrentGlobal 1 at most one autonomous claude child, user-wide
|
|
26
|
+
* maxLaunchesPerHour 2
|
|
27
|
+
* maxLaunchesPerDay 12
|
|
28
|
+
* pauseOnQuotaErrorMinutes 60 circuit breaker on 429/quota responses
|
|
29
|
+
*/
|
|
30
|
+
import * as fs from 'fs';
|
|
31
|
+
import { join } from 'path';
|
|
32
|
+
import { homedir } from 'os';
|
|
33
|
+
export const DEFAULT_AI_BUDGET_LIMITS = {
|
|
34
|
+
maxConcurrentGlobal: 1,
|
|
35
|
+
maxLaunchesPerHour: 2,
|
|
36
|
+
maxLaunchesPerDay: 12,
|
|
37
|
+
pauseOnQuotaErrorMinutes: 60,
|
|
38
|
+
};
|
|
39
|
+
const HOUR_MS = 60 * 60 * 1000;
|
|
40
|
+
const DAY_MS = 24 * HOUR_MS;
|
|
41
|
+
// Active reservations older than this are treated as abandoned (crashed
|
|
42
|
+
// daemon). Must exceed the daemon's 16-min worker timeout with margin.
|
|
43
|
+
const ACTIVE_STALE_MS = 30 * 60 * 1000;
|
|
44
|
+
// A mutation lock older than this belongs to a crashed process — take it over.
|
|
45
|
+
const LOCK_STALE_MS = 10_000;
|
|
46
|
+
const RECEIPTS_MAX_BYTES = 512 * 1024;
|
|
47
|
+
const RECEIPTS_KEEP_LINES = 200;
|
|
48
|
+
/**
|
|
49
|
+
* Heuristic match for Anthropic quota / rate-limit failures. Only ever
|
|
50
|
+
* applied to ERROR output of a FAILED launch (never to successful analysis
|
|
51
|
+
* output, which may legitimately discuss "rate limiting" in the user's code).
|
|
52
|
+
*/
|
|
53
|
+
export function isQuotaErrorText(text) {
|
|
54
|
+
if (!text)
|
|
55
|
+
return false;
|
|
56
|
+
return /\b429\b|rate[\s_-]?limit|usage[\s_-]?limit|quota|too many requests|overloaded_error/i.test(text);
|
|
57
|
+
}
|
|
58
|
+
function envPositiveInt(name) {
|
|
59
|
+
const raw = process.env[name];
|
|
60
|
+
if (raw === undefined || raw.trim() === '')
|
|
61
|
+
return undefined;
|
|
62
|
+
const n = Number.parseInt(raw, 10);
|
|
63
|
+
return Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
64
|
+
}
|
|
65
|
+
function isProcessAlive(pid) {
|
|
66
|
+
try {
|
|
67
|
+
process.kill(pid, 0);
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/** Invariant 9: registry files must never be symlinks. */
|
|
75
|
+
function assertNotSymlink(path) {
|
|
76
|
+
try {
|
|
77
|
+
const st = fs.lstatSync(path);
|
|
78
|
+
if (st.isSymbolicLink()) {
|
|
79
|
+
throw new Error(`AI budget file is a symlink (refusing): ${path}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
if (e.code === 'ENOENT')
|
|
84
|
+
return;
|
|
85
|
+
throw e;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function delay(ms) {
|
|
89
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
90
|
+
}
|
|
91
|
+
export class GlobalAiBudget {
|
|
92
|
+
dir;
|
|
93
|
+
ledgerFile;
|
|
94
|
+
lockFile;
|
|
95
|
+
receiptsFile;
|
|
96
|
+
limits;
|
|
97
|
+
constructor(options) {
|
|
98
|
+
this.dir = options?.baseDir
|
|
99
|
+
?? process.env.RUFLO_AI_BUDGET_DIR
|
|
100
|
+
?? join(homedir(), '.claude-flow');
|
|
101
|
+
this.ledgerFile = join(this.dir, 'ai-budget.json');
|
|
102
|
+
this.lockFile = join(this.dir, 'ai-budget.lock');
|
|
103
|
+
this.receiptsFile = join(this.dir, 'ai-budget-receipts.jsonl');
|
|
104
|
+
this.limits = {
|
|
105
|
+
maxConcurrentGlobal: options?.limits?.maxConcurrentGlobal
|
|
106
|
+
?? envPositiveInt('RUFLO_AI_MAX_CONCURRENT')
|
|
107
|
+
?? DEFAULT_AI_BUDGET_LIMITS.maxConcurrentGlobal,
|
|
108
|
+
maxLaunchesPerHour: options?.limits?.maxLaunchesPerHour
|
|
109
|
+
?? envPositiveInt('RUFLO_AI_MAX_PER_HOUR')
|
|
110
|
+
?? DEFAULT_AI_BUDGET_LIMITS.maxLaunchesPerHour,
|
|
111
|
+
maxLaunchesPerDay: options?.limits?.maxLaunchesPerDay
|
|
112
|
+
?? envPositiveInt('RUFLO_AI_MAX_PER_DAY')
|
|
113
|
+
?? DEFAULT_AI_BUDGET_LIMITS.maxLaunchesPerDay,
|
|
114
|
+
pauseOnQuotaErrorMinutes: options?.limits?.pauseOnQuotaErrorMinutes
|
|
115
|
+
?? envPositiveInt('RUFLO_AI_QUOTA_PAUSE_MINUTES')
|
|
116
|
+
?? DEFAULT_AI_BUDGET_LIMITS.pauseOnQuotaErrorMinutes,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
getLimits() {
|
|
120
|
+
return { ...this.limits };
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Atomically reserve one launch slot. Denials carry a machine-readable
|
|
124
|
+
* reason and are receipted. The reservation is counted as a launch
|
|
125
|
+
* immediately (the hourly/daily invariant is on launches, not completions);
|
|
126
|
+
* `release()` only frees the concurrency slot.
|
|
127
|
+
*
|
|
128
|
+
* Fails CLOSED: if the ledger cannot be read or locked, the launch is
|
|
129
|
+
* denied — an unaccountable launch is exactly what this fuse exists to
|
|
130
|
+
* prevent. `RUFLO_AI_BUDGET_DISABLE=1` is the explicit escape hatch.
|
|
131
|
+
*/
|
|
132
|
+
async reserve(req) {
|
|
133
|
+
if (process.env.RUFLO_AI_BUDGET_DISABLE === '1') {
|
|
134
|
+
return { allowed: true, permitId: `bypass_${Date.now()}_${process.pid}` };
|
|
135
|
+
}
|
|
136
|
+
let unlock = null;
|
|
137
|
+
try {
|
|
138
|
+
unlock = await this.acquireLock();
|
|
139
|
+
const now = Date.now();
|
|
140
|
+
const ledger = this.readLedger(now);
|
|
141
|
+
let reason = null;
|
|
142
|
+
if (ledger.pausedUntil && ledger.pausedUntil > now) {
|
|
143
|
+
reason = `circuit-open until ${new Date(ledger.pausedUntil).toISOString()} (${ledger.pauseReason ?? 'quota error'})`;
|
|
144
|
+
}
|
|
145
|
+
else if (ledger.active.length >= this.limits.maxConcurrentGlobal) {
|
|
146
|
+
reason = `global-concurrency (${ledger.active.length}/${this.limits.maxConcurrentGlobal} active)`;
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
const lastHour = ledger.launches.filter((l) => now - l.at < HOUR_MS).length;
|
|
150
|
+
const lastDay = ledger.launches.length; // already pruned to 24h
|
|
151
|
+
if (lastHour >= this.limits.maxLaunchesPerHour) {
|
|
152
|
+
reason = `hourly-budget (${lastHour}/${this.limits.maxLaunchesPerHour} in last hour)`;
|
|
153
|
+
}
|
|
154
|
+
else if (lastDay >= this.limits.maxLaunchesPerDay) {
|
|
155
|
+
reason = `daily-budget (${lastDay}/${this.limits.maxLaunchesPerDay} in last 24h)`;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if (reason) {
|
|
159
|
+
this.appendReceipt({ event: 'deny', at: now, reason, ...req });
|
|
160
|
+
return { allowed: false, reason };
|
|
161
|
+
}
|
|
162
|
+
const permitId = `permit_${now}_${process.pid}_${Math.random().toString(36).slice(2, 8)}`;
|
|
163
|
+
ledger.launches.push({ at: now, pid: process.pid, workerType: req.workerType, model: req.model, workspace: req.workspace });
|
|
164
|
+
ledger.active.push({ permitId, at: now, pid: process.pid, workerType: req.workerType });
|
|
165
|
+
this.writeLedger(ledger);
|
|
166
|
+
this.appendReceipt({ event: 'launch', at: now, permitId, ...req });
|
|
167
|
+
return { allowed: true, permitId };
|
|
168
|
+
}
|
|
169
|
+
catch (e) {
|
|
170
|
+
// Fail closed — see docstring.
|
|
171
|
+
const reason = `budget-ledger-error: ${e instanceof Error ? e.message : String(e)}`;
|
|
172
|
+
try {
|
|
173
|
+
this.appendReceipt({ event: 'deny', at: Date.now(), reason, ...req });
|
|
174
|
+
}
|
|
175
|
+
catch { /* best-effort */ }
|
|
176
|
+
return { allowed: false, reason };
|
|
177
|
+
}
|
|
178
|
+
finally {
|
|
179
|
+
unlock?.();
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/** Free the concurrency slot held by a permit. Best-effort. */
|
|
183
|
+
async release(permitId) {
|
|
184
|
+
if (!permitId || permitId.startsWith('bypass_'))
|
|
185
|
+
return;
|
|
186
|
+
let unlock = null;
|
|
187
|
+
try {
|
|
188
|
+
unlock = await this.acquireLock();
|
|
189
|
+
const ledger = this.readLedger(Date.now());
|
|
190
|
+
const before = ledger.active.length;
|
|
191
|
+
ledger.active = ledger.active.filter((a) => a.permitId !== permitId);
|
|
192
|
+
if (ledger.active.length !== before)
|
|
193
|
+
this.writeLedger(ledger);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
// Abandoned reservations expire via ACTIVE_STALE_MS pruning.
|
|
197
|
+
}
|
|
198
|
+
finally {
|
|
199
|
+
unlock?.();
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Open the user-global circuit breaker: a quota/429 response from ANY
|
|
204
|
+
* daemon pauses ALL autonomous Claude launches for the cooldown window.
|
|
205
|
+
*/
|
|
206
|
+
async recordQuotaError(detail) {
|
|
207
|
+
let unlock = null;
|
|
208
|
+
try {
|
|
209
|
+
unlock = await this.acquireLock();
|
|
210
|
+
const now = Date.now();
|
|
211
|
+
const ledger = this.readLedger(now);
|
|
212
|
+
ledger.pausedUntil = now + this.limits.pauseOnQuotaErrorMinutes * 60 * 1000;
|
|
213
|
+
ledger.pauseReason = detail.slice(0, 200);
|
|
214
|
+
this.writeLedger(ledger);
|
|
215
|
+
this.appendReceipt({ event: 'quota-pause', at: now, until: ledger.pausedUntil, detail: ledger.pauseReason });
|
|
216
|
+
}
|
|
217
|
+
catch {
|
|
218
|
+
// best-effort — the hourly budget still bounds retries
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
unlock?.();
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/** Snapshot for `daemon status` / diagnostics. */
|
|
225
|
+
getUsage() {
|
|
226
|
+
try {
|
|
227
|
+
const now = Date.now();
|
|
228
|
+
const ledger = this.readLedger(now);
|
|
229
|
+
const byWs = new Map();
|
|
230
|
+
for (const l of ledger.launches) {
|
|
231
|
+
byWs.set(l.workspace, (byWs.get(l.workspace) ?? 0) + 1);
|
|
232
|
+
}
|
|
233
|
+
return {
|
|
234
|
+
lastHour: ledger.launches.filter((l) => now - l.at < HOUR_MS).length,
|
|
235
|
+
lastDay: ledger.launches.length,
|
|
236
|
+
active: ledger.active.length,
|
|
237
|
+
pausedUntil: ledger.pausedUntil && ledger.pausedUntil > now ? ledger.pausedUntil : undefined,
|
|
238
|
+
pauseReason: ledger.pausedUntil && ledger.pausedUntil > now ? ledger.pauseReason : undefined,
|
|
239
|
+
byWorkspace: Array.from(byWs.entries())
|
|
240
|
+
.map(([workspace, launches]) => ({ workspace, launches }))
|
|
241
|
+
.sort((a, b) => b.launches - a.launches),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
return { lastHour: 0, lastDay: 0, active: 0, byWorkspace: [] };
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
// -- internals ----------------------------------------------------------
|
|
249
|
+
ensureDir() {
|
|
250
|
+
if (!fs.existsSync(this.dir)) {
|
|
251
|
+
fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
async acquireLock() {
|
|
255
|
+
this.ensureDir();
|
|
256
|
+
const deadline = Date.now() + 2000;
|
|
257
|
+
for (;;) {
|
|
258
|
+
try {
|
|
259
|
+
const fd = fs.openSync(this.lockFile, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, 0o600);
|
|
260
|
+
fs.writeSync(fd, String(process.pid));
|
|
261
|
+
fs.closeSync(fd);
|
|
262
|
+
return () => {
|
|
263
|
+
try {
|
|
264
|
+
fs.unlinkSync(this.lockFile);
|
|
265
|
+
}
|
|
266
|
+
catch { /* already gone */ }
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
catch (e) {
|
|
270
|
+
if (e.code !== 'EEXIST')
|
|
271
|
+
throw e;
|
|
272
|
+
// Stale lock from a crashed process — take over.
|
|
273
|
+
try {
|
|
274
|
+
const st = fs.lstatSync(this.lockFile);
|
|
275
|
+
if (Date.now() - st.mtimeMs > LOCK_STALE_MS) {
|
|
276
|
+
fs.unlinkSync(this.lockFile);
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
catch { /* raced — retry */ }
|
|
281
|
+
if (Date.now() > deadline) {
|
|
282
|
+
throw new Error('timed out acquiring ai-budget lock');
|
|
283
|
+
}
|
|
284
|
+
await delay(25);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
/** Read + prune the ledger. Caller must hold the lock for read-modify-write. */
|
|
289
|
+
readLedger(now) {
|
|
290
|
+
assertNotSymlink(this.ledgerFile);
|
|
291
|
+
let ledger = { version: 1, launches: [], active: [] };
|
|
292
|
+
if (fs.existsSync(this.ledgerFile)) {
|
|
293
|
+
try {
|
|
294
|
+
const raw = JSON.parse(fs.readFileSync(this.ledgerFile, 'utf-8'));
|
|
295
|
+
if (raw && typeof raw === 'object') {
|
|
296
|
+
ledger = {
|
|
297
|
+
version: 1,
|
|
298
|
+
launches: Array.isArray(raw.launches) ? raw.launches.filter((l) => typeof l?.at === 'number') : [],
|
|
299
|
+
active: Array.isArray(raw.active) ? raw.active.filter((a) => typeof a?.at === 'number') : [],
|
|
300
|
+
pausedUntil: typeof raw.pausedUntil === 'number' ? raw.pausedUntil : undefined,
|
|
301
|
+
pauseReason: typeof raw.pauseReason === 'string' ? raw.pauseReason : undefined,
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
catch {
|
|
306
|
+
// Corrupt ledger: start fresh rather than blocking forever. The next
|
|
307
|
+
// write re-establishes it; worst case one over-budget launch.
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
ledger.launches = ledger.launches.filter((l) => now - l.at < DAY_MS);
|
|
311
|
+
// Drop reservations whose process died (tolerating PID reuse via the
|
|
312
|
+
// staleness cutoff) or that outlived any plausible worker run.
|
|
313
|
+
ledger.active = ledger.active.filter((a) => now - a.at < ACTIVE_STALE_MS && isProcessAlive(a.pid));
|
|
314
|
+
if (ledger.pausedUntil && ledger.pausedUntil <= now) {
|
|
315
|
+
ledger.pausedUntil = undefined;
|
|
316
|
+
ledger.pauseReason = undefined;
|
|
317
|
+
}
|
|
318
|
+
return ledger;
|
|
319
|
+
}
|
|
320
|
+
writeLedger(ledger) {
|
|
321
|
+
this.ensureDir();
|
|
322
|
+
assertNotSymlink(this.ledgerFile);
|
|
323
|
+
const tmp = `${this.ledgerFile}.tmp`;
|
|
324
|
+
fs.writeFileSync(tmp, JSON.stringify(ledger), { mode: 0o600 });
|
|
325
|
+
fs.renameSync(tmp, this.ledgerFile);
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Invariant 10: every launch, denial, and pause emits a receipt. Only
|
|
329
|
+
* operational metadata is persisted — never prompts or source content.
|
|
330
|
+
*/
|
|
331
|
+
appendReceipt(receipt) {
|
|
332
|
+
try {
|
|
333
|
+
this.ensureDir();
|
|
334
|
+
assertNotSymlink(this.receiptsFile);
|
|
335
|
+
fs.appendFileSync(this.receiptsFile, JSON.stringify(receipt) + '\n', { mode: 0o600 });
|
|
336
|
+
const st = fs.statSync(this.receiptsFile);
|
|
337
|
+
if (st.size > RECEIPTS_MAX_BYTES) {
|
|
338
|
+
const lines = fs.readFileSync(this.receiptsFile, 'utf-8').split('\n');
|
|
339
|
+
fs.writeFileSync(this.receiptsFile, lines.slice(-RECEIPTS_KEEP_LINES).join('\n'), { mode: 0o600 });
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
catch {
|
|
343
|
+
// Receipts are best-effort; never block a decision on them.
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
// Singleton — one budget per process, shared by all executors.
|
|
348
|
+
let budgetInstance = null;
|
|
349
|
+
export function getGlobalAiBudget() {
|
|
350
|
+
if (!budgetInstance) {
|
|
351
|
+
budgetInstance = new GlobalAiBudget();
|
|
352
|
+
}
|
|
353
|
+
return budgetInstance;
|
|
354
|
+
}
|
|
355
|
+
/** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
|
|
356
|
+
export function resetGlobalAiBudgetForTests() {
|
|
357
|
+
budgetInstance = null;
|
|
358
|
+
}
|
|
359
|
+
//# sourceMappingURL=global-ai-budget.js.map
|
|
@@ -131,6 +131,13 @@ export interface HeadlessExecutionResult {
|
|
|
131
131
|
error?: string;
|
|
132
132
|
/** Execution ID for tracking */
|
|
133
133
|
executionId: string;
|
|
134
|
+
/**
|
|
135
|
+
* #2661 — true when the launch was skipped because the same job
|
|
136
|
+
* (repositoryId + HEAD + worker + config) succeeded within the freshness
|
|
137
|
+
* window, e.g. in a sibling worktree. No model call happened; consumers
|
|
138
|
+
* must not overwrite persisted metrics with this result.
|
|
139
|
+
*/
|
|
140
|
+
dedupSkipped?: boolean;
|
|
134
141
|
}
|
|
135
142
|
/**
|
|
136
143
|
* Pool status information
|
|
@@ -246,6 +253,13 @@ export declare class HeadlessWorkerExecutor extends EventEmitter {
|
|
|
246
253
|
* Get number of active executions
|
|
247
254
|
*/
|
|
248
255
|
getActiveCount(): number;
|
|
256
|
+
/**
|
|
257
|
+
* #2661 — signal a pool entry's whole process group, not just the head.
|
|
258
|
+
* Children are spawned `detached: true` on POSIX precisely so their MCP
|
|
259
|
+
* bridge grandchildren can be reaped with `kill(-pid)`; a head-only kill
|
|
260
|
+
* orphans them (#2098B).
|
|
261
|
+
*/
|
|
262
|
+
private killEntryTree;
|
|
249
263
|
/**
|
|
250
264
|
* Cancel a running execution
|
|
251
265
|
*/
|
|
@@ -22,6 +22,9 @@ import { spawn, execSync } from 'child_process';
|
|
|
22
22
|
import { EventEmitter } from 'events';
|
|
23
23
|
import { existsSync, readFileSync, readdirSync, mkdirSync, writeFileSync } from 'fs';
|
|
24
24
|
import { join } from 'path';
|
|
25
|
+
import { getGlobalAiBudget, isQuotaErrorText } from './global-ai-budget.js';
|
|
26
|
+
import { resolveGitWorkspaceIdentity } from './git-workspace-identity.js';
|
|
27
|
+
import { getAiJobDedupRegistry, computeAiJobKey, hashWorkerConfig } from './ai-job-dedup.js';
|
|
25
28
|
// ============================================
|
|
26
29
|
// Constants
|
|
27
30
|
// ============================================
|
|
@@ -517,6 +520,25 @@ export class HeadlessWorkerExecutor extends EventEmitter {
|
|
|
517
520
|
getActiveCount() {
|
|
518
521
|
return this.processPool.size;
|
|
519
522
|
}
|
|
523
|
+
/**
|
|
524
|
+
* #2661 — signal a pool entry's whole process group, not just the head.
|
|
525
|
+
* Children are spawned `detached: true` on POSIX precisely so their MCP
|
|
526
|
+
* bridge grandchildren can be reaped with `kill(-pid)`; a head-only kill
|
|
527
|
+
* orphans them (#2098B).
|
|
528
|
+
*/
|
|
529
|
+
killEntryTree(proc, signal) {
|
|
530
|
+
if (process.platform !== 'win32' && typeof proc.pid === 'number') {
|
|
531
|
+
try {
|
|
532
|
+
process.kill(-proc.pid, signal);
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
catch { /* fall through */ }
|
|
536
|
+
}
|
|
537
|
+
try {
|
|
538
|
+
proc.kill(signal);
|
|
539
|
+
}
|
|
540
|
+
catch { /* already dead */ }
|
|
541
|
+
}
|
|
520
542
|
/**
|
|
521
543
|
* Cancel a running execution
|
|
522
544
|
*/
|
|
@@ -526,7 +548,7 @@ export class HeadlessWorkerExecutor extends EventEmitter {
|
|
|
526
548
|
return false;
|
|
527
549
|
}
|
|
528
550
|
clearTimeout(entry.timeout);
|
|
529
|
-
entry.process
|
|
551
|
+
this.killEntryTree(entry.process, 'SIGTERM');
|
|
530
552
|
this.processPool.delete(executionId);
|
|
531
553
|
this.emit('cancelled', { executionId });
|
|
532
554
|
// Process next in queue
|
|
@@ -542,12 +564,12 @@ export class HeadlessWorkerExecutor extends EventEmitter {
|
|
|
542
564
|
const entries = Array.from(this.processPool.entries());
|
|
543
565
|
for (const [executionId, entry] of entries) {
|
|
544
566
|
clearTimeout(entry.timeout);
|
|
545
|
-
entry.process
|
|
567
|
+
this.killEntryTree(entry.process, 'SIGTERM');
|
|
546
568
|
// SIGKILL fallback after 5s to prevent orphan processes (#1395 Bug 6)
|
|
547
569
|
setTimeout(() => {
|
|
548
570
|
try {
|
|
549
571
|
if (!entry.process.killed)
|
|
550
|
-
entry.process
|
|
572
|
+
this.killEntryTree(entry.process, 'SIGKILL');
|
|
551
573
|
}
|
|
552
574
|
catch { /* already dead */ }
|
|
553
575
|
}, 5000).unref();
|
|
@@ -612,6 +634,63 @@ export class HeadlessWorkerExecutor extends EventEmitter {
|
|
|
612
634
|
const headless = { ...baseConfig.headless, ...configOverrides };
|
|
613
635
|
const startTime = Date.now();
|
|
614
636
|
const executionId = `${workerType}_${startTime}_${Math.random().toString(36).slice(2, 8)}`;
|
|
637
|
+
// #2661 invariant 5 — cross-worktree job dedup. Worktrees of one
|
|
638
|
+
// repository at the same HEAD would otherwise run identical analyses
|
|
639
|
+
// once per worktree. jobKey = sha256(repositoryId, HEAD, worker,
|
|
640
|
+
// configHash); a success within the freshness window (the worker's own
|
|
641
|
+
// interval, floor 10 min) skips the launch entirely — no budget spend,
|
|
642
|
+
// no process. HEAD moves → new key → the job runs again.
|
|
643
|
+
const identity = resolveGitWorkspaceIdentity(this.projectRoot);
|
|
644
|
+
const jobKey = computeAiJobKey({
|
|
645
|
+
repositoryId: identity.repositoryId,
|
|
646
|
+
head: identity.head,
|
|
647
|
+
workerType,
|
|
648
|
+
configHash: hashWorkerConfig(headless),
|
|
649
|
+
});
|
|
650
|
+
const dedup = getAiJobDedupRegistry();
|
|
651
|
+
const envWindowSecs = Number.parseInt(process.env.RUFLO_AI_DEDUP_WINDOW_SECS || '', 10);
|
|
652
|
+
const freshnessMs = Number.isFinite(envWindowSecs) && envWindowSecs >= 0
|
|
653
|
+
? envWindowSecs * 1000
|
|
654
|
+
: Math.max(baseConfig.intervalMs || 0, 10 * 60 * 1000);
|
|
655
|
+
const freshness = dedup.isFresh(jobKey, freshnessMs);
|
|
656
|
+
if (freshness.fresh) {
|
|
657
|
+
const skipped = {
|
|
658
|
+
success: true,
|
|
659
|
+
dedupSkipped: true,
|
|
660
|
+
output: '',
|
|
661
|
+
parsedOutput: undefined,
|
|
662
|
+
durationMs: 0,
|
|
663
|
+
model: 'none',
|
|
664
|
+
sandboxMode: headless.sandbox,
|
|
665
|
+
workerType,
|
|
666
|
+
timestamp: new Date(),
|
|
667
|
+
executionId,
|
|
668
|
+
};
|
|
669
|
+
this.logExecution(executionId, 'result', `dedup-skip: job ${jobKey.slice(0, 12)} succeeded ${Math.round((Date.now() - (freshness.lastRunAt ?? Date.now())) / 1000)}s ago (repo ${identity.repositoryId.slice(0, 12)}, head ${identity.head.slice(0, 12) || 'n/a'})`);
|
|
670
|
+
this.emit('dedup:skipped', { executionId, workerType, jobKey, lastRunAt: freshness.lastRunAt });
|
|
671
|
+
this.processQueue();
|
|
672
|
+
return skipped;
|
|
673
|
+
}
|
|
674
|
+
// #2661 — every autonomous launch must reserve a slot in the USER-GLOBAL
|
|
675
|
+
// AI budget before any process is created. This is the hard invariant
|
|
676
|
+
// that bounds aggregate launches across all worktree daemons: per-daemon
|
|
677
|
+
// maxConcurrent limits multiply with worktree count, the global budget
|
|
678
|
+
// does not. Denials return an error result (with a receipted reason)
|
|
679
|
+
// instead of queueing, so denied work never piles up into a retry storm.
|
|
680
|
+
const budget = getGlobalAiBudget();
|
|
681
|
+
const model = headless.model || 'sonnet';
|
|
682
|
+
const permit = await budget.reserve({ workerType, model, workspace: this.projectRoot });
|
|
683
|
+
if (!permit.allowed) {
|
|
684
|
+
const denied = this.createErrorResult(workerType, `Denied by global AI budget: ${permit.reason}`);
|
|
685
|
+
denied.executionId = executionId;
|
|
686
|
+
this.logExecution(executionId, 'error', `budget-denied: ${permit.reason}`);
|
|
687
|
+
// NOTE: deliberately no `emit('error', ...)` here — Node treats
|
|
688
|
+
// unlistened 'error' events as throws, and callers consume the
|
|
689
|
+
// returned error result; `budget:denied` is the observable signal.
|
|
690
|
+
this.emit('budget:denied', { executionId, workerType, reason: permit.reason });
|
|
691
|
+
this.processQueue();
|
|
692
|
+
return denied;
|
|
693
|
+
}
|
|
615
694
|
this.emit('start', { executionId, workerType, config: headless });
|
|
616
695
|
try {
|
|
617
696
|
// Build context from file patterns
|
|
@@ -651,6 +730,23 @@ export class HeadlessWorkerExecutor extends EventEmitter {
|
|
|
651
730
|
};
|
|
652
731
|
// Log result
|
|
653
732
|
this.logExecution(executionId, 'result', JSON.stringify(executionResult, null, 2));
|
|
733
|
+
// #2661 invariant 5 — record the success so sibling worktrees at the
|
|
734
|
+
// same HEAD skip this job for the rest of the freshness window.
|
|
735
|
+
if (result.success) {
|
|
736
|
+
dedup.recordSuccess(jobKey, {
|
|
737
|
+
workerType,
|
|
738
|
+
repositoryId: identity.repositoryId,
|
|
739
|
+
workspace: this.projectRoot,
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
// #2661 — a quota/429/usage-limit failure opens the user-global
|
|
743
|
+
// circuit breaker so EVERY daemon stops launching for the cooldown
|
|
744
|
+
// window instead of retrying into an exhausted quota. Only inspected
|
|
745
|
+
// on failure: successful analysis output may legitimately discuss
|
|
746
|
+
// rate limiting in the user's own code.
|
|
747
|
+
if (!result.success && isQuotaErrorText(result.error)) {
|
|
748
|
+
await budget.recordQuotaError(`${workerType}: ${(result.error ?? '').slice(0, 200)}`);
|
|
749
|
+
}
|
|
654
750
|
this.emit('complete', executionResult);
|
|
655
751
|
return executionResult;
|
|
656
752
|
}
|
|
@@ -660,10 +756,15 @@ export class HeadlessWorkerExecutor extends EventEmitter {
|
|
|
660
756
|
executionResult.executionId = executionId;
|
|
661
757
|
executionResult.durationMs = Date.now() - startTime;
|
|
662
758
|
this.logExecution(executionId, 'error', errorMessage);
|
|
759
|
+
if (isQuotaErrorText(errorMessage)) {
|
|
760
|
+
await budget.recordQuotaError(`${workerType}: ${errorMessage.slice(0, 200)}`);
|
|
761
|
+
}
|
|
663
762
|
this.emit('error', executionResult);
|
|
664
763
|
return executionResult;
|
|
665
764
|
}
|
|
666
765
|
finally {
|
|
766
|
+
// #2661 — free the global concurrency slot (launch counts persist).
|
|
767
|
+
await budget.release(permit.permitId);
|
|
667
768
|
// Process next in queue
|
|
668
769
|
this.processQueue();
|
|
669
770
|
}
|
|
@@ -58,6 +58,7 @@ export interface DaemonConfig {
|
|
|
58
58
|
};
|
|
59
59
|
ttlMs: number;
|
|
60
60
|
idleShutdownMs: number;
|
|
61
|
+
aiWorkersEnabled: boolean;
|
|
61
62
|
workers: WorkerConfig[];
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
@@ -237,6 +238,19 @@ export declare class WorkerDaemon extends EventEmitter {
|
|
|
237
238
|
* on its own. A no-op when both limits are disabled (0).
|
|
238
239
|
*/
|
|
239
240
|
private startLifecycleMonitor;
|
|
241
|
+
/**
|
|
242
|
+
* Decide whether the daemon should self-shutdown, and why. Extracted from
|
|
243
|
+
* the lifecycle timer so it is testable without racing a 60s interval or
|
|
244
|
+
* calling process.exit().
|
|
245
|
+
*
|
|
246
|
+
* #2661 (invariant 6, containment form): a removed worktree makes its
|
|
247
|
+
* daemon ineligible within one check interval — the daemon detects that
|
|
248
|
+
* its workspace directory is gone and shuts down instead of continuing to
|
|
249
|
+
* schedule jobs against a deleted tree. The full lease architecture
|
|
250
|
+
* (supervisor-dispatched jobs, heartbeats) is follow-up work; this stops
|
|
251
|
+
* the leak where recreated/removed worktrees leave schedulers behind.
|
|
252
|
+
*/
|
|
253
|
+
private lifecycleShutdownReason;
|
|
240
254
|
/**
|
|
241
255
|
* Most recent worker start/finish time across all workers (epoch ms), or
|
|
242
256
|
* null if no worker has ever started. Used for idle-shutdown detection.
|