jules-orchestrator-kit 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/bin/agentctl.mjs +256 -7
- package/index.mjs +30 -2
- package/package.json +1 -1
- package/src/budget.mjs +105 -78
- package/src/config.mjs +48 -4
- package/src/flaky-ledger.mjs +205 -2
- package/src/ops/command-registry.mjs +54 -0
- package/src/ops/doctor-registry.mjs +37 -0
- package/src/state.mjs +146 -52
- package/src/webhook.mjs +415 -119
- package/src/wizard-init.mjs +6 -1
package/src/budget.mjs
CHANGED
|
@@ -1,25 +1,32 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync, openSync, fsyncSync, closeSync, renameSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { resolveRoot } from "./config.mjs";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
getStateDir,
|
|
6
|
+
ensureDir,
|
|
7
|
+
appendLedger,
|
|
8
|
+
checkDailyBudget,
|
|
9
|
+
scanBudgetWindow,
|
|
10
|
+
ROLLING_WINDOW_MS,
|
|
11
|
+
} from "./state.mjs";
|
|
5
12
|
|
|
6
13
|
/**
|
|
7
14
|
* Where the observed quota ceiling lives, outside the ledger so it survives
|
|
8
15
|
* rotation.
|
|
9
16
|
*
|
|
10
|
-
* Deliberately
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
17
|
+
* Deliberately short-lived. The local ledger only counts tasks dispatched from
|
|
18
|
+
* *this* checkout, while the account's quota is also spent from the web UI and
|
|
19
|
+
* other machines, so the local count at the moment of a refusal is a lower
|
|
20
|
+
* bound on the real allowance — not the allowance. Treated as permanent it
|
|
21
|
+
* would hard-block the operator below their own quota, which is the exact
|
|
22
|
+
* failure this whole mechanism exists to prevent.
|
|
16
23
|
*
|
|
17
|
-
* What it does mean is precise and useful:
|
|
18
|
-
* refusing, so stop asking until
|
|
24
|
+
* What it does mean is precise and useful: within the last 24 hours the
|
|
25
|
+
* provider started refusing, so stop asking until that refusal ages out.
|
|
19
26
|
*/
|
|
20
27
|
export const CEILING_FILE = "budget-ceiling.json";
|
|
21
28
|
|
|
22
|
-
/** Local calendar day,
|
|
29
|
+
/** Local calendar day, retained as the fallback key for pre-0.34 records. */
|
|
23
30
|
function today() {
|
|
24
31
|
return new Date().toISOString().split("T")[0];
|
|
25
32
|
}
|
|
@@ -67,10 +74,17 @@ function writeAtomic(filePath, content) {
|
|
|
67
74
|
|
|
68
75
|
/**
|
|
69
76
|
* Read the stored ceiling, whenever it was observed.
|
|
77
|
+
*
|
|
78
|
+
* A refusal expires 24 hours after it happened, matching the window the quota
|
|
79
|
+
* itself resets on. Expiring it at midnight instead — as this did before —
|
|
80
|
+
* either freed the operator hours before the provider would, or kept them
|
|
81
|
+
* blocked hours after it already had.
|
|
82
|
+
*
|
|
70
83
|
* @param {string} [root]
|
|
71
|
-
* @
|
|
84
|
+
* @param {number} [now] - Epoch ms; injectable for tests.
|
|
85
|
+
* @returns {{ ceiling: number, day: string, observedAt: string, source: string, stale: boolean, expiresAt: string, msRemaining: number } | null}
|
|
72
86
|
*/
|
|
73
|
-
export function readObservedCeiling(root = resolveRoot()) {
|
|
87
|
+
export function readObservedCeiling(root = resolveRoot(), now = Date.now()) {
|
|
74
88
|
const filePath = join(getStateDir(root), CEILING_FILE);
|
|
75
89
|
if (!existsSync(filePath)) return null;
|
|
76
90
|
try {
|
|
@@ -78,12 +92,24 @@ export function readObservedCeiling(root = resolveRoot()) {
|
|
|
78
92
|
if (!parsed || typeof parsed.ceiling !== "number" || !Number.isFinite(parsed.ceiling)) return null;
|
|
79
93
|
if (parsed.ceiling < 0) return null;
|
|
80
94
|
const day = typeof parsed.day === "string" ? parsed.day : String(parsed.observedAt || "").slice(0, 10);
|
|
95
|
+
const observedAt = typeof parsed.observedAt === "string" ? parsed.observedAt : "";
|
|
96
|
+
|
|
97
|
+
// Records written before 0.34.0 carry only a day. Falling back to the old
|
|
98
|
+
// calendar comparison keeps them honest rather than reviving a ceiling
|
|
99
|
+
// whose age cannot be established.
|
|
100
|
+
const observedMs = Date.parse(observedAt);
|
|
101
|
+
const dated = Number.isFinite(observedMs);
|
|
102
|
+
const age = dated ? now - observedMs : Number.POSITIVE_INFINITY;
|
|
103
|
+
const stale = dated ? age >= ROLLING_WINDOW_MS : day !== today();
|
|
104
|
+
|
|
81
105
|
return {
|
|
82
106
|
ceiling: Math.floor(parsed.ceiling),
|
|
83
107
|
day,
|
|
84
|
-
observedAt
|
|
108
|
+
observedAt,
|
|
85
109
|
source: typeof parsed.source === "string" ? parsed.source : "provider-rejection",
|
|
86
|
-
stale
|
|
110
|
+
stale,
|
|
111
|
+
expiresAt: dated ? new Date(observedMs + ROLLING_WINDOW_MS).toISOString() : "",
|
|
112
|
+
msRemaining: dated ? Math.max(0, ROLLING_WINDOW_MS - age) : 0,
|
|
87
113
|
};
|
|
88
114
|
} catch (_) {
|
|
89
115
|
return null;
|
|
@@ -94,14 +120,14 @@ export function readObservedCeiling(root = resolveRoot()) {
|
|
|
94
120
|
* The ceiling only if it still applies — i.e. observed today.
|
|
95
121
|
* @param {string} [root]
|
|
96
122
|
*/
|
|
97
|
-
export function readActiveCeiling(root = resolveRoot()) {
|
|
98
|
-
const rec = readObservedCeiling(root);
|
|
123
|
+
export function readActiveCeiling(root = resolveRoot(), now = Date.now()) {
|
|
124
|
+
const rec = readObservedCeiling(root, now);
|
|
99
125
|
return rec && !rec.stale ? rec : null;
|
|
100
126
|
}
|
|
101
127
|
|
|
102
128
|
/**
|
|
103
129
|
* Record that the provider refused further work after `usedAtRejection` tasks
|
|
104
|
-
* were dispatched locally
|
|
130
|
+
* were dispatched locally inside the current window.
|
|
105
131
|
*
|
|
106
132
|
* Zero is a legitimate value: it means the quota was already spent elsewhere
|
|
107
133
|
* (the web UI, another machine) before this checkout dispatched anything.
|
|
@@ -166,16 +192,17 @@ export function resolveDailyLimit(config, root = resolveRoot()) {
|
|
|
166
192
|
};
|
|
167
193
|
}
|
|
168
194
|
|
|
169
|
-
// Only
|
|
170
|
-
// about
|
|
171
|
-
// operator locked out after the
|
|
195
|
+
// Only a refusal from the last 24 hours may enforce. An older one says
|
|
196
|
+
// nothing about the remaining quota, and carrying it forward would keep the
|
|
197
|
+
// operator locked out after the window had already reset.
|
|
172
198
|
const learned = readActiveCeiling(root);
|
|
173
199
|
if (learned) {
|
|
200
|
+
const hours = Math.max(1, Math.round(learned.msRemaining / 3600000));
|
|
174
201
|
return {
|
|
175
202
|
limit: learned.ceiling,
|
|
176
203
|
source: "learned",
|
|
177
204
|
certain: true,
|
|
178
|
-
note: `the provider refused further work
|
|
205
|
+
note: `the provider refused further work after ${learned.ceiling} local task(s); that refusal ages out in ~${hours}h`,
|
|
179
206
|
};
|
|
180
207
|
}
|
|
181
208
|
|
|
@@ -189,71 +216,61 @@ export function resolveDailyLimit(config, root = resolveRoot()) {
|
|
|
189
216
|
}
|
|
190
217
|
|
|
191
218
|
/**
|
|
192
|
-
*
|
|
219
|
+
* Resolve how many workers may run at once, and against what ceiling.
|
|
220
|
+
*
|
|
221
|
+
* The same provenance rule as {@link resolveDailyLimit}: a figure the operator
|
|
222
|
+
* stated is authoritative, a tier preset is a default. The difference is that
|
|
223
|
+
* exceeding this one is not the kit's call to refuse — the provider enforces
|
|
224
|
+
* its own slot limit, and an operator pooling several accounts legitimately
|
|
225
|
+
* runs past any single plan's ceiling. So an overrun is reported, never
|
|
226
|
+
* blocked.
|
|
227
|
+
*
|
|
228
|
+
* @param {object} config - A config from loadConfig().
|
|
229
|
+
* @returns {{ concurrency: number, ceiling: number, source: "config"|"tier", overCeiling: boolean, note: string }}
|
|
230
|
+
*/
|
|
231
|
+
export function resolveConcurrency(config) {
|
|
232
|
+
const concurrency = Math.max(1, Math.floor(Number(config?.limits?.concurrency) || 1));
|
|
233
|
+
const ceiling = Math.floor(Number(config?.limits?.maxConcurrency) || 0);
|
|
234
|
+
const source = config?.provenance?.concurrency === "config" ? "config" : "tier";
|
|
235
|
+
const overCeiling = ceiling > 0 && concurrency > ceiling;
|
|
236
|
+
const tier = config?.tier || "unknown";
|
|
237
|
+
|
|
238
|
+
let note;
|
|
239
|
+
if (overCeiling) {
|
|
240
|
+
note = `${concurrency} workers exceeds what the "${tier}" plan allows (${ceiling}); sessions past the ${ceiling}th will be refused unless the account pools several plans`;
|
|
241
|
+
} else if (source === "config") {
|
|
242
|
+
note = ceiling > 0 ? `set in .agent/config.yml (plan allows up to ${ceiling})` : "set in .agent/config.yml";
|
|
243
|
+
} else {
|
|
244
|
+
note = ceiling > 0
|
|
245
|
+
? `tier default for "${tier}" — the plan allows up to ${ceiling}, held back to leave slots for sessions this ledger cannot see`
|
|
246
|
+
: `tier default for "${tier}"`;
|
|
247
|
+
}
|
|
248
|
+
return { concurrency, ceiling, source, overCeiling, note };
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* List reservations the rolling 24-hour window still counts as spent.
|
|
193
253
|
*
|
|
194
254
|
* A reservation is open until a `budget_rolled_back` or `budget_released` entry
|
|
195
255
|
* names it. `budget_committed` deliberately does not close one — a committed
|
|
196
256
|
* dispatch really did consume quota — so the open set is "everything reserved
|
|
197
|
-
*
|
|
257
|
+
* inside the window that was not given back".
|
|
198
258
|
*
|
|
199
|
-
* Anonymous reservations (no `reservationId`, as older kit versions
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
* is nothing a targeted release could name. They are matched by count, the same
|
|
204
|
-
* way the counter itself treats them.
|
|
259
|
+
* Anonymous reservations (no `reservationId`, as older kit versions wrote them)
|
|
260
|
+
* appear as records with `reservationId: null`. They must, or a reconcile would
|
|
261
|
+
* silently leave them charged: the counter counts them, and with no id there is
|
|
262
|
+
* nothing a targeted release could name.
|
|
205
263
|
*
|
|
206
264
|
* @param {string} [root]
|
|
265
|
+
* @param {object} [opts] - Forwarded to scanBudgetWindow (`now`, `windowMs`).
|
|
207
266
|
* @returns {{ reservationId: string|null, timestamp: string, committed: boolean }[]}
|
|
208
267
|
*/
|
|
209
|
-
export function listOpenReservations(root = resolveRoot()) {
|
|
210
|
-
|
|
211
|
-
if (!existsSync(filePath)) return [];
|
|
212
|
-
|
|
213
|
-
/** @type {Map<string, { reservationId: string, timestamp: string, committed: boolean }>} */
|
|
214
|
-
const open = new Map();
|
|
215
|
-
/** @type {{ reservationId: null, timestamp: string, committed: boolean }[]} */
|
|
216
|
-
const anonymous = [];
|
|
217
|
-
let raw = "";
|
|
218
|
-
try {
|
|
219
|
-
raw = readFileSync(filePath, "utf-8");
|
|
220
|
-
} catch (_) {
|
|
221
|
-
return [];
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
for (const line of raw.split("\n").filter(Boolean)) {
|
|
225
|
-
let entry;
|
|
226
|
-
try {
|
|
227
|
-
entry = JSON.parse(line);
|
|
228
|
-
} catch (_) {
|
|
229
|
-
continue;
|
|
230
|
-
}
|
|
231
|
-
if (!entry || !entry.event) continue;
|
|
232
|
-
|
|
233
|
-
if (entry.event === "budget_reserved") {
|
|
234
|
-
if (entry.reservationId) {
|
|
235
|
-
open.set(entry.reservationId, {
|
|
236
|
-
reservationId: entry.reservationId,
|
|
237
|
-
timestamp: entry.timestamp || "",
|
|
238
|
-
committed: false,
|
|
239
|
-
});
|
|
240
|
-
} else {
|
|
241
|
-
anonymous.push({ reservationId: null, timestamp: entry.timestamp || "", committed: false });
|
|
242
|
-
}
|
|
243
|
-
} else if (entry.event === "budget_committed") {
|
|
244
|
-
const rec = entry.reservationId ? open.get(entry.reservationId) : null;
|
|
245
|
-
if (rec) rec.committed = true;
|
|
246
|
-
} else if (entry.event === "budget_rolled_back" || entry.event === "budget_released") {
|
|
247
|
-
if (entry.reservationId) open.delete(entry.reservationId);
|
|
248
|
-
else anonymous.pop();
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
return [...open.values(), ...anonymous];
|
|
268
|
+
export function listOpenReservations(root = resolveRoot(), opts = {}) {
|
|
269
|
+
return scanBudgetWindow(root, opts).open;
|
|
253
270
|
}
|
|
254
271
|
|
|
255
272
|
/**
|
|
256
|
-
* Give
|
|
273
|
+
* Give the window's open reservations back, by appending `budget_released` entries.
|
|
257
274
|
*
|
|
258
275
|
* The ledger is append-only and hash-chained, so a miscounted day is corrected
|
|
259
276
|
* forwards — never by editing or deleting the file, which would break the chain
|
|
@@ -286,10 +303,16 @@ export function releaseOpenReservations(opts = {}) {
|
|
|
286
303
|
|
|
287
304
|
for (const rec of openRecords) {
|
|
288
305
|
const entry = { event: "budget_released", reason: opts.reason || "operator-reconcile" };
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
306
|
+
if (rec.reservationId) {
|
|
307
|
+
entry.reservationId = rec.reservationId;
|
|
308
|
+
} else {
|
|
309
|
+
// An anonymous reservation is released by an equally anonymous entry —
|
|
310
|
+
// naming an id here would leave the original charged and subtract from
|
|
311
|
+
// someone else's total instead. The timestamp pins which one it cancels,
|
|
312
|
+
// so the pair does not drift apart as the rolling window advances past
|
|
313
|
+
// the reservation but not yet past its release.
|
|
314
|
+
entry.releasedTimestamp = rec.timestamp;
|
|
315
|
+
}
|
|
293
316
|
appendLedger(entry, root);
|
|
294
317
|
}
|
|
295
318
|
return result;
|
|
@@ -311,6 +334,10 @@ export function budgetStatus(config, root = resolveRoot()) {
|
|
|
311
334
|
source: resolved.source,
|
|
312
335
|
certain: resolved.certain,
|
|
313
336
|
note: resolved.note,
|
|
337
|
+
// The count is a rolling 24h window, not a calendar day — surfaced so a
|
|
338
|
+
// caller reporting "used today" cannot quietly mean something else.
|
|
339
|
+
windowStart: check.windowStart || "",
|
|
340
|
+
windowHours: ROLLING_WINDOW_MS / 3600000,
|
|
314
341
|
// Only a limit we actually know may stop a dispatch.
|
|
315
342
|
enforced: resolved.certain,
|
|
316
343
|
exhausted: check.used >= resolved.limit,
|
package/src/config.mjs
CHANGED
|
@@ -281,35 +281,63 @@ export function resolveVerify(root = process.cwd(), userVerify = {}) {
|
|
|
281
281
|
* guess — see `isTierGuess()` and the learned-ceiling logic in src/state.mjs.
|
|
282
282
|
* An explicit `limits.daily_tasks` in .agent/config.yml always wins.
|
|
283
283
|
*/
|
|
284
|
+
/**
|
|
285
|
+
* Per-tier defaults, with the vendor's own ceiling recorded alongside them.
|
|
286
|
+
*
|
|
287
|
+
* `maxConcurrency` is what the plan allows; `concurrency` is what the kit will
|
|
288
|
+
* start by using. They differ on purpose, for the same reason the daily count
|
|
289
|
+
* is a lower bound: this ledger sees one checkout, while the account's slots
|
|
290
|
+
* are also taken by the web UI, the CLI and other machines. A default sitting
|
|
291
|
+
* on the ceiling would collide with every session the kit cannot see.
|
|
292
|
+
*
|
|
293
|
+
* The defaults used to sit at 1/2/3 against ceilings of 3/15/60 — a Pro
|
|
294
|
+
* account running two workers where it could run fifteen. Raising them is the
|
|
295
|
+
* single largest throughput change available; leaving headroom is what keeps
|
|
296
|
+
* it from being reckless. An operator who knows their account is theirs alone
|
|
297
|
+
* can state `limits.concurrency` up to the ceiling.
|
|
298
|
+
*
|
|
299
|
+
* Ceilings verified against the vendor's published limits page
|
|
300
|
+
* (jules.google/docs/usage-limits, 2026-08-20). They are a vendor number and
|
|
301
|
+
* may change without notice. The URL is written without a scheme on purpose:
|
|
302
|
+
* test/egress-allowlist.test.mjs treats every host literal in shipped source
|
|
303
|
+
* as a destination this kit might contact, and a citation is not one.
|
|
304
|
+
*/
|
|
284
305
|
export const TIER_PRESETS = {
|
|
285
306
|
free: {
|
|
286
307
|
dailyTasks: 15,
|
|
287
308
|
repairAttempts: 1,
|
|
288
|
-
|
|
309
|
+
// The whole allowance is 15 tasks a day; there is no headroom worth
|
|
310
|
+
// reserving, so the default is the ceiling.
|
|
311
|
+
concurrency: 3,
|
|
312
|
+
maxConcurrency: 3,
|
|
289
313
|
staggerMs: 3000,
|
|
290
314
|
diffKb: 50,
|
|
291
315
|
},
|
|
292
316
|
pro: {
|
|
293
317
|
dailyTasks: 100,
|
|
294
318
|
repairAttempts: 2,
|
|
295
|
-
concurrency:
|
|
319
|
+
concurrency: 8,
|
|
320
|
+
maxConcurrency: 15,
|
|
296
321
|
staggerMs: 1500,
|
|
297
322
|
diffKb: 75,
|
|
298
323
|
},
|
|
299
324
|
ultra: {
|
|
300
325
|
dailyTasks: 300,
|
|
301
326
|
repairAttempts: 3,
|
|
302
|
-
concurrency:
|
|
327
|
+
concurrency: 15,
|
|
328
|
+
maxConcurrency: 60,
|
|
303
329
|
staggerMs: 1000,
|
|
304
330
|
diffKb: 75,
|
|
305
331
|
},
|
|
306
332
|
// Not a vendor plan: a self-hosted/pooled profile for operators who front
|
|
307
333
|
// several accounts. Defined here so `tier: enterprise` resolves to what the
|
|
308
|
-
// wizard writes instead of silently collapsing onto the ultra preset.
|
|
334
|
+
// wizard writes instead of silently collapsing onto the ultra preset. Its
|
|
335
|
+
// ceiling is whatever the pool adds up to, so the kit does not claim one.
|
|
309
336
|
enterprise: {
|
|
310
337
|
dailyTasks: 1000,
|
|
311
338
|
repairAttempts: 3,
|
|
312
339
|
concurrency: 10,
|
|
340
|
+
maxConcurrency: 0,
|
|
313
341
|
staggerMs: 500,
|
|
314
342
|
diffKb: 100,
|
|
315
343
|
},
|
|
@@ -401,6 +429,21 @@ export function loadConfig(root = resolveRoot(), explicitPath = null) {
|
|
|
401
429
|
complex: parsed.router?.complex || "",
|
|
402
430
|
threshold: Number.isFinite(Number(parsed.router?.threshold)) ? Number(parsed.router.threshold) : 0,
|
|
403
431
|
},
|
|
432
|
+
notifications: {
|
|
433
|
+
mode: parsed.notifications?.mode || "immediate",
|
|
434
|
+
threshold: Number.isFinite(Number(parsed.notifications?.threshold)) ? Number(parsed.notifications.threshold) : 5,
|
|
435
|
+
windowMs: Number.isFinite(Number(parsed.notifications?.window_ms ?? parsed.notifications?.windowMs))
|
|
436
|
+
? Number(parsed.notifications.window_ms ?? parsed.notifications.windowMs)
|
|
437
|
+
: 300000,
|
|
438
|
+
budgetPerHour: Number.isFinite(Number(parsed.notifications?.budget_per_hour ?? parsed.notifications?.budgetPerHour))
|
|
439
|
+
? Number(parsed.notifications.budget_per_hour ?? parsed.notifications.budgetPerHour)
|
|
440
|
+
: 3,
|
|
441
|
+
criticalReasons: Array.isArray(parsed.notifications?.critical_reasons)
|
|
442
|
+
? parsed.notifications.critical_reasons
|
|
443
|
+
: ["R3_GATE_VIOLATION", "AWAITING_USER_FEEDBACK", "OODA_REPAIR_EXHAUSTED", "SECRET_LEAK_DETECTED", "CRITICAL_FAILURE"],
|
|
444
|
+
slackWebhookUrl: parsed.notifications?.slack_webhook_url || parsed.notifications?.slackWebhookUrl || "",
|
|
445
|
+
discordWebhookUrl: parsed.notifications?.discord_webhook_url || parsed.notifications?.discordWebhookUrl || "",
|
|
446
|
+
},
|
|
404
447
|
scope: normalizeScope(parsed),
|
|
405
448
|
limits: {
|
|
406
449
|
...DEFAULTS.limits,
|
|
@@ -420,6 +463,7 @@ export function loadConfig(root = resolveRoot(), explicitPath = null) {
|
|
|
420
463
|
: normalizedLimits.dailyTasks !== undefined
|
|
421
464
|
? "config"
|
|
422
465
|
: "tier",
|
|
466
|
+
concurrency: normalizedLimits.concurrency !== undefined ? "config" : "tier",
|
|
423
467
|
},
|
|
424
468
|
isolation: parsed.isolation || DEFAULTS.isolation,
|
|
425
469
|
runner: parsed.runner || DEFAULTS.runner,
|
package/src/flaky-ledger.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { readFileSync, existsSync, appendFileSync } from "node:fs";
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync, appendFileSync, unlinkSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { getStateDir, ensureDir } from "./state.mjs";
|
|
3
|
+
import { getStateDir, getQueueDir, ensureDir } from "./state.mjs";
|
|
4
|
+
import { resolveRoot } from "./config.mjs";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* Calculates Wilson score interval for binomial proportion.
|
|
@@ -169,3 +170,205 @@ export function flakyVerdict(runs = []) {
|
|
|
169
170
|
wilson,
|
|
170
171
|
};
|
|
171
172
|
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Lists all test suites/commands currently classified as QUARANTINED in .agent/state/flaky.jsonl.
|
|
176
|
+
* @param {string} [root]
|
|
177
|
+
* @returns {Array<object>} Quarantined tests with oscillation & Wilson stats
|
|
178
|
+
*/
|
|
179
|
+
export function listQuarantinedTests(root = resolveRoot()) {
|
|
180
|
+
const allRuns = readVerifyRuns(root);
|
|
181
|
+
if (allRuns.length === 0) return [];
|
|
182
|
+
|
|
183
|
+
// Group by testCmd
|
|
184
|
+
const grouped = new Map();
|
|
185
|
+
for (const run of allRuns) {
|
|
186
|
+
const cmd = run.testCmd || "default";
|
|
187
|
+
if (!grouped.has(cmd)) grouped.set(cmd, []);
|
|
188
|
+
grouped.get(cmd).push(run);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const quarantined = [];
|
|
192
|
+
for (const [testCmd, runs] of grouped.entries()) {
|
|
193
|
+
const verdict = flakyVerdict(runs);
|
|
194
|
+
if (verdict.verdict === "QUARANTINED") {
|
|
195
|
+
const lastRun = runs[runs.length - 1];
|
|
196
|
+
quarantined.push({
|
|
197
|
+
testCmd,
|
|
198
|
+
verdict: verdict.verdict,
|
|
199
|
+
oscillation: verdict.oscillation,
|
|
200
|
+
fails: verdict.fails,
|
|
201
|
+
passes: verdict.passes,
|
|
202
|
+
n: verdict.n,
|
|
203
|
+
wilson: verdict.wilson,
|
|
204
|
+
lastRunTimestamp: lastRun?.timestamp || new Date().toISOString(),
|
|
205
|
+
lastDurationMs: lastRun?.durationMs || 0,
|
|
206
|
+
fingerprint: lastRun?.fingerprint || null,
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
return quarantined;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Clears flaky ledger entries for all tests or a specific command.
|
|
216
|
+
* @param {string} [root]
|
|
217
|
+
* @param {string} [testCmd]
|
|
218
|
+
*/
|
|
219
|
+
export function clearFlakyLedger(root = resolveRoot(), testCmd = null) {
|
|
220
|
+
const stateDir = getStateDir(root);
|
|
221
|
+
const filePath = join(stateDir, "flaky.jsonl");
|
|
222
|
+
if (!existsSync(filePath)) return { ok: true, cleared: 0 };
|
|
223
|
+
|
|
224
|
+
if (!testCmd) {
|
|
225
|
+
try {
|
|
226
|
+
unlinkSync(filePath);
|
|
227
|
+
} catch (_) {
|
|
228
|
+
writeFileSync(filePath, "", "utf-8");
|
|
229
|
+
}
|
|
230
|
+
return { ok: true, cleared: "all" };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const runs = readVerifyRuns(root);
|
|
234
|
+
const retained = runs.filter((r) => r.testCmd !== testCmd);
|
|
235
|
+
const clearedCount = runs.length - retained.length;
|
|
236
|
+
writeFileSync(filePath, retained.map((r) => JSON.stringify(r)).join("\n") + (retained.length ? "\n" : ""), "utf-8");
|
|
237
|
+
return { ok: true, cleared: clearedCount, remaining: retained.length };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Synthesizes a specialized anti-flakiness prompt envelope for a quarantined test.
|
|
242
|
+
* @param {object|string} quarantinedItem - Quarantined item object or test command string
|
|
243
|
+
* @param {object} [options]
|
|
244
|
+
* @returns {{ taskId: string, title: string, prompt: string, role: string, verifyCmd: string, fullEnvelope: string }}
|
|
245
|
+
*/
|
|
246
|
+
export function synthesizeFlakyHealingTask(quarantinedItem, options = {}) {
|
|
247
|
+
const item = typeof quarantinedItem === "string" ? { testCmd: quarantinedItem, oscillation: 0.5, fails: 3, passes: 3, n: 6 } : quarantinedItem;
|
|
248
|
+
const testCmd = item.testCmd || "npm test";
|
|
249
|
+
const oscillationPct = Math.round((item.oscillation || 0.4) * 100);
|
|
250
|
+
const role = options.role || "janitor";
|
|
251
|
+
const slug = String(testCmd).replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").slice(0, 20).toLowerCase();
|
|
252
|
+
const taskId = options.taskId || `flaky-heal-${slug}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
|
|
253
|
+
|
|
254
|
+
// Build verification oracle that verifies deterministic stability over multiple consecutive runs
|
|
255
|
+
const verifyOracle = options.verifyCmd || `${testCmd} && ${testCmd} && ${testCmd}`;
|
|
256
|
+
|
|
257
|
+
const prompt = `# Task: Eliminate Flaky Test Timing Oscillations & Race Conditions
|
|
258
|
+
|
|
259
|
+
## Context & Statistical Diagnosis
|
|
260
|
+
The test command \`${testCmd}\` has been quarantined by the Wilson-Score Statistical Flaky Guard (Exit Code 8).
|
|
261
|
+
- **Oscillation Rate**: ${oscillationPct}% state transitions between passes and failures
|
|
262
|
+
- **Total Sample Window**: ${item.n || 6} runs (${item.fails || 3} failures, ${item.passes || 3} passes)
|
|
263
|
+
- **Quarantine Verdict**: Statistical non-determinism detected (Wilson CI interior score).
|
|
264
|
+
|
|
265
|
+
## Root Cause Remediations Required
|
|
266
|
+
1. **Eliminate Arbitrary Sleep & Polling Race Conditions**:
|
|
267
|
+
- Replace arbitrary timers (\`sleep()\`, \`setTimeout()\`, \`delay()\`) with deterministic condition-based assertions or event-driven completion promises.
|
|
268
|
+
2. **State & Resource Isolation**:
|
|
269
|
+
- Ensure tests clean up global variables, open socket descriptors, temporary filesystem directories, or database handles in \`afterEach\` / teardown hooks.
|
|
270
|
+
- Prevent port collisions by binding to ephemeral ports (port \`0\`) or namespaced mutexes.
|
|
271
|
+
3. **Mock Unreliable Boundaries**:
|
|
272
|
+
- Intercept and mock non-deterministic external network calls, system clocks, and asynchronous background timers.
|
|
273
|
+
4. **STRICT INVARIANT: NO TEST WEAKENING**:
|
|
274
|
+
- You are **strictly forbidden** from deleting failing assertions, commenting out checks, skipping test cases, or increasing broad timeouts indefinitely to force a pass.
|
|
275
|
+
- The test requirements and assertions must remain 100% rigorous; only the underlying non-determinism, race conditions, or unhandled timing flaws must be fixed.
|
|
276
|
+
|
|
277
|
+
## Verification Gate
|
|
278
|
+
Before opening PR, the test must pass cleanly across consecutive executions without a single oscillation:
|
|
279
|
+
\`\`\`bash
|
|
280
|
+
${verifyOracle}
|
|
281
|
+
\`\`\``;
|
|
282
|
+
|
|
283
|
+
const title = `Heal Flaky Test: ${testCmd.slice(0, 50)}`;
|
|
284
|
+
|
|
285
|
+
const envelopeMetadata = {
|
|
286
|
+
version: 1,
|
|
287
|
+
id: taskId,
|
|
288
|
+
title,
|
|
289
|
+
role,
|
|
290
|
+
verifyCmd: verifyOracle,
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
const fullEnvelope = `<!-- JULES_TASK_ENVELOPE: ${JSON.stringify(envelopeMetadata)} -->
|
|
294
|
+
# ${title}
|
|
295
|
+
# Task ID: ${taskId}
|
|
296
|
+
|
|
297
|
+
[TASK INSTRUCTIONS]
|
|
298
|
+
${prompt}
|
|
299
|
+
|
|
300
|
+
[VERIFICATION ORACLE]
|
|
301
|
+
Test/Verification Command: ${verifyOracle}
|
|
302
|
+
`;
|
|
303
|
+
|
|
304
|
+
return {
|
|
305
|
+
taskId,
|
|
306
|
+
title,
|
|
307
|
+
prompt,
|
|
308
|
+
role,
|
|
309
|
+
verifyCmd: verifyOracle,
|
|
310
|
+
fullEnvelope,
|
|
311
|
+
item,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Runs the Flaky Healing Swarm: scans quarantined tests, synthesizes healing envelopes,
|
|
317
|
+
* and either enqueues them in .agent/jules-queue/ or dispatches them directly.
|
|
318
|
+
* @param {string} [root]
|
|
319
|
+
* @param {object} [options]
|
|
320
|
+
* @returns {Promise<object>}
|
|
321
|
+
*/
|
|
322
|
+
export async function runFlakyHealingSwarm(root = resolveRoot(), options = {}) {
|
|
323
|
+
const quarantined = listQuarantinedTests(root);
|
|
324
|
+
if (quarantined.length === 0) {
|
|
325
|
+
return { count: 0, tasks: [], message: "No quarantined flaky tests detected in repository." };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const tasks = [];
|
|
329
|
+
const queueDir = getQueueDir(root);
|
|
330
|
+
|
|
331
|
+
for (const item of quarantined) {
|
|
332
|
+
const taskPlan = synthesizeFlakyHealingTask(item, options);
|
|
333
|
+
tasks.push(taskPlan);
|
|
334
|
+
|
|
335
|
+
if (options.dryRun) {
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
if (options.dispatch) {
|
|
340
|
+
const { dispatch } = await import("./engine.mjs");
|
|
341
|
+
const { loadConfig } = await import("./config.mjs");
|
|
342
|
+
const config = loadConfig(root);
|
|
343
|
+
try {
|
|
344
|
+
const session = await dispatch(
|
|
345
|
+
{
|
|
346
|
+
title: taskPlan.title,
|
|
347
|
+
prompt: taskPlan.prompt,
|
|
348
|
+
role: taskPlan.role,
|
|
349
|
+
},
|
|
350
|
+
{ root, config }
|
|
351
|
+
);
|
|
352
|
+
taskPlan.session = session;
|
|
353
|
+
taskPlan.dispatched = true;
|
|
354
|
+
} catch (err) {
|
|
355
|
+
taskPlan.dispatchError = err.message;
|
|
356
|
+
taskPlan.dispatched = false;
|
|
357
|
+
}
|
|
358
|
+
} else {
|
|
359
|
+
const fileName = `${taskPlan.taskId}.md`;
|
|
360
|
+
const filePath = join(queueDir, fileName);
|
|
361
|
+
writeFileSync(filePath, taskPlan.fullEnvelope, "utf-8");
|
|
362
|
+
taskPlan.taskFile = filePath;
|
|
363
|
+
taskPlan.queued = true;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
return {
|
|
368
|
+
count: tasks.length,
|
|
369
|
+
tasks,
|
|
370
|
+
queued: !options.dispatch && !options.dryRun,
|
|
371
|
+
dispatched: Boolean(options.dispatch && !options.dryRun),
|
|
372
|
+
dryRun: Boolean(options.dryRun),
|
|
373
|
+
};
|
|
374
|
+
}
|
|
@@ -222,6 +222,60 @@ export const COMMAND_REGISTRY = [
|
|
|
222
222
|
{ name: "json", type: "boolean", description: "Output status in JSON format" },
|
|
223
223
|
],
|
|
224
224
|
},
|
|
225
|
+
{
|
|
226
|
+
id: "escalate",
|
|
227
|
+
path: ["escalate"],
|
|
228
|
+
title: "escalate",
|
|
229
|
+
description: "Dispatch or manage webhook escalation incidents with Silence Governor",
|
|
230
|
+
category: "Operate",
|
|
231
|
+
mutates: true,
|
|
232
|
+
risk: "low",
|
|
233
|
+
interactive: "never",
|
|
234
|
+
requiresRepository: true,
|
|
235
|
+
shortcuts: ["esc"],
|
|
236
|
+
examples: [
|
|
237
|
+
'agentctl escalate sess-123 --reason "AWAITING_USER_FEEDBACK"',
|
|
238
|
+
"agentctl escalate --status",
|
|
239
|
+
"agentctl escalate --flush",
|
|
240
|
+
"agentctl escalate --clear",
|
|
241
|
+
],
|
|
242
|
+
flags: [
|
|
243
|
+
{ name: "reason", type: "string", description: "Escalation reason category" },
|
|
244
|
+
{ name: "branch", type: "string", description: "Target git branch" },
|
|
245
|
+
{ name: "logs", type: "string", description: "Error logs text" },
|
|
246
|
+
{ name: "critical", type: "boolean", description: "Bypass Silence Governor and alert immediately" },
|
|
247
|
+
{ name: "flush", type: "boolean", description: "Flush buffered escalation digest" },
|
|
248
|
+
{ name: "status", type: "boolean", description: "Inspect digest status and interruption budget" },
|
|
249
|
+
{ name: "clear", type: "boolean", description: "Clear pending digest buffer" },
|
|
250
|
+
{ name: "dry-run", type: "boolean", description: "Simulate dispatch without sending HTTP requests" },
|
|
251
|
+
{ name: "json", type: "boolean", description: "Output JSON structured response" },
|
|
252
|
+
],
|
|
253
|
+
},
|
|
254
|
+
{
|
|
255
|
+
id: "flaky",
|
|
256
|
+
path: ["flaky"],
|
|
257
|
+
title: "flaky",
|
|
258
|
+
description: "Manage Wilson-quarantined tests and dispatch healing swarm",
|
|
259
|
+
category: "Repair",
|
|
260
|
+
mutates: true,
|
|
261
|
+
risk: "moderate",
|
|
262
|
+
interactive: "never",
|
|
263
|
+
requiresRepository: true,
|
|
264
|
+
shortcuts: ["flk"],
|
|
265
|
+
examples: [
|
|
266
|
+
"agentctl flaky status",
|
|
267
|
+
"agentctl flaky heal",
|
|
268
|
+
'agentctl flaky heal "npm test"',
|
|
269
|
+
"agentctl flaky reset",
|
|
270
|
+
],
|
|
271
|
+
flags: [
|
|
272
|
+
{ name: "dispatch", type: "boolean", description: "Dispatch healing tasks directly to AI agents" },
|
|
273
|
+
{ name: "role", type: "string", description: "Agent persona role (default: janitor)" },
|
|
274
|
+
{ name: "test-cmd", type: "string", description: "Target specific test command" },
|
|
275
|
+
{ name: "dry-run", type: "boolean", description: "Simulate healing swarm generation without writing" },
|
|
276
|
+
{ name: "json", type: "boolean", description: "Output JSON structured response" },
|
|
277
|
+
],
|
|
278
|
+
},
|
|
225
279
|
];
|
|
226
280
|
|
|
227
281
|
/**
|