jules-orchestrator-kit 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -80
- package/bin/agentctl.mjs +3 -3
- package/bin/init.js +2 -3
- package/index.mjs +14 -2
- package/package.json +9 -14
- package/scripts/jules-queue-runner.mjs +3 -22
- package/scripts/utils.mjs +65 -0
- package/src/engine.mjs +138 -24
- package/src/envelope.mjs +3 -3
- package/src/flaky-ledger.mjs +0 -2
- package/src/git.mjs +1 -6
- package/src/mcp.mjs +1 -1
- package/src/merge-verify.mjs +7 -7
- package/src/provider.mjs +88 -6
- package/src/state.mjs +45 -11
- package/src/telemetry.mjs +55 -19
- package/.agent/jules-queue/README.md +0 -30
- package/.agent/jules.yml +0 -13
- package/.env.example +0 -73
- package/.github/workflows/jules-audit.yml +0 -49
- package/scripts/jules-cleanup.mjs +0 -49
- package/scripts/jules-swarm.mjs +0 -38
- package/scripts/lock-manager.mjs +0 -22
package/src/engine.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { loadConfig, parseYaml, normalizeScope } from "./config.mjs";
|
|
2
2
|
import { checkScope, scanDiff, redactSecrets } from "./security.mjs";
|
|
3
3
|
import { changedFiles, diffBytes, diffText, showFromOrigin, runCmd } from "./git.mjs";
|
|
4
|
-
import { createProvider } from "./provider.mjs";
|
|
5
|
-
import { withBudget, appendLedger, getQueueDir, ensureDir } from "./state.mjs";
|
|
4
|
+
import { createProvider, ProviderRateLimitError, ProviderUnavailableError } from "./provider.mjs";
|
|
5
|
+
import { withBudget, appendLedger, getQueueDir, ensureDir, rollbackBudgetReservation } from "./state.mjs";
|
|
6
6
|
import { sanitizeUntrustedData, buildAgentEnvelope } from "./prompt-guard.mjs";
|
|
7
7
|
import { recordVerifyRun, readVerifyRuns, flakyVerdict } from "./flaky-ledger.mjs";
|
|
8
8
|
import { readdirSync, readFileSync, renameSync, existsSync } from "node:fs";
|
|
@@ -346,6 +346,24 @@ export async function repair(failure, opts = {}) {
|
|
|
346
346
|
config.limits.dailyTasks
|
|
347
347
|
);
|
|
348
348
|
} catch (err) {
|
|
349
|
+
if (err instanceof ProviderRateLimitError || err instanceof ProviderUnavailableError) {
|
|
350
|
+
if (err.reservationId) {
|
|
351
|
+
rollbackBudgetReservation(root, err.reservationId);
|
|
352
|
+
}
|
|
353
|
+
const retryAfterMs = err.retryAfterMs || 60000;
|
|
354
|
+
const backoffSec = Math.ceil(retryAfterMs / 1000);
|
|
355
|
+
console.warn(`[PROVIDER_INFRASTRUCTURE_FAILURE] ${err.name}: ${err.message}. Recommended backoff: ${backoffSec}s.`);
|
|
356
|
+
attempts.push({ n, ok: false, error: err.message, providerError: true, retryAfterMs });
|
|
357
|
+
appendTelemetry(root, "ooda_repair_attempt", { attempt: n, ok: false, error: err.message, providerError: true });
|
|
358
|
+
return {
|
|
359
|
+
ok: false,
|
|
360
|
+
attempts,
|
|
361
|
+
finalStatus: "PROVIDER_INFRASTRUCTURE_FAILURE",
|
|
362
|
+
error: err.message,
|
|
363
|
+
retryAfterMs,
|
|
364
|
+
providerError: true,
|
|
365
|
+
};
|
|
366
|
+
}
|
|
349
367
|
attempts.push({ n, ok: false, error: err.message });
|
|
350
368
|
appendTelemetry(root, "ooda_repair_attempt", { attempt: n, ok: false, error: err.message });
|
|
351
369
|
break;
|
|
@@ -354,6 +372,16 @@ export async function repair(failure, opts = {}) {
|
|
|
354
372
|
attempts.push({ n, session, ok: true });
|
|
355
373
|
appendTelemetry(root, "ooda_repair_attempt", { attempt: n, ok: true });
|
|
356
374
|
|
|
375
|
+
// Poll async provider for terminal session state before executing re-verification gates
|
|
376
|
+
if (provider && session) {
|
|
377
|
+
await pollSessionState(provider, session, {
|
|
378
|
+
root,
|
|
379
|
+
dryRun: opts.dryRun,
|
|
380
|
+
pollIntervalMs: opts.pollIntervalMs,
|
|
381
|
+
maxPollAttempts: opts.maxPollAttempts,
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
|
|
357
385
|
// Re-verify after repair attempt
|
|
358
386
|
const gateRes = await gate({ root, config, fix: false, progressBus, progressToken });
|
|
359
387
|
if (gateRes.ok) {
|
|
@@ -379,6 +407,50 @@ export async function repair(failure, opts = {}) {
|
|
|
379
407
|
return { ok: false, attempts, finalStatus: "OODA_EXHAUSTED" };
|
|
380
408
|
}
|
|
381
409
|
|
|
410
|
+
/**
|
|
411
|
+
* Polls an async provider for terminal session state (COMPLETED / FAILED) before re-verification.
|
|
412
|
+
*/
|
|
413
|
+
export async function pollSessionState(provider, session, opts = {}) {
|
|
414
|
+
if (!session || !session.id) return { status: "COMPLETED" };
|
|
415
|
+
const initialStatus = String(session.status || session.state || "").toUpperCase();
|
|
416
|
+
if (initialStatus === "COMPLETED" || initialStatus === "FAILED" || opts.dryRun || session.id === "dry-run-session-id") {
|
|
417
|
+
return { status: initialStatus || "COMPLETED" };
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const maxAttempts = opts.maxPollAttempts || 30;
|
|
421
|
+
const pollIntervalMs = opts.pollIntervalMs || 1000;
|
|
422
|
+
const timeoutMs = opts.pollTimeoutMs || 300000;
|
|
423
|
+
const startTime = Date.now();
|
|
424
|
+
|
|
425
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
426
|
+
if (Date.now() - startTime > timeoutMs) break;
|
|
427
|
+
|
|
428
|
+
let currentSession = null;
|
|
429
|
+
if (provider && typeof provider.getSession === "function") {
|
|
430
|
+
try {
|
|
431
|
+
currentSession = await provider.getSession(session.id, opts);
|
|
432
|
+
} catch (_) {}
|
|
433
|
+
} else if (typeof opts.pollFn === "function") {
|
|
434
|
+
try {
|
|
435
|
+
currentSession = await opts.pollFn(session.id);
|
|
436
|
+
} catch (_) {}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
if (currentSession) {
|
|
440
|
+
const status = String(currentSession.status || currentSession.state || "").toUpperCase();
|
|
441
|
+
if (status === "COMPLETED" || status === "FAILED") {
|
|
442
|
+
return { ...currentSession, status };
|
|
443
|
+
}
|
|
444
|
+
} else {
|
|
445
|
+
break;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
return { ...session, status: String(session.status || "COMPLETED").toUpperCase() };
|
|
452
|
+
}
|
|
453
|
+
|
|
382
454
|
function buildRepairPrompt(failure, attempt, _config) {
|
|
383
455
|
const cleanStderr = redactSecrets(failure.stderr || failure.stdout || "Unknown Error");
|
|
384
456
|
|
|
@@ -428,39 +500,81 @@ export async function dispatch(task, opts = {}) {
|
|
|
428
500
|
|
|
429
501
|
const cleanTask = { ...task, prompt: envelopedPrompt };
|
|
430
502
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
503
|
+
try {
|
|
504
|
+
return await withBudget(
|
|
505
|
+
() => provider.dispatch(cleanTask, { root, dryRun: opts.dryRun }),
|
|
506
|
+
root,
|
|
507
|
+
config.limits.dailyTasks
|
|
508
|
+
);
|
|
509
|
+
} catch (err) {
|
|
510
|
+
if (err instanceof ProviderRateLimitError || err instanceof ProviderUnavailableError) {
|
|
511
|
+
if (err.reservationId) {
|
|
512
|
+
rollbackBudgetReservation(root, err.reservationId);
|
|
513
|
+
}
|
|
514
|
+
const retryAfterMs = err.retryAfterMs || 60000;
|
|
515
|
+
const status = err instanceof ProviderRateLimitError ? "RATE_LIMITED" : "PROVIDER_UNAVAILABLE";
|
|
516
|
+
const backoffSec = Math.ceil(retryAfterMs / 1000);
|
|
517
|
+
console.warn(`[PROVIDER_INFRASTRUCTURE_FAILURE] ${err.name}: ${err.message}. Recommended backoff: ${backoffSec}s.`);
|
|
518
|
+
return {
|
|
519
|
+
ok: false,
|
|
520
|
+
status,
|
|
521
|
+
error: err.message,
|
|
522
|
+
retryAfterMs,
|
|
523
|
+
providerError: true,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
throw err;
|
|
527
|
+
}
|
|
436
528
|
}
|
|
437
529
|
|
|
438
|
-
export async function run(opts = {}) {
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
530
|
+
export async function run(tasksOrOpts = {}, opts = {}) {
|
|
531
|
+
let tasks = null;
|
|
532
|
+
let options = opts;
|
|
533
|
+
if (Array.isArray(tasksOrOpts)) {
|
|
534
|
+
tasks = tasksOrOpts;
|
|
535
|
+
options = opts || {};
|
|
536
|
+
} else if (tasksOrOpts && typeof tasksOrOpts === "object") {
|
|
537
|
+
options = tasksOrOpts;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
const root = options.root || process.cwd();
|
|
541
|
+
const config = options.config || loadConfig(root);
|
|
542
|
+
const concurrency = options.concurrency || config.limits.concurrency || 1;
|
|
543
|
+
const isDry = options.dryRun !== undefined ? options.dryRun : (process.env.JULES_DRY_RUN === "true" || process.env.JULES_DRY_RUN === "1");
|
|
442
544
|
const queueDir = getQueueDir(root);
|
|
443
545
|
const completedDir = join(queueDir, "completed");
|
|
444
546
|
ensureDir(completedDir);
|
|
445
547
|
|
|
446
|
-
|
|
548
|
+
let filesToProcess = [];
|
|
549
|
+
if (Array.isArray(tasks)) {
|
|
550
|
+
filesToProcess = tasks.map((t) => (typeof t === "string" ? t : t.id || t.file || t.title));
|
|
551
|
+
} else {
|
|
552
|
+
filesToProcess = readdirSync(queueDir).filter((f) => isTaskFile(f, queueDir));
|
|
553
|
+
}
|
|
554
|
+
|
|
447
555
|
const results = [];
|
|
448
556
|
|
|
449
|
-
for (let i = 0; i <
|
|
450
|
-
const batch =
|
|
557
|
+
for (let i = 0; i < filesToProcess.length; i += concurrency) {
|
|
558
|
+
const batch = filesToProcess.slice(i, i + concurrency);
|
|
451
559
|
await Promise.all(
|
|
452
|
-
batch.map(async (
|
|
453
|
-
const
|
|
560
|
+
batch.map(async (fileOrItem) => {
|
|
561
|
+
const fileName = typeof fileOrItem === "string" ? fileOrItem : fileOrItem.id || fileOrItem.file;
|
|
562
|
+
const srcPath = join(queueDir, fileName);
|
|
454
563
|
try {
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
564
|
+
let prompt = typeof fileOrItem === "object" && fileOrItem.prompt ? fileOrItem.prompt : "";
|
|
565
|
+
if (!prompt && existsSync(srcPath)) {
|
|
566
|
+
prompt = readFileSync(srcPath, "utf-8");
|
|
567
|
+
}
|
|
568
|
+
const task = { title: fileName, prompt };
|
|
569
|
+
const session = await dispatch(task, { root, config, dryRun: isDry });
|
|
570
|
+
const dstPath = join(completedDir, fileName);
|
|
571
|
+
if (existsSync(srcPath)) {
|
|
572
|
+
renameSync(srcPath, dstPath);
|
|
573
|
+
}
|
|
574
|
+
appendLedger({ event: "task_completed", file: fileName, session }, root);
|
|
575
|
+
results.push({ file: fileName, ok: true, session });
|
|
462
576
|
} catch (err) {
|
|
463
|
-
results.push({ file, ok: false, error: err.message });
|
|
577
|
+
results.push({ file: fileName, ok: false, error: err.message });
|
|
464
578
|
}
|
|
465
579
|
})
|
|
466
580
|
);
|
package/src/envelope.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { git, resolveBase } from "./git.mjs";
|
|
1
|
+
import { git, runCmd, resolveBase } from "./git.mjs";
|
|
2
2
|
import { checkScope } from "./security.mjs";
|
|
3
3
|
import { normalizeScope } from "./config.mjs";
|
|
4
4
|
import { existsSync } from "node:fs";
|
|
@@ -70,8 +70,8 @@ export function validateEnvelope(envelope = {}, opts = {}) {
|
|
|
70
70
|
const absPath = join(root, relPath);
|
|
71
71
|
let existsInGit = false;
|
|
72
72
|
if (resolvedBase) {
|
|
73
|
-
const catRes =
|
|
74
|
-
existsInGit =
|
|
73
|
+
const catRes = runCmd(["git", "cat-file", "-e", `${resolvedBase}:${relPath}`], { cwd: root, ignoreError: true });
|
|
74
|
+
existsInGit = catRes.status === 0;
|
|
75
75
|
}
|
|
76
76
|
const existsOnDisk = existsSync(absPath);
|
|
77
77
|
if (!existsInGit && !existsOnDisk) {
|
package/src/flaky-ledger.mjs
CHANGED
package/src/git.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { execFileSync } from "node:child_process";
|
|
2
|
-
import {
|
|
2
|
+
import { normalizePath } from "./config.mjs";
|
|
3
3
|
import { journalIntent, journalDone } from "./journal.mjs";
|
|
4
4
|
|
|
5
5
|
export const NET_GUARD_PRELOAD_URL = new URL("./preload-net-guard.mjs", import.meta.url).href;
|
|
@@ -13,11 +13,6 @@ export class GateError extends Error {
|
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
function normalizePath(p) {
|
|
17
|
-
if (!p || typeof p !== "string") return "";
|
|
18
|
-
return p.split(sep).join("/").replace(/\\/g, "/");
|
|
19
|
-
}
|
|
20
|
-
|
|
21
16
|
const DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
|
|
22
17
|
const DEFAULT_MAX_BUFFER = 10 * 1024 * 1024; // 10 MB
|
|
23
18
|
|
package/src/mcp.mjs
CHANGED
package/src/merge-verify.mjs
CHANGED
|
@@ -31,14 +31,14 @@ export function mergeVerifyChain(mergedText, fileName, root = process.cwd()) {
|
|
|
31
31
|
tool = "python3 -m py_compile";
|
|
32
32
|
} else if (ext === ".ts" || ext === ".tsx" || ext === ".mts" || ext === ".cts") {
|
|
33
33
|
const hasTsConfig = existsSync(join(root, "tsconfig.json"));
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
const localTsc = join(root, "node_modules", ".bin", "tsc");
|
|
35
|
+
const hasLocalTsc = existsSync(localTsc);
|
|
36
|
+
if (hasTsConfig && hasLocalTsc) {
|
|
37
|
+
cmd = localTsc;
|
|
38
|
+
args = ["--noEmit", tempFilePath];
|
|
37
39
|
tool = "tsc --noEmit";
|
|
38
40
|
} else {
|
|
39
|
-
|
|
40
|
-
args = ["--check", tempFilePath];
|
|
41
|
-
tool = "node --check";
|
|
41
|
+
return { ok: true, tool: "ts-skipped-no-tsconfig" };
|
|
42
42
|
}
|
|
43
43
|
} else {
|
|
44
44
|
// JS, MJS, CJS or default
|
|
@@ -47,7 +47,7 @@ export function mergeVerifyChain(mergedText, fileName, root = process.cwd()) {
|
|
|
47
47
|
tool = "node --check";
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
const res = spawnSync(cmd, args, { encoding: "utf8" });
|
|
50
|
+
const res = spawnSync(cmd, args, { encoding: "utf8", timeout: 120000 });
|
|
51
51
|
const ok = res.status === 0;
|
|
52
52
|
|
|
53
53
|
return {
|
package/src/provider.mjs
CHANGED
|
@@ -31,6 +31,45 @@ export const CODEX_PRESET = {
|
|
|
31
31
|
promptViaStdin: false,
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
+
export class ProviderRateLimitError extends Error {
|
|
35
|
+
constructor(message, opts = {}) {
|
|
36
|
+
super(message);
|
|
37
|
+
this.name = "ProviderRateLimitError";
|
|
38
|
+
this.retryAfterMs = opts.retryAfterMs ?? 60000;
|
|
39
|
+
this.status = opts.status || 429;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export class ProviderUnavailableError extends Error {
|
|
44
|
+
constructor(message, opts = {}) {
|
|
45
|
+
super(message);
|
|
46
|
+
this.name = "ProviderUnavailableError";
|
|
47
|
+
this.status = opts.status || 503;
|
|
48
|
+
this.retryAfterMs = opts.retryAfterMs;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export class ProviderSchemaError extends Error {
|
|
53
|
+
constructor(message, opts = {}) {
|
|
54
|
+
super(message);
|
|
55
|
+
this.name = "ProviderSchemaError";
|
|
56
|
+
this.status = opts.status;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function parseRetryAfter(header) {
|
|
61
|
+
if (!header) return null;
|
|
62
|
+
const seconds = Number(header);
|
|
63
|
+
if (!isNaN(seconds)) {
|
|
64
|
+
return Math.max(0, Math.round(seconds * 1000));
|
|
65
|
+
}
|
|
66
|
+
const dateMs = Date.parse(header);
|
|
67
|
+
if (!isNaN(dateMs)) {
|
|
68
|
+
return Math.max(0, dateMs - Date.now());
|
|
69
|
+
}
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
|
|
34
73
|
function interpolateString(template, data) {
|
|
35
74
|
if (typeof template !== "string") return template;
|
|
36
75
|
return template.replace(/\{(\w+)\}/g, (_, key) => data[key] ?? "");
|
|
@@ -111,20 +150,63 @@ export function createProvider(spec = "jules", config = {}) {
|
|
|
111
150
|
body = JSON.stringify(data);
|
|
112
151
|
}
|
|
113
152
|
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
153
|
+
const timeoutMs = ctx.timeoutMs || config.timeoutMs || providerSpec.timeoutMs || 120_000;
|
|
154
|
+
let res;
|
|
155
|
+
try {
|
|
156
|
+
res = await fetch(url, {
|
|
157
|
+
method: providerSpec.method || "POST",
|
|
158
|
+
headers,
|
|
159
|
+
body,
|
|
160
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
161
|
+
});
|
|
162
|
+
} catch (err) {
|
|
163
|
+
if (err.name === "TimeoutError" || err.name === "AbortError" || err.code === "ABORT_ERR") {
|
|
164
|
+
throw new ProviderUnavailableError(`Provider HTTP Timeout (${timeoutMs}ms): ${err.message}`, {
|
|
165
|
+
status: 504,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
throw err;
|
|
169
|
+
}
|
|
119
170
|
|
|
120
171
|
if (!res.ok) {
|
|
121
172
|
const text = await res.text();
|
|
122
173
|
const cleanText = text.slice(0, 500);
|
|
123
174
|
const sanitizedText = rawToken ? cleanText.split(rawToken).join("[REDACTED]") : cleanText;
|
|
175
|
+
const retryAfterHeader = res.headers ? res.headers.get("retry-after") : null;
|
|
176
|
+
const retryAfterMs = parseRetryAfter(retryAfterHeader);
|
|
177
|
+
|
|
178
|
+
if (res.status === 429) {
|
|
179
|
+
throw new ProviderRateLimitError(`Provider HTTP Error (429): ${sanitizedText}`, {
|
|
180
|
+
retryAfterMs: retryAfterMs ?? 60000,
|
|
181
|
+
status: 429,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (res.status >= 500 && res.status < 600) {
|
|
186
|
+
throw new ProviderUnavailableError(`Provider HTTP Error (${res.status}): ${sanitizedText}`, {
|
|
187
|
+
status: res.status,
|
|
188
|
+
retryAfterMs: retryAfterMs ?? undefined,
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
|
|
124
192
|
throw new Error(`Provider HTTP Error (${res.status}): ${sanitizedText}`);
|
|
125
193
|
}
|
|
126
194
|
|
|
127
|
-
|
|
195
|
+
let json;
|
|
196
|
+
try {
|
|
197
|
+
json = await res.json();
|
|
198
|
+
} catch (err) {
|
|
199
|
+
throw new ProviderSchemaError(`Provider Payload Error: Invalid JSON response: ${err.message}`, {
|
|
200
|
+
status: res.status,
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (!json || typeof json !== "object") {
|
|
205
|
+
throw new ProviderSchemaError("Provider Payload Error: Expected JSON object response", {
|
|
206
|
+
status: res.status,
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
|
|
128
210
|
return {
|
|
129
211
|
id: json.id || json.name || "http-session",
|
|
130
212
|
status: json.state || "active",
|
package/src/state.mjs
CHANGED
|
@@ -189,19 +189,31 @@ export function checkDailyBudget(arg1 = resolveRoot(), arg2 = 300) {
|
|
|
189
189
|
const content = readFileSync(filePath, "utf-8");
|
|
190
190
|
const lines = content.split("\n").filter(Boolean);
|
|
191
191
|
let count = 0;
|
|
192
|
+
const activeIds = new Set();
|
|
192
193
|
for (const line of lines) {
|
|
193
194
|
try {
|
|
194
195
|
const entry = JSON.parse(line);
|
|
195
196
|
if (entry && entry.event === "budget_reserved") {
|
|
196
|
-
|
|
197
|
+
if (entry.reservationId) {
|
|
198
|
+
activeIds.add(entry.reservationId);
|
|
199
|
+
} else {
|
|
200
|
+
count++;
|
|
201
|
+
}
|
|
202
|
+
} else if (entry && (entry.event === "budget_rolled_back" || entry.event === "budget_released")) {
|
|
203
|
+
if (entry.reservationId) {
|
|
204
|
+
activeIds.delete(entry.reservationId);
|
|
205
|
+
} else {
|
|
206
|
+
count = Math.max(0, count - 1);
|
|
207
|
+
}
|
|
197
208
|
}
|
|
198
209
|
} catch (_) {}
|
|
199
210
|
}
|
|
211
|
+
const used = count + activeIds.size;
|
|
200
212
|
return {
|
|
201
|
-
ok:
|
|
202
|
-
used
|
|
213
|
+
ok: used < limit,
|
|
214
|
+
used,
|
|
203
215
|
budget: limit,
|
|
204
|
-
remaining: Math.max(0, limit -
|
|
216
|
+
remaining: Math.max(0, limit - used),
|
|
205
217
|
};
|
|
206
218
|
} catch (_) {
|
|
207
219
|
return { ok: true, used: 0, budget: limit, remaining: limit };
|
|
@@ -226,6 +238,7 @@ export function reserveBudgetAtomic(stateDirOrRoot = resolveRoot(), limit = 300,
|
|
|
226
238
|
const filePath = join(stateDir, `ledger-${dateStr}.jsonl`);
|
|
227
239
|
|
|
228
240
|
let count = 0;
|
|
241
|
+
const activeIds = new Set();
|
|
229
242
|
let prevHash = "0".repeat(64);
|
|
230
243
|
|
|
231
244
|
if (existsSync(filePath)) {
|
|
@@ -236,7 +249,17 @@ export function reserveBudgetAtomic(stateDirOrRoot = resolveRoot(), limit = 300,
|
|
|
236
249
|
try {
|
|
237
250
|
const entry = JSON.parse(line);
|
|
238
251
|
if (entry && entry.event === "budget_reserved") {
|
|
239
|
-
|
|
252
|
+
if (entry.reservationId) {
|
|
253
|
+
activeIds.add(entry.reservationId);
|
|
254
|
+
} else {
|
|
255
|
+
count++;
|
|
256
|
+
}
|
|
257
|
+
} else if (entry && (entry.event === "budget_rolled_back" || entry.event === "budget_released")) {
|
|
258
|
+
if (entry.reservationId) {
|
|
259
|
+
activeIds.delete(entry.reservationId);
|
|
260
|
+
} else {
|
|
261
|
+
count = Math.max(0, count - 1);
|
|
262
|
+
}
|
|
240
263
|
}
|
|
241
264
|
if (entry && entry.hash) {
|
|
242
265
|
prevHash = entry.hash;
|
|
@@ -246,8 +269,9 @@ export function reserveBudgetAtomic(stateDirOrRoot = resolveRoot(), limit = 300,
|
|
|
246
269
|
} catch (_) {}
|
|
247
270
|
}
|
|
248
271
|
|
|
249
|
-
|
|
250
|
-
|
|
272
|
+
const used = count + activeIds.size;
|
|
273
|
+
if (used >= limit) {
|
|
274
|
+
throw new BudgetError(`Daily budget exhausted (${used}/${limit} tasks executed)`);
|
|
251
275
|
}
|
|
252
276
|
|
|
253
277
|
const timestamp = new Date().toISOString();
|
|
@@ -267,8 +291,8 @@ export function reserveBudgetAtomic(stateDirOrRoot = resolveRoot(), limit = 300,
|
|
|
267
291
|
return {
|
|
268
292
|
ok: true,
|
|
269
293
|
reservationId,
|
|
270
|
-
remaining: Math.max(0, limit - (
|
|
271
|
-
used:
|
|
294
|
+
remaining: Math.max(0, limit - (used + 1)),
|
|
295
|
+
used: used + 1,
|
|
272
296
|
};
|
|
273
297
|
}, opts);
|
|
274
298
|
}
|
|
@@ -282,6 +306,11 @@ export function commitBudgetReservation(rootOrOpts = resolveRoot(), reservationI
|
|
|
282
306
|
return appendLedger({ event: "budget_committed", reservationId }, root);
|
|
283
307
|
}
|
|
284
308
|
|
|
309
|
+
export function rollbackBudgetReservation(rootOrOpts = resolveRoot(), reservationId = "") {
|
|
310
|
+
const root = typeof rootOrOpts === "string" ? rootOrOpts : resolveRoot();
|
|
311
|
+
return appendLedger({ event: "budget_rolled_back", reservationId }, root);
|
|
312
|
+
}
|
|
313
|
+
|
|
285
314
|
export async function withBudget(fn, root = resolveRoot(), limit = 300) {
|
|
286
315
|
const reservation = reserveBudget(root, limit);
|
|
287
316
|
try {
|
|
@@ -289,6 +318,9 @@ export async function withBudget(fn, root = resolveRoot(), limit = 300) {
|
|
|
289
318
|
commitBudgetReservation(root, reservation.reservationId);
|
|
290
319
|
return result;
|
|
291
320
|
} catch (err) {
|
|
321
|
+
if (err && typeof err === "object") {
|
|
322
|
+
err.reservationId = reservation.reservationId;
|
|
323
|
+
}
|
|
292
324
|
appendLedger({ event: "budget_reservation_failed", reservationId: reservation.reservationId, error: err.message }, root);
|
|
293
325
|
throw err;
|
|
294
326
|
}
|
|
@@ -349,8 +381,9 @@ export function isPidAlive(pid, expectedStartTime = null) {
|
|
|
349
381
|
return true;
|
|
350
382
|
}
|
|
351
383
|
|
|
352
|
-
export function acquireLock(agentName, taskId, files = [], rootOrOpts = resolveRoot()) {
|
|
353
|
-
const root = typeof rootOrOpts === "string" ? rootOrOpts : resolveRoot();
|
|
384
|
+
export function acquireLock(agentName, taskId, files = [], rootOrOpts = resolveRoot(), opts = {}) {
|
|
385
|
+
const root = typeof rootOrOpts === "string" ? rootOrOpts : (rootOrOpts?.root || resolveRoot());
|
|
386
|
+
const branch = (typeof rootOrOpts === "object" && rootOrOpts?.branch) || opts?.branch || process.env.JULES_BRANCH || process.env.BRANCH_NAME || "";
|
|
354
387
|
const lockDir = getLockDir(root);
|
|
355
388
|
const lockFile = join(lockDir, `${taskId}.json`);
|
|
356
389
|
|
|
@@ -375,6 +408,7 @@ export function acquireLock(agentName, taskId, files = [], rootOrOpts = resolveR
|
|
|
375
408
|
const payload = {
|
|
376
409
|
agent: agentName,
|
|
377
410
|
taskId,
|
|
411
|
+
branch,
|
|
378
412
|
files,
|
|
379
413
|
pid: process.pid,
|
|
380
414
|
processStartTime: startTime,
|
package/src/telemetry.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
openSync,
|
|
3
3
|
writeSync,
|
|
4
|
+
readSync,
|
|
4
5
|
closeSync,
|
|
5
6
|
readFileSync,
|
|
6
7
|
existsSync,
|
|
@@ -16,6 +17,37 @@ import { safeAtomicWrite } from "./security.mjs";
|
|
|
16
17
|
export const MAX_TELEMETRY_SEGMENT_BYTES = 8 * 1024 * 1024; // 8 MB log segment ceiling
|
|
17
18
|
export const TELEMETRY_GENESIS_HASH = "0".repeat(64);
|
|
18
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Reads up to 64 KB from EOF of target log segment to extract true tail hash.
|
|
22
|
+
*/
|
|
23
|
+
function readTailHash(filePath, bufferSize = 64 * 1024) {
|
|
24
|
+
if (!existsSync(filePath)) return null;
|
|
25
|
+
try {
|
|
26
|
+
const stat = statSync(filePath);
|
|
27
|
+
if (stat.size === 0) return null;
|
|
28
|
+
const readSize = Math.min(stat.size, bufferSize);
|
|
29
|
+
const position = stat.size - readSize;
|
|
30
|
+
const fd = openSync(filePath, "r");
|
|
31
|
+
const buf = Buffer.alloc(readSize);
|
|
32
|
+
try {
|
|
33
|
+
readSync(fd, buf, 0, readSize, position);
|
|
34
|
+
} finally {
|
|
35
|
+
closeSync(fd);
|
|
36
|
+
}
|
|
37
|
+
const content = buf.toString("utf-8");
|
|
38
|
+
const lines = content.split("\n").filter(Boolean);
|
|
39
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
40
|
+
try {
|
|
41
|
+
const obj = JSON.parse(lines[i]);
|
|
42
|
+
if (obj && typeof obj.hash === "string") {
|
|
43
|
+
return obj.hash;
|
|
44
|
+
}
|
|
45
|
+
} catch (_) {}
|
|
46
|
+
}
|
|
47
|
+
} catch (_) {}
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
|
|
19
51
|
/**
|
|
20
52
|
* Returns telemetry file paths for a given date string sorted by segment index.
|
|
21
53
|
*/
|
|
@@ -50,18 +82,10 @@ function coldScanTelemetry(stateDir, dateStr) {
|
|
|
50
82
|
for (let i = segments.length - 1; i >= 0; i--) {
|
|
51
83
|
const seg = segments[i];
|
|
52
84
|
if (existsSync(seg.path)) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
try {
|
|
58
|
-
const entry = JSON.parse(lines[j]);
|
|
59
|
-
if (entry && typeof entry.hash === "string") {
|
|
60
|
-
return { hash: entry.hash, segment: seg.index };
|
|
61
|
-
}
|
|
62
|
-
} catch (_) {}
|
|
63
|
-
}
|
|
64
|
-
} catch (_) {}
|
|
85
|
+
const tailHash = readTailHash(seg.path);
|
|
86
|
+
if (tailHash) {
|
|
87
|
+
return { hash: tailHash, segment: seg.index };
|
|
88
|
+
}
|
|
65
89
|
}
|
|
66
90
|
}
|
|
67
91
|
|
|
@@ -70,7 +94,7 @@ function coldScanTelemetry(stateDir, dateStr) {
|
|
|
70
94
|
|
|
71
95
|
/**
|
|
72
96
|
* Appends a structured telemetry event to .agent/state/telemetry-<date>.jsonl with SHA-256 hash chaining.
|
|
73
|
-
* Uses .head atomic cache for O(1) constant-time appends.
|
|
97
|
+
* Uses .head atomic cache with true tail reconciliation for O(1) constant-time appends.
|
|
74
98
|
*
|
|
75
99
|
* @param {string} [rootOrOpts]
|
|
76
100
|
* @param {string} kind
|
|
@@ -102,18 +126,30 @@ export function appendTelemetry(rootOrOpts = resolveRoot(), kind = "event", fiel
|
|
|
102
126
|
} catch (_) {}
|
|
103
127
|
}
|
|
104
128
|
|
|
105
|
-
if (!headValid) {
|
|
106
|
-
const recovered = coldScanTelemetry(stateDir, dateStr);
|
|
107
|
-
prevHash = recovered.hash;
|
|
108
|
-
activeSegmentIndex = recovered.segment;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
129
|
let activeFileName =
|
|
112
130
|
activeSegmentIndex === 0
|
|
113
131
|
? `telemetry-${dateStr}.jsonl`
|
|
114
132
|
: `telemetry-${dateStr}-${activeSegmentIndex}.jsonl`;
|
|
115
133
|
let activeFilePath = join(stateDir, activeFileName);
|
|
116
134
|
|
|
135
|
+
// Reconcile .head hash against true log tail (reading last 64 KB from EOF)
|
|
136
|
+
const tailHash = readTailHash(activeFilePath);
|
|
137
|
+
if (tailHash) {
|
|
138
|
+
if (tailHash !== prevHash) {
|
|
139
|
+
prevHash = tailHash;
|
|
140
|
+
}
|
|
141
|
+
headValid = true;
|
|
142
|
+
} else if (!headValid) {
|
|
143
|
+
const recovered = coldScanTelemetry(stateDir, dateStr);
|
|
144
|
+
prevHash = recovered.hash;
|
|
145
|
+
activeSegmentIndex = recovered.segment;
|
|
146
|
+
activeFileName =
|
|
147
|
+
activeSegmentIndex === 0
|
|
148
|
+
? `telemetry-${dateStr}.jsonl`
|
|
149
|
+
: `telemetry-${dateStr}-${activeSegmentIndex}.jsonl`;
|
|
150
|
+
activeFilePath = join(stateDir, activeFileName);
|
|
151
|
+
}
|
|
152
|
+
|
|
117
153
|
if (existsSync(activeFilePath)) {
|
|
118
154
|
try {
|
|
119
155
|
const stat = statSync(activeFilePath);
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
# Jules Task Queue Directory
|
|
2
|
-
|
|
3
|
-
Drop markdown task specification files here (e.g. `TASK-001-feature-name.md`) to queue tasks for background execution by Google Jules.
|
|
4
|
-
|
|
5
|
-
### File Format Example
|
|
6
|
-
|
|
7
|
-
```markdown
|
|
8
|
-
# TASK-001: Implement User Rate Limiting
|
|
9
|
-
|
|
10
|
-
## Objective
|
|
11
|
-
Implement sliding window rate limiting for public API routes.
|
|
12
|
-
|
|
13
|
-
## Execution Rules
|
|
14
|
-
- Use Redis / memory store for tracking hit counts.
|
|
15
|
-
- Must return HTTP 429 Too Many Requests when limit is exceeded.
|
|
16
|
-
- Must pass `npm test` before submitting PR.
|
|
17
|
-
```
|
|
18
|
-
|
|
19
|
-
Process all queued tasks in batch:
|
|
20
|
-
|
|
21
|
-
```bash
|
|
22
|
-
agentctl queue
|
|
23
|
-
# or npm run jules:queue
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
Or dispatch a single task using `agentctl dispatch`:
|
|
27
|
-
|
|
28
|
-
```bash
|
|
29
|
-
agentctl dispatch --title "TASK-001 Rate Limiting" --prompt "$(cat .agent/jules-queue/TASK-001-rate-limiting.md)"
|
|
30
|
-
```
|