@phnx-labs/agents-cli 1.20.51 → 1.20.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/README.md +12 -0
- package/dist/bin/agents +0 -0
- package/dist/commands/browser.js +215 -7
- package/dist/commands/cloud.d.ts +3 -0
- package/dist/commands/cloud.js +8 -1
- package/dist/commands/events.d.ts +1 -1
- package/dist/commands/events.js +2 -3
- package/dist/commands/exec.js +82 -3
- package/dist/commands/factory.js +8 -0
- package/dist/commands/feed.d.ts +38 -0
- package/dist/commands/feed.js +274 -0
- package/dist/commands/logs.d.ts +5 -1
- package/dist/commands/logs.js +248 -3
- package/dist/commands/mcp.js +7 -0
- package/dist/commands/memory.d.ts +9 -0
- package/dist/commands/memory.js +164 -0
- package/dist/commands/message.d.ts +11 -6
- package/dist/commands/message.js +140 -5
- package/dist/commands/routines.js +12 -0
- package/dist/commands/secrets-migrate.d.ts +2 -1
- package/dist/commands/secrets-migrate.js +88 -13
- package/dist/commands/secrets.d.ts +22 -0
- package/dist/commands/secrets.js +176 -42
- package/dist/commands/sessions.js +10 -1
- package/dist/commands/teams.js +4 -0
- package/dist/commands/worktree.js +4 -2
- package/dist/index.js +21 -22
- package/dist/lib/agents.js +249 -17
- package/dist/lib/answer-router.d.ts +75 -0
- package/dist/lib/answer-router.js +149 -0
- package/dist/lib/ask-classifier.d.ts +71 -0
- package/dist/lib/ask-classifier.js +197 -0
- package/dist/lib/browser/login-detection.d.ts +94 -0
- package/dist/lib/browser/login-detection.js +274 -0
- package/dist/lib/browser/profiles.d.ts +17 -8
- package/dist/lib/browser/profiles.js +27 -8
- package/dist/lib/browser/secret-ref.d.ts +10 -0
- package/dist/lib/browser/secret-ref.js +14 -0
- package/dist/lib/browser/service.js +14 -12
- package/dist/lib/cloud/antigravity.d.ts +0 -2
- package/dist/lib/cloud/antigravity.js +2 -17
- package/dist/lib/cloud/codex.js +3 -18
- package/dist/lib/cloud/rush.d.ts +15 -0
- package/dist/lib/cloud/rush.js +10 -16
- package/dist/lib/cloud/stream.js +2 -0
- package/dist/lib/cloud/types.d.ts +21 -0
- package/dist/lib/cloud/types.js +81 -0
- package/dist/lib/crabbox/cli.d.ts +1 -1
- package/dist/lib/crabbox/cli.js +12 -2
- package/dist/lib/crabbox/lease.d.ts +19 -0
- package/dist/lib/crabbox/lease.js +22 -11
- package/dist/lib/crabbox/progress.d.ts +62 -0
- package/dist/lib/crabbox/progress.js +129 -0
- package/dist/lib/crabbox/runtimes.d.ts +38 -1
- package/dist/lib/crabbox/runtimes.js +98 -5
- package/dist/lib/daemon.d.ts +12 -9
- package/dist/lib/daemon.js +32 -17
- package/dist/lib/events.d.ts +31 -5
- package/dist/lib/events.js +291 -101
- package/dist/lib/exec.js +20 -1
- package/dist/lib/feed-outcome.d.ts +101 -0
- package/dist/lib/feed-outcome.js +244 -0
- package/dist/lib/feed-policy.d.ts +30 -0
- package/dist/lib/feed-policy.js +133 -0
- package/dist/lib/feed.d.ts +180 -0
- package/dist/lib/feed.js +627 -0
- package/dist/lib/git.d.ts +17 -1
- package/dist/lib/git.js +20 -1
- package/dist/lib/hooks.js +529 -14
- package/dist/lib/hosts/passthrough.d.ts +3 -3
- package/dist/lib/hosts/passthrough.js +3 -3
- package/dist/lib/mailbox-gc.d.ts +22 -0
- package/dist/lib/mailbox-gc.js +161 -0
- package/dist/lib/mailbox.d.ts +26 -2
- package/dist/lib/mailbox.js +80 -5
- package/dist/lib/mcp.js +82 -0
- package/dist/lib/memory.d.ts +55 -0
- package/dist/lib/memory.js +274 -0
- package/dist/lib/notify.d.ts +16 -0
- package/dist/lib/notify.js +61 -0
- package/dist/lib/operator.d.ts +26 -0
- package/dist/lib/operator.js +107 -0
- package/dist/lib/plugins.d.ts +35 -0
- package/dist/lib/plugins.js +217 -0
- package/dist/lib/remote-agents-json.d.ts +14 -0
- package/dist/lib/remote-agents-json.js +94 -0
- package/dist/lib/resources/mcp.js +44 -0
- package/dist/lib/resources/memory.d.ts +15 -0
- package/dist/lib/resources/memory.js +46 -0
- package/dist/lib/resources/types.d.ts +2 -2
- package/dist/lib/rotate.js +2 -0
- package/dist/lib/runner.d.ts +43 -0
- package/dist/lib/runner.js +323 -74
- package/dist/lib/sandbox.js +6 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/CodeResources +0 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/MacOS/Agents CLI +0 -0
- package/dist/lib/secrets/agent.d.ts +21 -0
- package/dist/lib/secrets/agent.js +63 -1
- package/dist/lib/secrets/bundles.d.ts +33 -1
- package/dist/lib/secrets/bundles.js +76 -21
- package/dist/lib/secrets/icloud-import.d.ts +79 -0
- package/dist/lib/secrets/icloud-import.js +203 -0
- package/dist/lib/secrets/index.d.ts +142 -2
- package/dist/lib/secrets/index.js +695 -30
- package/dist/lib/secrets/remote.js +1 -1
- package/dist/lib/secrets/sync.js +1 -1
- package/dist/lib/session/active.d.ts +18 -0
- package/dist/lib/session/active.js +47 -17
- package/dist/lib/session/db.d.ts +9 -1
- package/dist/lib/session/db.js +18 -3
- package/dist/lib/session/discover.d.ts +13 -0
- package/dist/lib/session/discover.js +32 -2
- package/dist/lib/session/parse.d.ts +8 -0
- package/dist/lib/session/parse.js +42 -21
- package/dist/lib/session/remote-active.js +8 -89
- package/dist/lib/session/state.d.ts +11 -0
- package/dist/lib/session/state.js +50 -1
- package/dist/lib/session/tail.d.ts +23 -4
- package/dist/lib/session/tail.js +34 -16
- package/dist/lib/session/throughput.d.ts +30 -0
- package/dist/lib/session/throughput.js +86 -0
- package/dist/lib/shim-heal.d.ts +12 -3
- package/dist/lib/shim-heal.js +12 -6
- package/dist/lib/staleness/detectors/subagents.js +57 -3
- package/dist/lib/staleness/writers/hooks.js +7 -3
- package/dist/lib/staleness/writers/subagents.js +37 -6
- package/dist/lib/startup/command-registry.d.ts +2 -0
- package/dist/lib/startup/command-registry.js +4 -0
- package/dist/lib/state.d.ts +2 -0
- package/dist/lib/state.js +25 -8
- package/dist/lib/subagents.d.ts +52 -0
- package/dist/lib/subagents.js +315 -12
- package/dist/lib/teams/agents.js +6 -3
- package/dist/lib/teams/worktree.d.ts +8 -0
- package/dist/lib/teams/worktree.js +8 -0
- package/dist/lib/types.d.ts +20 -2
- package/dist/lib/versions.js +38 -48
- package/dist/lib/whats-new.d.ts +5 -3
- package/dist/lib/whats-new.js +25 -5
- package/package.json +4 -3
- package/scripts/postinstall.js +61 -1
package/dist/lib/runner.js
CHANGED
|
@@ -5,6 +5,13 @@
|
|
|
5
5
|
* sandboxed or unsandboxed environments, captures stdout to log files,
|
|
6
6
|
* enforces timeouts, and extracts the final assistant report from the
|
|
7
7
|
* agent's stream-JSON output.
|
|
8
|
+
*
|
|
9
|
+
* Version/account selection mirrors `agents run`: when a routine does not pin
|
|
10
|
+
* `version:`, the runner uses the configured run strategy (default `balanced`)
|
|
11
|
+
* to pick a healthy install, pins the absolute binary via `getBinaryPath`, and
|
|
12
|
+
* arms same-agent failover across other healthy accounts when a rate/usage
|
|
13
|
+
* limit is detected mid-run (foreground `executeJob` only — detached daemon
|
|
14
|
+
* fires once with the pre-flight pick).
|
|
8
15
|
*/
|
|
9
16
|
import { spawn } from 'child_process';
|
|
10
17
|
import * as fs from 'fs';
|
|
@@ -15,8 +22,10 @@ import { getRunsDir } from './state.js';
|
|
|
15
22
|
import { prepareJobHome, buildSpawnEnv } from './sandbox.js';
|
|
16
23
|
import { resolveModel, buildReasoningFlags } from './models.js';
|
|
17
24
|
import { createTimer, maybeRotate, redactPrompt } from './events.js';
|
|
18
|
-
import { normalizeMode } from './exec.js';
|
|
25
|
+
import { normalizeMode, buildExecEnv, detectRateLimit, } from './exec.js';
|
|
19
26
|
import { backgroundSpawnOptions } from './platform/process.js';
|
|
27
|
+
import { getBinaryPath, isVersionInstalled, resolveVersion } from './versions.js';
|
|
28
|
+
import { getConfiguredRunStrategy, resolveRunVersion, rotationFailoverChain, readinessFromCandidate, } from './rotate.js';
|
|
20
29
|
/** CLI command templates per agent, with {prompt} as a placeholder. */
|
|
21
30
|
const AGENT_COMMANDS = {
|
|
22
31
|
claude: ['claude', '-p', '--verbose', '{prompt}', '--output-format', 'stream-json', '--permission-mode', 'plan'],
|
|
@@ -161,6 +170,201 @@ function appendModelAndReasoning(cmd, config) {
|
|
|
161
170
|
function generateRunId() {
|
|
162
171
|
return new Date().toISOString().replace(/[:.]/g, '-');
|
|
163
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Resolve the version/account chain for a routine the same way `agents run`
|
|
175
|
+
* does: honor an explicit `version:` pin; otherwise use the configured run
|
|
176
|
+
* strategy (default `balanced`) so credit-exhausted / rate-limited accounts
|
|
177
|
+
* are skipped pre-flight, and synthesize a same-agent failover chain from the
|
|
178
|
+
* other healthy accounts for mid-run rate limits.
|
|
179
|
+
*
|
|
180
|
+
* Workflows are left alone — `agents run <workflow>` owns selection.
|
|
181
|
+
*/
|
|
182
|
+
export async function resolveRoutineLaunch(config, cwd = process.cwd()) {
|
|
183
|
+
if (config.workflow) {
|
|
184
|
+
return { chain: [], rotation: null, pinned: false };
|
|
185
|
+
}
|
|
186
|
+
const agent = config.agent;
|
|
187
|
+
if (config.version) {
|
|
188
|
+
const version = config.version;
|
|
189
|
+
if (!isVersionInstalled(agent, version)) {
|
|
190
|
+
process.stderr.write(`[agents] routine ${config.name}: pinned ${agent}@${version} is not installed\n`);
|
|
191
|
+
}
|
|
192
|
+
return {
|
|
193
|
+
chain: [{ agent, version }],
|
|
194
|
+
rotation: null,
|
|
195
|
+
pinned: true,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
const strategy = getConfiguredRunStrategy(agent, cwd);
|
|
199
|
+
let version;
|
|
200
|
+
let rotation = null;
|
|
201
|
+
try {
|
|
202
|
+
const resolved = await resolveRunVersion(agent, strategy, cwd);
|
|
203
|
+
version = resolved.version ?? undefined;
|
|
204
|
+
rotation = resolved.rotation;
|
|
205
|
+
if (rotation) {
|
|
206
|
+
const label = rotation.picked.email
|
|
207
|
+
? `${rotation.picked.email} · ${agent}@${rotation.picked.version}`
|
|
208
|
+
: `${agent}@${rotation.picked.version}`;
|
|
209
|
+
const ratio = `${rotation.healthy.length} of ${rotation.healthy.length + rotation.excluded.length} healthy`;
|
|
210
|
+
process.stderr.write(`[agents] routine ${config.name}: ${strategy} picked ${label} (${ratio})\n`);
|
|
211
|
+
if (rotation.excluded.length > 0) {
|
|
212
|
+
const reasons = rotation.excluded
|
|
213
|
+
.map((c) => {
|
|
214
|
+
const r = readinessFromCandidate(c);
|
|
215
|
+
const why = r.ready ? 'deduped' : r.reason;
|
|
216
|
+
return `${c.agent}@${c.version}=${why}`;
|
|
217
|
+
})
|
|
218
|
+
.join(', ');
|
|
219
|
+
process.stderr.write(`[agents] routine ${config.name}: skipped ${reasons}\n`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
else if (!version) {
|
|
223
|
+
process.stderr.write(`[agents] routine ${config.name}: strategy ${strategy} found no usable ${agent} version; ` +
|
|
224
|
+
`falling back to default pin\n`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
catch (err) {
|
|
228
|
+
process.stderr.write(`[agents] routine ${config.name}: strategy ${strategy} skipped: ${err.message}\n`);
|
|
229
|
+
}
|
|
230
|
+
if (!version) {
|
|
231
|
+
version = resolveVersion(agent, cwd) ?? undefined;
|
|
232
|
+
}
|
|
233
|
+
if (!version) {
|
|
234
|
+
process.stderr.write(`[agents] routine ${config.name}: no version of ${agent} configured — ` +
|
|
235
|
+
`run: agents add ${agent}@<version> && agents use ${agent} <version>\n`);
|
|
236
|
+
return { chain: [{ agent }], rotation: null, pinned: false };
|
|
237
|
+
}
|
|
238
|
+
const failover = rotationFailoverChain(rotation, version);
|
|
239
|
+
if (failover.length > 0) {
|
|
240
|
+
const labels = failover.map((f) => `${f.agent}@${f.version}`).join(', ');
|
|
241
|
+
process.stderr.write(`[agents] routine ${config.name}: credit/rate-limit failover armed → ${labels}\n`);
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
chain: [{ agent, version }, ...failover],
|
|
245
|
+
rotation,
|
|
246
|
+
pinned: false,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Rewrite `cmd[0]` to the absolute binary for `agent@version` when installed.
|
|
251
|
+
* Bypasses the bare-name shim so a sandboxed HOME / missing default pin cannot
|
|
252
|
+
* surface as "agents: no version of X configured".
|
|
253
|
+
*/
|
|
254
|
+
export function pinJobBinary(cmd, agent, version) {
|
|
255
|
+
if (!version || cmd.length === 0)
|
|
256
|
+
return cmd;
|
|
257
|
+
if (!isVersionInstalled(agent, version))
|
|
258
|
+
return cmd;
|
|
259
|
+
const binary = getBinaryPath(agent, version);
|
|
260
|
+
if (!binary || !fs.existsSync(binary))
|
|
261
|
+
return cmd;
|
|
262
|
+
const next = [...cmd];
|
|
263
|
+
next[0] = binary;
|
|
264
|
+
return next;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Merge sandbox/base env with the canonical per-version exec env
|
|
268
|
+
* (CLAUDE_CONFIG_DIR / CODEX_HOME / …) so routines share account isolation
|
|
269
|
+
* with `agents run`.
|
|
270
|
+
*/
|
|
271
|
+
export function buildRoutineSpawnEnv(baseEnv, agent, version, timezone) {
|
|
272
|
+
const execEnv = buildExecEnv({
|
|
273
|
+
agent,
|
|
274
|
+
version,
|
|
275
|
+
mode: 'plan',
|
|
276
|
+
effort: 'auto',
|
|
277
|
+
headless: true,
|
|
278
|
+
env: baseEnv,
|
|
279
|
+
});
|
|
280
|
+
const out = {};
|
|
281
|
+
for (const [k, v] of Object.entries(execEnv)) {
|
|
282
|
+
if (v !== undefined)
|
|
283
|
+
out[k] = v;
|
|
284
|
+
}
|
|
285
|
+
if (timezone)
|
|
286
|
+
out.TZ = timezone;
|
|
287
|
+
return out;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Spawn one attempt, capture logs to `attemptLogPath`, enforce timeout.
|
|
291
|
+
* Rate-limit scanning uses only this attempt's log (not prior failover output).
|
|
292
|
+
* The attempt log is also appended into `combinedLogPath` for a continuous trail.
|
|
293
|
+
*/
|
|
294
|
+
function spawnJobAttempt(cmd, env, attemptLogPath, timeoutMs, combinedLogPath) {
|
|
295
|
+
// Isolate this attempt's output so detectRateLimit never sees prior attempts.
|
|
296
|
+
fs.writeFileSync(attemptLogPath, '', { mode: 0o600 });
|
|
297
|
+
const stdoutFd = fs.openSync(attemptLogPath, 'a', 0o600);
|
|
298
|
+
return new Promise((resolve) => {
|
|
299
|
+
const child = spawn(cmd[0], cmd.slice(1), {
|
|
300
|
+
stdio: ['ignore', stdoutFd, stdoutFd],
|
|
301
|
+
...backgroundSpawnOptions({ fdStdio: true }),
|
|
302
|
+
env,
|
|
303
|
+
});
|
|
304
|
+
let settled = false;
|
|
305
|
+
const finish = (result) => {
|
|
306
|
+
if (settled)
|
|
307
|
+
return;
|
|
308
|
+
settled = true;
|
|
309
|
+
try {
|
|
310
|
+
fs.closeSync(stdoutFd);
|
|
311
|
+
}
|
|
312
|
+
catch { /* fd already closed */ }
|
|
313
|
+
let logText = '';
|
|
314
|
+
try {
|
|
315
|
+
logText = fs.readFileSync(attemptLogPath, 'utf-8');
|
|
316
|
+
}
|
|
317
|
+
catch { /* missing log */ }
|
|
318
|
+
if (combinedLogPath) {
|
|
319
|
+
try {
|
|
320
|
+
fs.appendFileSync(combinedLogPath, logText);
|
|
321
|
+
}
|
|
322
|
+
catch { /* best-effort trail */ }
|
|
323
|
+
}
|
|
324
|
+
resolve({ ...result, logText });
|
|
325
|
+
};
|
|
326
|
+
const timeoutTimer = setTimeout(() => {
|
|
327
|
+
try {
|
|
328
|
+
if (child.pid)
|
|
329
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
330
|
+
}
|
|
331
|
+
catch { /* process already exited */ }
|
|
332
|
+
setTimeout(() => {
|
|
333
|
+
try {
|
|
334
|
+
if (child.pid)
|
|
335
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
336
|
+
}
|
|
337
|
+
catch { /* process already exited */ }
|
|
338
|
+
}, 5000);
|
|
339
|
+
finish({
|
|
340
|
+
exitCode: null,
|
|
341
|
+
status: 'timeout',
|
|
342
|
+
pid: child.pid || null,
|
|
343
|
+
logText: '',
|
|
344
|
+
});
|
|
345
|
+
}, timeoutMs);
|
|
346
|
+
child.on('exit', (code) => {
|
|
347
|
+
clearTimeout(timeoutTimer);
|
|
348
|
+
finish({
|
|
349
|
+
exitCode: code,
|
|
350
|
+
status: code === 0 ? 'completed' : 'failed',
|
|
351
|
+
pid: child.pid || null,
|
|
352
|
+
logText: '',
|
|
353
|
+
});
|
|
354
|
+
});
|
|
355
|
+
child.on('error', (err) => {
|
|
356
|
+
clearTimeout(timeoutTimer);
|
|
357
|
+
finish({
|
|
358
|
+
exitCode: 1,
|
|
359
|
+
status: 'failed',
|
|
360
|
+
error: err.message,
|
|
361
|
+
pid: child.pid || null,
|
|
362
|
+
logText: '',
|
|
363
|
+
});
|
|
364
|
+
});
|
|
365
|
+
child.unref();
|
|
366
|
+
});
|
|
367
|
+
}
|
|
164
368
|
/**
|
|
165
369
|
* Execute a job synchronously (waits for completion or timeout before resolving).
|
|
166
370
|
*
|
|
@@ -169,12 +373,17 @@ function generateRunId() {
|
|
|
169
373
|
* workflow `loop:` blocks (issue #400). The optional `deps` parameter provides
|
|
170
374
|
* injectable seams (runIteration, sleep, writeCheckpoint) used by tests; production
|
|
171
375
|
* callers omit it and get the defaults.
|
|
376
|
+
*
|
|
377
|
+
* Single-shot path: pre-flight version/account selection + mid-run rate-limit
|
|
378
|
+
* failover across healthy same-agent accounts (RUSH-1016).
|
|
172
379
|
*/
|
|
173
380
|
export async function executeJob(config, deps) {
|
|
174
381
|
maybeRotate();
|
|
382
|
+
const launch = await resolveRoutineLaunch(config);
|
|
383
|
+
const primaryVersion = launch.chain[0]?.version ?? config.version;
|
|
175
384
|
const timer = createTimer('agent.run', {
|
|
176
385
|
agent: config.agent,
|
|
177
|
-
version:
|
|
386
|
+
version: primaryVersion,
|
|
178
387
|
jobName: config.name,
|
|
179
388
|
mode: config.mode,
|
|
180
389
|
...redactPrompt(config.prompt),
|
|
@@ -186,10 +395,9 @@ export async function executeJob(config, deps) {
|
|
|
186
395
|
const runId = generateRunId();
|
|
187
396
|
const runDir = getRunDir(config.name, runId);
|
|
188
397
|
fs.mkdirSync(runDir, { recursive: true });
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
}
|
|
398
|
+
const baseEnv = useSandbox
|
|
399
|
+
? buildSpawnEnv(overlayHome)
|
|
400
|
+
: { ...process.env };
|
|
193
401
|
// Workflows run via `agents run <workflow>` which delegates to claude under the hood.
|
|
194
402
|
// Use 'claude' as the effective agent for report extraction and metadata when workflow is set.
|
|
195
403
|
const effectiveAgent = config.workflow ? 'claude' : config.agent;
|
|
@@ -208,9 +416,10 @@ export async function executeJob(config, deps) {
|
|
|
208
416
|
const timeoutMs = parseTimeout(config.timeout) || 10 * 60 * 1000;
|
|
209
417
|
// Loop path: delegate to runLoop (same driver as `agents run --loop` / workflow loop:).
|
|
210
418
|
if (config.loop) {
|
|
419
|
+
const spawnEnv = buildRoutineSpawnEnv(baseEnv, effectiveAgent, primaryVersion, config.timezone);
|
|
211
420
|
const execOptions = {
|
|
212
421
|
agent: effectiveAgent,
|
|
213
|
-
version:
|
|
422
|
+
version: primaryVersion,
|
|
214
423
|
prompt: resolvedPrompt,
|
|
215
424
|
mode: normalizeMode(config.mode),
|
|
216
425
|
effort: config.effort,
|
|
@@ -229,7 +438,7 @@ export async function executeJob(config, deps) {
|
|
|
229
438
|
runId,
|
|
230
439
|
runDir,
|
|
231
440
|
agent: effectiveAgent,
|
|
232
|
-
version:
|
|
441
|
+
version: primaryVersion,
|
|
233
442
|
}, deps);
|
|
234
443
|
meta.status = loopResult.stoppedBy === 'error' ? 'failed' : 'completed';
|
|
235
444
|
meta.completedAt = new Date().toISOString();
|
|
@@ -238,83 +447,105 @@ export async function executeJob(config, deps) {
|
|
|
238
447
|
timer.end({ status: meta.status, exitCode: meta.exitCode ?? undefined, runId });
|
|
239
448
|
return { meta, reportPath: null };
|
|
240
449
|
}
|
|
241
|
-
// Single-shot path
|
|
242
|
-
|
|
450
|
+
// Single-shot path: build the command once, then walk the launch chain on
|
|
451
|
+
// rate/usage-limit failures (same detectRateLimit patterns as agents run).
|
|
452
|
+
const baseCmd = buildJobCommand(config, resolvedPrompt);
|
|
243
453
|
const stdoutPath = path.join(runDir, 'stdout.log');
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
454
|
+
// Truncate the log for a clean run; failover attempts append.
|
|
455
|
+
fs.writeFileSync(stdoutPath, '', { mode: 0o600 });
|
|
456
|
+
const chain = launch.chain.length > 0
|
|
457
|
+
? launch.chain
|
|
458
|
+
: [{ agent: effectiveAgent, version: primaryVersion }];
|
|
459
|
+
timer.mark('startup');
|
|
460
|
+
for (let i = 0; i < chain.length; i++) {
|
|
461
|
+
const entry = chain[i];
|
|
462
|
+
const attemptAgent = entry.agent;
|
|
463
|
+
const attemptVersion = entry.version;
|
|
464
|
+
const label = attemptVersion ? `${attemptAgent}@${attemptVersion}` : attemptAgent;
|
|
465
|
+
if (i === 0) {
|
|
466
|
+
process.stderr.write(`[agents] routine ${config.name}: running ${label}\n`);
|
|
467
|
+
}
|
|
468
|
+
const cmd = config.workflow
|
|
469
|
+
? baseCmd
|
|
470
|
+
: pinJobBinary(baseCmd, attemptAgent, attemptVersion);
|
|
471
|
+
const spawnEnv = config.workflow
|
|
472
|
+
? (() => {
|
|
473
|
+
const e = { ...baseEnv };
|
|
474
|
+
if (config.timezone)
|
|
475
|
+
e.TZ = config.timezone;
|
|
476
|
+
return e;
|
|
477
|
+
})()
|
|
478
|
+
: buildRoutineSpawnEnv(baseEnv, attemptAgent, attemptVersion, config.timezone);
|
|
479
|
+
// Remaining timeout budget shared across failover attempts.
|
|
480
|
+
const elapsed = Date.now() - Date.parse(meta.startedAt);
|
|
481
|
+
const remaining = Math.max(1_000, timeoutMs - (Number.isFinite(elapsed) ? elapsed : 0));
|
|
482
|
+
const attemptLogPath = path.join(runDir, `stdout.attempt-${i}.log`);
|
|
483
|
+
const attempt = await spawnJobAttempt(cmd, spawnEnv, attemptLogPath, remaining, stdoutPath);
|
|
484
|
+
meta.pid = attempt.pid;
|
|
254
485
|
writeRunMeta(meta);
|
|
255
|
-
|
|
256
|
-
const timeoutTimer = setTimeout(() => {
|
|
257
|
-
if (settled)
|
|
258
|
-
return;
|
|
259
|
-
settled = true;
|
|
260
|
-
try {
|
|
261
|
-
if (child.pid)
|
|
262
|
-
process.kill(-child.pid, 'SIGTERM');
|
|
263
|
-
}
|
|
264
|
-
catch { /* process already exited */ }
|
|
265
|
-
setTimeout(() => {
|
|
266
|
-
try {
|
|
267
|
-
if (child.pid)
|
|
268
|
-
process.kill(-child.pid, 'SIGKILL');
|
|
269
|
-
}
|
|
270
|
-
catch { /* process already exited */ }
|
|
271
|
-
}, 5000);
|
|
486
|
+
if (attempt.status === 'timeout') {
|
|
272
487
|
meta.status = 'timeout';
|
|
273
488
|
meta.completedAt = new Date().toISOString();
|
|
274
489
|
writeRunMeta(meta);
|
|
275
490
|
timer.end({ status: 'timeout', runId });
|
|
276
491
|
const reportPath = extractAndSaveReport(stdoutPath, effectiveAgent, runDir);
|
|
277
|
-
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
settled = true;
|
|
283
|
-
clearTimeout(timeoutTimer);
|
|
284
|
-
try {
|
|
285
|
-
fs.closeSync(stdoutFd);
|
|
286
|
-
}
|
|
287
|
-
catch { /* fd already closed */ }
|
|
288
|
-
meta.exitCode = code;
|
|
289
|
-
meta.status = code === 0 ? 'completed' : 'failed';
|
|
492
|
+
return { meta, reportPath };
|
|
493
|
+
}
|
|
494
|
+
if (attempt.status === 'completed') {
|
|
495
|
+
meta.exitCode = 0;
|
|
496
|
+
meta.status = 'completed';
|
|
290
497
|
meta.completedAt = new Date().toISOString();
|
|
291
498
|
writeRunMeta(meta);
|
|
292
|
-
timer.end({ status:
|
|
499
|
+
timer.end({ status: 'completed', exitCode: 0, runId });
|
|
293
500
|
const reportPath = extractAndSaveReport(stdoutPath, effectiveAgent, runDir);
|
|
294
|
-
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
501
|
+
return { meta, reportPath };
|
|
502
|
+
}
|
|
503
|
+
// Failed — cascade only on rate/usage limit when more chain entries remain.
|
|
504
|
+
const isLast = i === chain.length - 1;
|
|
505
|
+
const rateLimited = detectRateLimit(attempt.logText) || (attempt.error ? detectRateLimit(attempt.error) : false);
|
|
506
|
+
if (!isLast && rateLimited) {
|
|
507
|
+
const next = chain[i + 1];
|
|
508
|
+
const nextLabel = next.version ? `${next.agent}@${next.version}` : next.agent;
|
|
509
|
+
process.stderr.write(`[agents] routine ${config.name}: ${label} failed with credit/rate limit, trying ${nextLabel}\n`);
|
|
510
|
+
fs.appendFileSync(stdoutPath, `\n[agents] ${label} hit rate/usage limit — failover → ${nextLabel}\n`);
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
if (attempt.error) {
|
|
514
|
+
process.stderr.write(`[agents] routine ${config.name}: spawn failed for ${label}: ${attempt.error}\n`);
|
|
515
|
+
}
|
|
516
|
+
meta.exitCode = attempt.exitCode ?? 1;
|
|
517
|
+
meta.status = 'failed';
|
|
518
|
+
meta.completedAt = new Date().toISOString();
|
|
519
|
+
writeRunMeta(meta);
|
|
520
|
+
timer.end({
|
|
521
|
+
status: 'failed',
|
|
522
|
+
exitCode: meta.exitCode ?? undefined,
|
|
523
|
+
runId,
|
|
524
|
+
...(attempt.error ? { error: attempt.error } : {}),
|
|
310
525
|
});
|
|
311
|
-
|
|
312
|
-
|
|
526
|
+
const reportPath = extractAndSaveReport(stdoutPath, effectiveAgent, runDir);
|
|
527
|
+
return { meta, reportPath };
|
|
528
|
+
}
|
|
529
|
+
// Unreachable: chain is always non-empty, but keep a safe fallback.
|
|
530
|
+
meta.status = 'failed';
|
|
531
|
+
meta.exitCode = 1;
|
|
532
|
+
meta.completedAt = new Date().toISOString();
|
|
533
|
+
writeRunMeta(meta);
|
|
534
|
+
timer.end({ status: 'failed', exitCode: 1, runId });
|
|
535
|
+
return { meta, reportPath: null };
|
|
313
536
|
}
|
|
314
537
|
/** Spawn a job as a detached process and return immediately with run metadata. */
|
|
315
538
|
export async function executeJobDetached(config) {
|
|
539
|
+
// Pre-flight: pick a healthy version/account so the daemon does not launch
|
|
540
|
+
// into a credit-exhausted install. Detached cannot mid-run failover (no exit
|
|
541
|
+
// wait); the next schedule tick re-selects if this attempt still fails.
|
|
542
|
+
const launch = await resolveRoutineLaunch(config);
|
|
543
|
+
const version = launch.chain[0]?.version ?? config.version;
|
|
316
544
|
const resolvedPrompt = resolveJobPrompt(config);
|
|
317
|
-
|
|
545
|
+
let cmd = buildJobCommand(config, resolvedPrompt);
|
|
546
|
+
if (!config.workflow && version) {
|
|
547
|
+
cmd = pinJobBinary(cmd, config.agent, version);
|
|
548
|
+
}
|
|
318
549
|
const useSandbox = config.sandbox !== false;
|
|
319
550
|
const overlayHome = useSandbox ? prepareJobHome(config) : undefined;
|
|
320
551
|
const runId = generateRunId();
|
|
@@ -322,10 +553,17 @@ export async function executeJobDetached(config) {
|
|
|
322
553
|
fs.mkdirSync(runDir, { recursive: true });
|
|
323
554
|
const stdoutPath = path.join(runDir, 'stdout.log');
|
|
324
555
|
const stdoutFd = fs.openSync(stdoutPath, 'w', 0o600);
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
556
|
+
const baseEnv = useSandbox
|
|
557
|
+
? buildSpawnEnv(overlayHome)
|
|
558
|
+
: { ...process.env };
|
|
559
|
+
const spawnEnv = config.workflow
|
|
560
|
+
? (() => {
|
|
561
|
+
const e = { ...baseEnv };
|
|
562
|
+
if (config.timezone)
|
|
563
|
+
e.TZ = config.timezone;
|
|
564
|
+
return e;
|
|
565
|
+
})()
|
|
566
|
+
: buildRoutineSpawnEnv(baseEnv, config.agent, version, config.timezone);
|
|
329
567
|
const effectiveAgent = config.workflow ? 'claude' : config.agent;
|
|
330
568
|
const meta = {
|
|
331
569
|
jobName: config.name,
|
|
@@ -343,6 +581,17 @@ export async function executeJobDetached(config) {
|
|
|
343
581
|
...backgroundSpawnOptions({ fdStdio: true }),
|
|
344
582
|
env: spawnEnv,
|
|
345
583
|
});
|
|
584
|
+
child.on('error', (err) => {
|
|
585
|
+
try {
|
|
586
|
+
fs.closeSync(stdoutFd);
|
|
587
|
+
}
|
|
588
|
+
catch { /* fd already closed */ }
|
|
589
|
+
meta.status = 'failed';
|
|
590
|
+
meta.exitCode = 1;
|
|
591
|
+
meta.completedAt = new Date().toISOString();
|
|
592
|
+
writeRunMeta(meta);
|
|
593
|
+
process.stderr.write(`[agents] daemon: spawn failed for job "${config.name}": ${err.message}\n`);
|
|
594
|
+
});
|
|
346
595
|
child.unref();
|
|
347
596
|
try {
|
|
348
597
|
fs.closeSync(stdoutFd);
|
package/dist/lib/sandbox.js
CHANGED
|
@@ -43,6 +43,12 @@ const ENV_ALLOWLIST = [
|
|
|
43
43
|
'VISUAL',
|
|
44
44
|
'NO_COLOR',
|
|
45
45
|
'FORCE_COLOR',
|
|
46
|
+
// Headless Claude auth: the routines daemon injects CLAUDE_CODE_OAUTH_TOKEN
|
|
47
|
+
// from the `claude` secrets bundle (see daemon.ts). Without this allowlist
|
|
48
|
+
// entry, sandboxed routine spawns drop the token and look "unconfigured".
|
|
49
|
+
// Intentionally NOT ANTHROPIC_API_KEY / other provider secrets — those stay
|
|
50
|
+
// stripped (see tests/sandbox.test.ts "does not include sensitive env vars").
|
|
51
|
+
'CLAUDE_CODE_OAUTH_TOKEN',
|
|
46
52
|
];
|
|
47
53
|
/** Tools safe to grant as wildcards (no filesystem access). */
|
|
48
54
|
const SAFE_TOOLS = {
|
|
Binary file
|
|
Binary file
|
|
@@ -50,6 +50,18 @@ export declare const META_CACHE_PREFIX = "!meta:";
|
|
|
50
50
|
* cache on every bump and produced a recurring Touch ID storm.
|
|
51
51
|
*/
|
|
52
52
|
export declare function shouldSelfHealForUpgrade(persistent: boolean, storeSize: number, runningVersion: string, onDiskVersion: string): boolean;
|
|
53
|
+
/**
|
|
54
|
+
* Client-side twin of shouldSelfHealForUpgrade: whether ensureAgentRunning may
|
|
55
|
+
* tear down a reachable broker whose running version differs from the client's
|
|
56
|
+
* on-disk version. Only while it holds NO real unlocks — tearing down a hot
|
|
57
|
+
* broker wipes every held bundle, so the next read of each one re-prompts for
|
|
58
|
+
* Touch ID. On a machine where installed versions churn (dev builds stamp a
|
|
59
|
+
* fresh 0.0.0-dev.<sha> on every install; an npm copy and a dev copy invoke in
|
|
60
|
+
* turn), an unguarded teardown produced a rolling Touch ID storm — the exact
|
|
61
|
+
* failure #435 fixed on the server side. A hot, protocol-compatible broker
|
|
62
|
+
* keeps serving; its own sweep adopts the new code at the next quiet moment.
|
|
63
|
+
*/
|
|
64
|
+
export declare function shouldTeardownVersionSkewedBroker(realHeldBundles: number): boolean;
|
|
53
65
|
export interface StoredBundle {
|
|
54
66
|
bundle: SecretsBundle;
|
|
55
67
|
env: Record<string, string>;
|
|
@@ -172,6 +184,15 @@ export declare function agentGetSync(name: string): {
|
|
|
172
184
|
bundle: SecretsBundle;
|
|
173
185
|
env: Record<string, string>;
|
|
174
186
|
} | null;
|
|
187
|
+
/**
|
|
188
|
+
* Synchronously evict one bundle from the broker. Called after a mutating
|
|
189
|
+
* keychain write (add / rotate / remove / rename / delete) so the broker never
|
|
190
|
+
* keeps serving the pre-write snapshot for up to the ~7d hold — the next read
|
|
191
|
+
* re-resolves from the keychain (one prompt) and re-caches fresh values.
|
|
192
|
+
* Best-effort: no broker, no socket, or any failure is a silent no-op.
|
|
193
|
+
* macOS only.
|
|
194
|
+
*/
|
|
195
|
+
export declare function agentEvictSync(name: string): void;
|
|
175
196
|
/**
|
|
176
197
|
* Read the cached `secrets list` metadata snapshot for the given keychain
|
|
177
198
|
* name-set hash, or null on miss / no broker / off-darwin. Reuses the value
|
|
@@ -74,6 +74,20 @@ export function shouldSelfHealForUpgrade(persistent, storeSize, runningVersion,
|
|
|
74
74
|
return false;
|
|
75
75
|
return onDiskVersion !== runningVersion;
|
|
76
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Client-side twin of shouldSelfHealForUpgrade: whether ensureAgentRunning may
|
|
79
|
+
* tear down a reachable broker whose running version differs from the client's
|
|
80
|
+
* on-disk version. Only while it holds NO real unlocks — tearing down a hot
|
|
81
|
+
* broker wipes every held bundle, so the next read of each one re-prompts for
|
|
82
|
+
* Touch ID. On a machine where installed versions churn (dev builds stamp a
|
|
83
|
+
* fresh 0.0.0-dev.<sha> on every install; an npm copy and a dev copy invoke in
|
|
84
|
+
* turn), an unguarded teardown produced a rolling Touch ID storm — the exact
|
|
85
|
+
* failure #435 fixed on the server side. A hot, protocol-compatible broker
|
|
86
|
+
* keeps serving; its own sweep adopts the new code at the next quiet moment.
|
|
87
|
+
*/
|
|
88
|
+
export function shouldTeardownVersionSkewedBroker(realHeldBundles) {
|
|
89
|
+
return realHeldBundles === 0;
|
|
90
|
+
}
|
|
77
91
|
function onDarwin() {
|
|
78
92
|
return process.platform === 'darwin';
|
|
79
93
|
}
|
|
@@ -581,6 +595,50 @@ export function agentGetSync(name) {
|
|
|
581
595
|
return null;
|
|
582
596
|
}
|
|
583
597
|
}
|
|
598
|
+
/**
|
|
599
|
+
* Inline node program for the synchronous evict path. Mirrors SYNC_GET_PROGRAM:
|
|
600
|
+
* writeBundle is synchronous and called synchronously everywhere, so a stale
|
|
601
|
+
* broker entry must be evicted without awaiting a socket round-trip. Sends one
|
|
602
|
+
* {cmd:'lock', name} and exits 0 (evicted or nothing held) / 3 (agent down).
|
|
603
|
+
* argv after -e: [execPath, <socket>, <name>].
|
|
604
|
+
*/
|
|
605
|
+
const SYNC_LOCK_PROGRAM = `
|
|
606
|
+
const net = require('net');
|
|
607
|
+
const sock = process.argv[1], name = process.argv[2];
|
|
608
|
+
const c = net.createConnection(sock);
|
|
609
|
+
let buf = '';
|
|
610
|
+
const down = () => { try { c.destroy(); } catch (e) {} process.exit(3); };
|
|
611
|
+
const timer = setTimeout(down, 2000);
|
|
612
|
+
c.on('error', down);
|
|
613
|
+
c.on('connect', () => c.write(JSON.stringify({ cmd: 'lock', name }) + '\\n'));
|
|
614
|
+
c.setEncoding('utf-8');
|
|
615
|
+
c.on('data', (d) => {
|
|
616
|
+
buf += d;
|
|
617
|
+
const nl = buf.indexOf('\\n');
|
|
618
|
+
if (nl < 0) return;
|
|
619
|
+
clearTimeout(timer);
|
|
620
|
+
try { c.destroy(); } catch (e) {}
|
|
621
|
+
process.exit(0);
|
|
622
|
+
});
|
|
623
|
+
`;
|
|
624
|
+
/**
|
|
625
|
+
* Synchronously evict one bundle from the broker. Called after a mutating
|
|
626
|
+
* keychain write (add / rotate / remove / rename / delete) so the broker never
|
|
627
|
+
* keeps serving the pre-write snapshot for up to the ~7d hold — the next read
|
|
628
|
+
* re-resolves from the keychain (one prompt) and re-caches fresh values.
|
|
629
|
+
* Best-effort: no broker, no socket, or any failure is a silent no-op.
|
|
630
|
+
* macOS only.
|
|
631
|
+
*/
|
|
632
|
+
export function agentEvictSync(name) {
|
|
633
|
+
if (!onDarwin())
|
|
634
|
+
return;
|
|
635
|
+
if (!agentSocketExists())
|
|
636
|
+
return;
|
|
637
|
+
try {
|
|
638
|
+
spawnSync(process.execPath, ['-e', SYNC_LOCK_PROGRAM, socketPath(), name], { timeout: 3000 });
|
|
639
|
+
}
|
|
640
|
+
catch { /* best-effort */ }
|
|
641
|
+
}
|
|
584
642
|
// Key inside the cached entry's env that holds the JSON metadata snapshot.
|
|
585
643
|
const META_SNAPSHOT_KEY = '__snapshot__';
|
|
586
644
|
/**
|
|
@@ -733,11 +791,15 @@ export async function ensureAgentRunning(timeoutMs = 5000) {
|
|
|
733
791
|
// Self-heal: if a broker is reachable but running pre-upgrade code (its
|
|
734
792
|
// reported version != the version on disk now), tear it down so the paths
|
|
735
793
|
// below bring up a fresh one on current code. A current, reachable broker is
|
|
736
|
-
// accepted immediately
|
|
794
|
+
// accepted immediately — and so is a version-skewed one that still holds
|
|
795
|
+
// real unlocks (see shouldTeardownVersionSkewedBroker: wiping a hot cache
|
|
796
|
+
// re-prompts Touch ID for every held bundle).
|
|
737
797
|
const ping = await agentPing();
|
|
738
798
|
if (ping.reachable) {
|
|
739
799
|
if (ping.cliVersion === undefined || ping.cliVersion === getCliVersionFresh())
|
|
740
800
|
return true;
|
|
801
|
+
if (!shouldTeardownVersionSkewedBroker((await agentStatus()).length))
|
|
802
|
+
return true;
|
|
741
803
|
await teardownStaleBroker();
|
|
742
804
|
}
|
|
743
805
|
// Path 1: the persistent service. installSecretsAgentService is idempotent and
|