@phnx-labs/agents-cli 1.22.26 → 1.22.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +293 -0
- package/README.md +80 -27
- package/dist/bin/agents +0 -0
- package/dist/commands/bench.d.ts +2 -0
- package/dist/commands/bench.js +101 -0
- package/dist/commands/doctor.js +24 -10
- package/dist/commands/exec.js +71 -26
- package/dist/commands/hosts.js +4 -3
- package/dist/commands/insights.js +39 -5
- package/dist/commands/output.js +100 -25
- package/dist/commands/perf.d.ts +10 -0
- package/dist/commands/perf.js +14 -6
- package/dist/commands/projects.js +24 -10
- package/dist/commands/resume.d.ts +11 -0
- package/dist/commands/resume.js +51 -0
- package/dist/commands/routines.js +31 -4
- package/dist/commands/run-account-picker.d.ts +37 -0
- package/dist/commands/run-account-picker.js +101 -17
- package/dist/commands/secrets.js +86 -2
- package/dist/commands/sessions-picker.js +12 -7
- package/dist/commands/sessions.d.ts +73 -7
- package/dist/commands/sessions.js +391 -49
- package/dist/commands/ssh.js +184 -11
- package/dist/commands/teams.js +22 -11
- package/dist/index.js +16 -20
- package/dist/lib/activity.js +45 -99
- package/dist/lib/auth-health.d.ts +47 -3
- package/dist/lib/auth-health.js +74 -15
- package/dist/lib/bench/index.d.ts +4 -0
- package/dist/lib/bench/index.js +4 -0
- package/dist/lib/bench/runner.d.ts +16 -0
- package/dist/lib/bench/runner.js +111 -0
- package/dist/lib/bench/schema.d.ts +5 -0
- package/dist/lib/bench/schema.js +91 -0
- package/dist/lib/bench/storage.d.ts +5 -0
- package/dist/lib/bench/storage.js +32 -0
- package/dist/lib/bench/types.d.ts +40 -0
- package/dist/lib/bench/types.js +1 -0
- package/dist/lib/claude-account-token.d.ts +2 -0
- package/dist/lib/claude-account-token.js +56 -3
- package/dist/lib/cloud/host.js +1 -0
- package/dist/lib/crabbox/cli.d.ts +2 -0
- package/dist/lib/crabbox/cli.js +2 -0
- package/dist/lib/crabbox/lease.js +7 -1
- package/dist/lib/daemon.js +20 -0
- package/dist/lib/devices/connect.d.ts +2 -0
- package/dist/lib/devices/connect.js +7 -0
- package/dist/lib/devices/doctor-findings.d.ts +4 -1
- package/dist/lib/devices/doctor-findings.js +15 -1
- package/dist/lib/devices/harness-inventory.d.ts +97 -0
- package/dist/lib/devices/harness-inventory.js +0 -0
- package/dist/lib/devices/registry.d.ts +2 -0
- package/dist/lib/devices/resolve-target.d.ts +1 -0
- package/dist/lib/devices/resolve-target.js +9 -2
- package/dist/lib/devices/ssh-config.js +3 -0
- package/dist/lib/devices/windows-ssh-enrollment.d.ts +20 -0
- package/dist/lib/devices/windows-ssh-enrollment.js +98 -0
- package/dist/lib/events.d.ts +1 -1
- package/dist/lib/events.js +2 -2
- package/dist/lib/exec.js +18 -8
- package/dist/lib/fleet/apply.js +9 -7
- package/dist/lib/fleet/remote-login.d.ts +4 -3
- package/dist/lib/fleet/remote-login.js +11 -9
- package/dist/lib/gemini-settings.d.ts +0 -1
- package/dist/lib/gemini-settings.js +12 -7
- package/dist/lib/git.d.ts +9 -3
- package/dist/lib/git.js +39 -22
- package/dist/lib/hooks/profile.d.ts +12 -1
- package/dist/lib/hooks/profile.js +6 -1
- package/dist/lib/hooks.d.ts +8 -5
- package/dist/lib/hooks.js +14 -7
- package/dist/lib/hosts/dispatch.d.ts +10 -0
- package/dist/lib/hosts/dispatch.js +85 -25
- package/dist/lib/hosts/logs.js +11 -1
- package/dist/lib/hosts/passthrough.d.ts +27 -0
- package/dist/lib/hosts/passthrough.js +19 -10
- package/dist/lib/hosts/progress.d.ts +12 -1
- package/dist/lib/hosts/progress.js +41 -8
- package/dist/lib/hosts/providers/devices.js +1 -0
- package/dist/lib/hosts/ready.d.ts +45 -4
- package/dist/lib/hosts/ready.js +124 -20
- package/dist/lib/hosts/reconcile.d.ts +2 -1
- package/dist/lib/hosts/reconcile.js +24 -7
- package/dist/lib/hosts/reconnect.d.ts +45 -12
- package/dist/lib/hosts/reconnect.js +94 -36
- package/dist/lib/hosts/registry.d.ts +2 -2
- package/dist/lib/hosts/registry.js +4 -5
- package/dist/lib/hosts/remote-cmd.d.ts +17 -0
- package/dist/lib/hosts/remote-cmd.js +29 -0
- package/dist/lib/hosts/tasks.d.ts +3 -0
- package/dist/lib/hosts/types.d.ts +1 -0
- package/dist/lib/hosts/types.js +3 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/CodeResources +0 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/MacOS/MenubarHelper +0 -0
- package/dist/lib/menubar/install-menubar.d.ts +9 -6
- package/dist/lib/menubar/install-menubar.js +20 -9
- package/dist/lib/menubar/snapshot.js +15 -2
- package/dist/lib/models.d.ts +42 -6
- package/dist/lib/models.js +55 -153
- package/dist/lib/perf/db.js +16 -3
- package/dist/lib/perf/types.d.ts +12 -1
- package/dist/lib/pricing/cost.d.ts +9 -0
- package/dist/lib/pricing/cost.js +24 -0
- package/dist/lib/pricing/index.d.ts +1 -1
- package/dist/lib/pricing/index.js +1 -1
- package/dist/lib/project-probe.d.ts +22 -3
- package/dist/lib/project-probe.js +105 -17
- package/dist/lib/project-status.d.ts +9 -0
- package/dist/lib/project-status.js +15 -0
- package/dist/lib/redact.js +8 -3
- package/dist/lib/refresh.js +22 -11
- package/dist/lib/remote-agents-json.d.ts +32 -0
- package/dist/lib/remote-agents-json.js +47 -16
- package/dist/lib/resource-profiles.js +1 -2
- package/dist/lib/rotate.d.ts +22 -0
- package/dist/lib/rotate.js +26 -0
- package/dist/lib/routine-notify-owner.d.ts +102 -0
- package/dist/lib/routine-notify-owner.js +232 -0
- package/dist/lib/routines.d.ts +10 -0
- package/dist/lib/runner.d.ts +9 -1
- package/dist/lib/runner.js +182 -17
- package/dist/lib/sandbox.d.ts +0 -2
- package/dist/lib/sandbox.js +2 -19
- package/dist/lib/secrets/Agents CLI.app/Contents/CodeResources +0 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/MacOS/Agents CLI +0 -0
- package/dist/lib/secrets/agent.d.ts +19 -3
- package/dist/lib/secrets/agent.js +61 -22
- package/dist/lib/secrets/audit.d.ts +1 -1
- package/dist/lib/secrets/audit.js +2 -0
- package/dist/lib/secrets/bundles.d.ts +3 -3
- package/dist/lib/secrets/bundles.js +15 -5
- package/dist/lib/secrets/filestore.d.ts +2 -0
- package/dist/lib/secrets/filestore.js +4 -0
- package/dist/lib/secrets/lease.d.ts +25 -0
- package/dist/lib/secrets/lease.js +44 -0
- package/dist/lib/secrets/session-store.d.ts +7 -0
- package/dist/lib/secrets/session-store.js +21 -0
- package/dist/lib/self-update.d.ts +20 -5
- package/dist/lib/self-update.js +93 -16
- package/dist/lib/session/active.d.ts +49 -0
- package/dist/lib/session/active.js +175 -11
- package/dist/lib/session/bash-command.d.ts +16 -0
- package/dist/lib/session/bash-command.js +65 -0
- package/dist/lib/session/db.d.ts +23 -2
- package/dist/lib/session/db.js +86 -20
- package/dist/lib/session/discover.d.ts +21 -1
- package/dist/lib/session/discover.js +124 -32
- package/dist/lib/session/insights.d.ts +19 -0
- package/dist/lib/session/insights.js +78 -10
- package/dist/lib/session/recovery.d.ts +17 -1
- package/dist/lib/session/recovery.js +111 -4
- package/dist/lib/session/remote-bundle.js +1 -1
- package/dist/lib/session/remote-list.d.ts +10 -9
- package/dist/lib/session/remote-list.js +25 -23
- package/dist/lib/session/resume-owner.d.ts +55 -0
- package/dist/lib/session/resume-owner.js +69 -0
- package/dist/lib/session/team-filter.d.ts +65 -0
- package/dist/lib/session/team-filter.js +98 -3
- package/dist/lib/session/types.d.ts +33 -0
- package/dist/lib/smart-launch.d.ts +31 -5
- package/dist/lib/smart-launch.js +43 -7
- package/dist/lib/ssh-exec.d.ts +27 -0
- package/dist/lib/ssh-exec.js +34 -1
- package/dist/lib/ssh-tunnel.d.ts +3 -2
- package/dist/lib/ssh-tunnel.js +25 -16
- package/dist/lib/staleness/detectors/skills.d.ts +2 -0
- package/dist/lib/staleness/detectors/skills.js +17 -2
- package/dist/lib/staleness/index.d.ts +10 -1
- package/dist/lib/staleness/index.js +28 -3
- package/dist/lib/startup/command-registry.d.ts +1 -0
- package/dist/lib/startup/command-registry.js +2 -0
- package/dist/lib/teams/agents.d.ts +1 -0
- package/dist/lib/teams/agents.js +38 -11
- package/dist/lib/teams/placement-probe.d.ts +1 -1
- package/dist/lib/teams/placement-probe.js +24 -24
- package/dist/lib/teams/remoteWorktree.d.ts +11 -7
- package/dist/lib/teams/remoteWorktree.js +29 -27
- package/dist/lib/tmux/session.d.ts +8 -0
- package/dist/lib/tmux/session.js +22 -0
- package/dist/lib/types.d.ts +2 -0
- package/dist/lib/versions.d.ts +1 -0
- package/dist/lib/versions.js +40 -22
- package/package.json +3 -1
|
@@ -156,10 +156,54 @@ export interface AuthProbeRow {
|
|
|
156
156
|
account?: string;
|
|
157
157
|
health: AuthHealth;
|
|
158
158
|
}
|
|
159
|
+
/** One installed (agent, version) home, tagged with its resolved account label. */
|
|
160
|
+
export interface FleetAuthInstall {
|
|
161
|
+
agent: AgentId;
|
|
162
|
+
version: string;
|
|
163
|
+
/** Human account label from {@link authAccountLabel}, or undefined when none resolves. */
|
|
164
|
+
account: string | undefined;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* A set of installs that share one provider account and MUST be probed once.
|
|
168
|
+
* The live probe runs against `probe` (the representative home); every entry in
|
|
169
|
+
* `members` — the representative included — then receives that one verdict.
|
|
170
|
+
*/
|
|
171
|
+
export interface FleetAuthProbeGroup<T extends FleetAuthInstall> {
|
|
172
|
+
probe: T;
|
|
173
|
+
members: T[];
|
|
174
|
+
}
|
|
159
175
|
/**
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
176
|
+
* Collapse installs so a live auth probe fires ONCE per (agent, account) rather
|
|
177
|
+
* than once per version home.
|
|
178
|
+
*
|
|
179
|
+
* Several version homes signed into the same provider account share one OAuth
|
|
180
|
+
* rate limit, so probing each of them concurrently — which the daemon did every
|
|
181
|
+
* three minutes across every installed home (RUSH-2111) — raced that limit into
|
|
182
|
+
* a 429 storm that then parked the whole box behind a `Retry-After` penalty (the
|
|
183
|
+
* incident {@link file:./usage-backoff.ts} was written to survive; this removes
|
|
184
|
+
* its cause). Grouping by account means N homes on one account issue ONE request.
|
|
185
|
+
*
|
|
186
|
+
* `isMergeable` scopes the dedup to the installs it actually helps: only agents
|
|
187
|
+
* that make a network probe ({@link LIVE_PROBE_AGENTS}) can 429, so only they are
|
|
188
|
+
* merged. A best-effort agent (its verdict is a cheap local file read, no rate
|
|
189
|
+
* limit) is left per-install — collapsing it would gain nothing and could
|
|
190
|
+
* silently override one home's local `signedIn` verdict with another's. Installs
|
|
191
|
+
* with no resolvable account label, or that `isMergeable` rejects, are never
|
|
192
|
+
* merged: each becomes its own group keyed by version, preserving the old
|
|
193
|
+
* per-install probe (also the `unconfigured` case dropped downstream). Pure: no
|
|
194
|
+
* fs, no network, so the dedup decision is unit-tested directly.
|
|
195
|
+
*/
|
|
196
|
+
export declare function groupFleetAuthInstalls<T extends FleetAuthInstall>(installs: readonly T[], isMergeable?: (install: T) => boolean): FleetAuthProbeGroup<T>[];
|
|
197
|
+
/**
|
|
198
|
+
* Enumerate every installed (agent, version) on THIS host and return one row per
|
|
199
|
+
* install (installs with no credential at all are dropped). Shared by
|
|
200
|
+
* `agents fleet ping --local` and the daemon refresh.
|
|
201
|
+
*
|
|
202
|
+
* The live network probe is deduped by account: homes sharing one provider
|
|
203
|
+
* account are probed ONCE and the verdict is fanned out to each home's row, so
|
|
204
|
+
* the daemon's every-3-minute refresh can no longer fire concurrent same-account
|
|
205
|
+
* requests and self-inflict a 429 (RUSH-2111). Each home still gets its own cache
|
|
206
|
+
* row (the cache key is per-version by design — see {@link authCacheKey}).
|
|
163
207
|
*/
|
|
164
208
|
export declare function probeLocalFleetAuth(opts?: {
|
|
165
209
|
cliVersion?: string | null;
|
package/dist/lib/auth-health.js
CHANGED
|
@@ -283,28 +283,87 @@ export async function probeAuthHealth(agent, home, opts) {
|
|
|
283
283
|
return { verdict: info?.signedIn ? 'unverified' : 'unconfigured', checkedAt };
|
|
284
284
|
}
|
|
285
285
|
/**
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
286
|
+
* Collapse installs so a live auth probe fires ONCE per (agent, account) rather
|
|
287
|
+
* than once per version home.
|
|
288
|
+
*
|
|
289
|
+
* Several version homes signed into the same provider account share one OAuth
|
|
290
|
+
* rate limit, so probing each of them concurrently — which the daemon did every
|
|
291
|
+
* three minutes across every installed home (RUSH-2111) — raced that limit into
|
|
292
|
+
* a 429 storm that then parked the whole box behind a `Retry-After` penalty (the
|
|
293
|
+
* incident {@link file:./usage-backoff.ts} was written to survive; this removes
|
|
294
|
+
* its cause). Grouping by account means N homes on one account issue ONE request.
|
|
295
|
+
*
|
|
296
|
+
* `isMergeable` scopes the dedup to the installs it actually helps: only agents
|
|
297
|
+
* that make a network probe ({@link LIVE_PROBE_AGENTS}) can 429, so only they are
|
|
298
|
+
* merged. A best-effort agent (its verdict is a cheap local file read, no rate
|
|
299
|
+
* limit) is left per-install — collapsing it would gain nothing and could
|
|
300
|
+
* silently override one home's local `signedIn` verdict with another's. Installs
|
|
301
|
+
* with no resolvable account label, or that `isMergeable` rejects, are never
|
|
302
|
+
* merged: each becomes its own group keyed by version, preserving the old
|
|
303
|
+
* per-install probe (also the `unconfigured` case dropped downstream). Pure: no
|
|
304
|
+
* fs, no network, so the dedup decision is unit-tested directly.
|
|
305
|
+
*/
|
|
306
|
+
export function groupFleetAuthInstalls(installs, isMergeable = () => true) {
|
|
307
|
+
const groups = new Map();
|
|
308
|
+
for (const inst of installs) {
|
|
309
|
+
// The `acct:` / `ver:` tokens make the two branches disjoint, so an account
|
|
310
|
+
// label can never collide with a version fallback key no matter its content.
|
|
311
|
+
const key = inst.account && isMergeable(inst)
|
|
312
|
+
? `${inst.agent} acct:${inst.account}`
|
|
313
|
+
: `${inst.agent} ver:${inst.version}`;
|
|
314
|
+
const existing = groups.get(key);
|
|
315
|
+
if (existing)
|
|
316
|
+
existing.members.push(inst);
|
|
317
|
+
else
|
|
318
|
+
groups.set(key, { probe: inst, members: [inst] });
|
|
319
|
+
}
|
|
320
|
+
return [...groups.values()];
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Enumerate every installed (agent, version) on THIS host and return one row per
|
|
324
|
+
* install (installs with no credential at all are dropped). Shared by
|
|
325
|
+
* `agents fleet ping --local` and the daemon refresh.
|
|
326
|
+
*
|
|
327
|
+
* The live network probe is deduped by account: homes sharing one provider
|
|
328
|
+
* account are probed ONCE and the verdict is fanned out to each home's row, so
|
|
329
|
+
* the daemon's every-3-minute refresh can no longer fire concurrent same-account
|
|
330
|
+
* requests and self-inflict a 429 (RUSH-2111). Each home still gets its own cache
|
|
331
|
+
* row (the cache key is per-version by design — see {@link authCacheKey}).
|
|
289
332
|
*/
|
|
290
333
|
export async function probeLocalFleetAuth(opts) {
|
|
291
334
|
const agentIds = opts?.agents ?? ALL_AGENT_IDS;
|
|
292
|
-
|
|
335
|
+
// Enumerate every install, then resolve its account label. getAccountInfo is a
|
|
336
|
+
// local credential-file read (no network), so this fan-out is cheap and cannot
|
|
337
|
+
// contribute to the rate limit the probe grouping below exists to avoid.
|
|
338
|
+
const installs = [];
|
|
293
339
|
for (const agent of agentIds) {
|
|
294
340
|
for (const version of listInstalledVersions(agent)) {
|
|
295
|
-
|
|
296
|
-
tasks.push((async () => {
|
|
297
|
-
const info = await getAccountInfo(agent, home).catch(() => null);
|
|
298
|
-
const health = await probeAuthHealth(agent, home, { cliVersion: opts?.cliVersion, info });
|
|
299
|
-
health.account = authAccountLabel(info);
|
|
300
|
-
if (health.verdict === 'unconfigured')
|
|
301
|
-
return null;
|
|
302
|
-
return { agent, version, account: health.account, health };
|
|
303
|
-
})());
|
|
341
|
+
installs.push({ agent, version, home: getVersionHomePath(agent, version), info: null, account: undefined });
|
|
304
342
|
}
|
|
305
343
|
}
|
|
306
|
-
|
|
307
|
-
|
|
344
|
+
await Promise.all(installs.map(async (inst) => {
|
|
345
|
+
inst.info = await getAccountInfo(inst.agent, inst.home).catch(() => null);
|
|
346
|
+
inst.account = authAccountLabel(inst.info);
|
|
347
|
+
}));
|
|
348
|
+
// Probe once per (agent, account) — but only for the network-probing agents
|
|
349
|
+
// that can actually 429; best-effort agents stay per-install (see
|
|
350
|
+
// groupFleetAuthInstalls). Groups run in parallel: they target distinct
|
|
351
|
+
// accounts, so no same-account concurrency is left to trip the throttle.
|
|
352
|
+
const perGroup = await Promise.all(groupFleetAuthInstalls(installs, (inst) => LIVE_PROBE_AGENTS.has(inst.agent)).map(async (group) => {
|
|
353
|
+
const rep = group.probe;
|
|
354
|
+
const health = await probeAuthHealth(rep.agent, rep.home, { cliVersion: opts?.cliVersion, info: rep.info });
|
|
355
|
+
health.account = authAccountLabel(rep.info);
|
|
356
|
+
if (health.verdict === 'unconfigured')
|
|
357
|
+
return [];
|
|
358
|
+
return group.members.map((inst) => ({
|
|
359
|
+
agent: inst.agent,
|
|
360
|
+
version: inst.version,
|
|
361
|
+
account: health.account,
|
|
362
|
+
// A distinct object per row so a later mutation of one can't bleed across.
|
|
363
|
+
health: { ...health },
|
|
364
|
+
}));
|
|
365
|
+
}));
|
|
366
|
+
return perGroup.flat();
|
|
308
367
|
}
|
|
309
368
|
/** Persist a host's probed rows into the cache (keyed by host+agent+version). */
|
|
310
369
|
export function writeFleetAuthRows(host, rows) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { BenchCell, BenchCellResult, BenchRunResult, BenchTask } from "./types.js";
|
|
2
|
+
export interface CellExecutionRequest extends BenchCell {
|
|
3
|
+
prompt: string;
|
|
4
|
+
cwd: string;
|
|
5
|
+
}
|
|
6
|
+
export type CellExecutor = (request: CellExecutionRequest) => Promise<Omit<BenchCellResult, keyof BenchCell | "wall_ms" | "status">>;
|
|
7
|
+
export declare function extractTokenCount(stdout: string, stderr: string): number | undefined;
|
|
8
|
+
export declare function executeCellViaAgentsRun(request: CellExecutionRequest): ReturnType<CellExecutor>;
|
|
9
|
+
export declare function runCells(options: {
|
|
10
|
+
task?: BenchTask;
|
|
11
|
+
prompt: string;
|
|
12
|
+
cells: BenchCell[];
|
|
13
|
+
concurrency: number;
|
|
14
|
+
execute?: CellExecutor;
|
|
15
|
+
tempRoot?: string;
|
|
16
|
+
}): Promise<BenchRunResult>;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { spawn } from "child_process";
|
|
2
|
+
import * as fs from "fs";
|
|
3
|
+
import * as os from "os";
|
|
4
|
+
import * as path from "path";
|
|
5
|
+
import { getCliLaunch } from "../cli-entry.js";
|
|
6
|
+
export function extractTokenCount(stdout, stderr) {
|
|
7
|
+
const matches = [
|
|
8
|
+
...`${stdout}\n${stderr}`.matchAll(/(?:tokens used|total tokens|token usage)\s*[:=]?\s*([\d,]+)/gi),
|
|
9
|
+
];
|
|
10
|
+
const raw = matches.at(-1)?.[1];
|
|
11
|
+
if (!raw)
|
|
12
|
+
return undefined;
|
|
13
|
+
const tokens = Number(raw.replaceAll(",", ""));
|
|
14
|
+
return Number.isSafeInteger(tokens) && tokens >= 0 ? tokens : undefined;
|
|
15
|
+
}
|
|
16
|
+
export async function executeCellViaAgentsRun(request) {
|
|
17
|
+
const args = [
|
|
18
|
+
"run",
|
|
19
|
+
request.agent,
|
|
20
|
+
request.prompt,
|
|
21
|
+
"--cwd",
|
|
22
|
+
request.cwd,
|
|
23
|
+
"--headless",
|
|
24
|
+
"--no-tmux",
|
|
25
|
+
];
|
|
26
|
+
if (request.model)
|
|
27
|
+
args.push("--model", request.model);
|
|
28
|
+
const launch = getCliLaunch(args);
|
|
29
|
+
return await new Promise((resolve, reject) => {
|
|
30
|
+
const child = spawn(launch.command, launch.args, {
|
|
31
|
+
cwd: request.cwd,
|
|
32
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
33
|
+
});
|
|
34
|
+
let stdout = "";
|
|
35
|
+
let stderr = "";
|
|
36
|
+
child.stdout.on("data", (chunk) => {
|
|
37
|
+
stdout += chunk.toString();
|
|
38
|
+
});
|
|
39
|
+
child.stderr.on("data", (chunk) => {
|
|
40
|
+
stderr += chunk.toString();
|
|
41
|
+
});
|
|
42
|
+
child.on("error", reject);
|
|
43
|
+
child.on("close", (code) => resolve({
|
|
44
|
+
exit: code,
|
|
45
|
+
stdout,
|
|
46
|
+
stderr,
|
|
47
|
+
tokens: extractTokenCount(stdout, stderr),
|
|
48
|
+
}));
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
function copyFixture(task, cellIndex, runRoot) {
|
|
52
|
+
const cwd = path.join(runRoot, `cell-${cellIndex}`);
|
|
53
|
+
fs.mkdirSync(cwd, { recursive: true });
|
|
54
|
+
if (fs.existsSync(task.fixtureDir))
|
|
55
|
+
fs.cpSync(task.fixtureDir, cwd, { recursive: true });
|
|
56
|
+
return cwd;
|
|
57
|
+
}
|
|
58
|
+
export async function runCells(options) {
|
|
59
|
+
if (!Number.isInteger(options.concurrency) || options.concurrency < 1)
|
|
60
|
+
throw new Error("concurrency must be a positive integer");
|
|
61
|
+
if (options.cells.length === 0)
|
|
62
|
+
throw new Error("at least one bench cell is required");
|
|
63
|
+
const runId = `${new Date().toISOString().replace(/[-:.TZ]/g, "")}-${process.pid}`;
|
|
64
|
+
const startedAt = new Date();
|
|
65
|
+
const runRoot = fs.mkdtempSync(path.join(options.tempRoot ?? os.tmpdir(), `agents-bench-${runId}-`));
|
|
66
|
+
const results = new Array(options.cells.length);
|
|
67
|
+
const execute = options.execute ?? executeCellViaAgentsRun;
|
|
68
|
+
let next = 0;
|
|
69
|
+
const worker = async () => {
|
|
70
|
+
for (;;) {
|
|
71
|
+
const index = next++;
|
|
72
|
+
if (index >= options.cells.length)
|
|
73
|
+
return;
|
|
74
|
+
const cell = options.cells[index];
|
|
75
|
+
const cwd = options.task
|
|
76
|
+
? copyFixture(options.task, index, runRoot)
|
|
77
|
+
: path.join(runRoot, `cell-${index}`);
|
|
78
|
+
fs.mkdirSync(cwd, { recursive: true });
|
|
79
|
+
const started = performance.now();
|
|
80
|
+
try {
|
|
81
|
+
const output = await execute({ ...cell, prompt: options.prompt, cwd });
|
|
82
|
+
results[index] = {
|
|
83
|
+
...cell,
|
|
84
|
+
...output,
|
|
85
|
+
wall_ms: Math.round(performance.now() - started),
|
|
86
|
+
status: output.exit === 0 ? "passed" : "failed",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
results[index] = {
|
|
91
|
+
...cell,
|
|
92
|
+
exit: null,
|
|
93
|
+
stdout: "",
|
|
94
|
+
stderr: error instanceof Error ? error.message : String(error),
|
|
95
|
+
wall_ms: Math.round(performance.now() - started),
|
|
96
|
+
status: "failed",
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
await Promise.all(Array.from({ length: Math.min(options.concurrency, options.cells.length) }, worker));
|
|
102
|
+
return {
|
|
103
|
+
schema_version: 1,
|
|
104
|
+
run_id: runId,
|
|
105
|
+
...(options.task ? { task_id: options.task.id } : {}),
|
|
106
|
+
prompt: options.prompt,
|
|
107
|
+
started_at: startedAt.toISOString(),
|
|
108
|
+
finished_at: new Date().toISOString(),
|
|
109
|
+
cells: results,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { BenchRunResult, BenchTask } from "./types.js";
|
|
2
|
+
export declare function parseTask(raw: unknown, sourcePath: string): BenchTask;
|
|
3
|
+
export declare function parseRunResult(raw: unknown): BenchRunResult;
|
|
4
|
+
export declare function validateRunId(runId: string): string;
|
|
5
|
+
export declare function loadTask(taskId: string, tasksRoot: string): BenchTask;
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
function object(value, label) {
|
|
4
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
5
|
+
throw new Error(`${label} must be an object`);
|
|
6
|
+
return value;
|
|
7
|
+
}
|
|
8
|
+
function string(value, label) {
|
|
9
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
10
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
export function parseTask(raw, sourcePath) {
|
|
14
|
+
const value = object(raw, "task.json");
|
|
15
|
+
const id = string(value.id, "task.id");
|
|
16
|
+
const prompt = string(value.prompt, "task.prompt");
|
|
17
|
+
if (!Array.isArray(value.pass) || value.pass.length === 0)
|
|
18
|
+
throw new Error("task.pass must be a non-empty array");
|
|
19
|
+
const pass = value.pass.map((item, index) => {
|
|
20
|
+
const criterion = object(item, `task.pass[${index}]`);
|
|
21
|
+
const type = string(criterion.type, `task.pass[${index}].type`);
|
|
22
|
+
if (type === "exit_zero")
|
|
23
|
+
return { type };
|
|
24
|
+
if (type === "file_exists")
|
|
25
|
+
return { type, path: string(criterion.path, `task.pass[${index}].path`) };
|
|
26
|
+
if (type === "stdout_contains")
|
|
27
|
+
return {
|
|
28
|
+
type,
|
|
29
|
+
value: string(criterion.value, `task.pass[${index}].value`),
|
|
30
|
+
};
|
|
31
|
+
if (type === "command_succeeds")
|
|
32
|
+
return {
|
|
33
|
+
type,
|
|
34
|
+
command: string(criterion.command, `task.pass[${index}].command`),
|
|
35
|
+
};
|
|
36
|
+
throw new Error(`task.pass[${index}].type is unsupported: ${type}`);
|
|
37
|
+
});
|
|
38
|
+
const taskDir = path.dirname(sourcePath);
|
|
39
|
+
return {
|
|
40
|
+
id,
|
|
41
|
+
prompt,
|
|
42
|
+
pass,
|
|
43
|
+
fixtureDir: path.join(taskDir, "fixture"),
|
|
44
|
+
sourcePath,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export function parseRunResult(raw) {
|
|
48
|
+
const value = object(raw, "bench result");
|
|
49
|
+
if (value.schema_version !== 1)
|
|
50
|
+
throw new Error("bench result schema_version must be 1");
|
|
51
|
+
string(value.run_id, "bench result run_id");
|
|
52
|
+
string(value.prompt, "bench result prompt");
|
|
53
|
+
string(value.started_at, "bench result started_at");
|
|
54
|
+
string(value.finished_at, "bench result finished_at");
|
|
55
|
+
if (value.task_id !== undefined)
|
|
56
|
+
string(value.task_id, "bench result task_id");
|
|
57
|
+
if (!Array.isArray(value.cells))
|
|
58
|
+
throw new Error("bench result cells must be an array");
|
|
59
|
+
value.cells.forEach((rawCell, index) => {
|
|
60
|
+
const cell = object(rawCell, `bench result cells[${index}]`);
|
|
61
|
+
string(cell.agent, `bench result cells[${index}].agent`);
|
|
62
|
+
if (cell.model !== undefined)
|
|
63
|
+
string(cell.model, `bench result cells[${index}].model`);
|
|
64
|
+
if (cell.status !== "passed" && cell.status !== "failed")
|
|
65
|
+
throw new Error(`bench result cells[${index}].status must be passed or failed`);
|
|
66
|
+
if (cell.exit !== null &&
|
|
67
|
+
(!Number.isInteger(cell.exit) || cell.exit < 0))
|
|
68
|
+
throw new Error(`bench result cells[${index}].exit must be a non-negative integer or null`);
|
|
69
|
+
if (typeof cell.wall_ms !== "number" ||
|
|
70
|
+
!Number.isFinite(cell.wall_ms) ||
|
|
71
|
+
cell.wall_ms < 0)
|
|
72
|
+
throw new Error(`bench result cells[${index}].wall_ms must be a non-negative number`);
|
|
73
|
+
if (cell.tokens !== undefined &&
|
|
74
|
+
(!Number.isSafeInteger(cell.tokens) || cell.tokens < 0))
|
|
75
|
+
throw new Error(`bench result cells[${index}].tokens must be a non-negative integer`);
|
|
76
|
+
if (typeof cell.stdout !== "string" || typeof cell.stderr !== "string")
|
|
77
|
+
throw new Error(`bench result cells[${index}] stdout and stderr must be strings`);
|
|
78
|
+
});
|
|
79
|
+
return value;
|
|
80
|
+
}
|
|
81
|
+
export function validateRunId(runId) {
|
|
82
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(runId))
|
|
83
|
+
throw new Error(`Invalid run id: ${runId}`);
|
|
84
|
+
return runId;
|
|
85
|
+
}
|
|
86
|
+
export function loadTask(taskId, tasksRoot) {
|
|
87
|
+
if (!/^[a-z0-9][a-z0-9-]*$/.test(taskId))
|
|
88
|
+
throw new Error(`Invalid task id: ${taskId}`);
|
|
89
|
+
const sourcePath = path.join(tasksRoot, taskId, "task.json");
|
|
90
|
+
return parseTask(JSON.parse(fs.readFileSync(sourcePath, "utf8")), sourcePath);
|
|
91
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { BenchRunResult } from "./types.js";
|
|
2
|
+
export declare function benchHistoryDir(): string;
|
|
3
|
+
export declare function saveRun(result: BenchRunResult, root?: string): string;
|
|
4
|
+
export declare function loadRun(runId: string, root?: string): BenchRunResult;
|
|
5
|
+
export declare function listRuns(root?: string): BenchRunResult[];
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import { getUserAgentsDir } from "../state.js";
|
|
4
|
+
import { parseRunResult, validateRunId } from "./schema.js";
|
|
5
|
+
export function benchHistoryDir() {
|
|
6
|
+
return path.join(getUserAgentsDir(), ".history", "bench");
|
|
7
|
+
}
|
|
8
|
+
export function saveRun(result, root = benchHistoryDir()) {
|
|
9
|
+
parseRunResult(result);
|
|
10
|
+
validateRunId(result.run_id);
|
|
11
|
+
fs.mkdirSync(root, { recursive: true });
|
|
12
|
+
const target = path.join(root, `${result.run_id}.json`);
|
|
13
|
+
const temporary = `${target}.${process.pid}.tmp`;
|
|
14
|
+
fs.writeFileSync(temporary, `${JSON.stringify(result, null, 2)}\n`, {
|
|
15
|
+
mode: 0o600,
|
|
16
|
+
});
|
|
17
|
+
fs.renameSync(temporary, target);
|
|
18
|
+
return target;
|
|
19
|
+
}
|
|
20
|
+
export function loadRun(runId, root = benchHistoryDir()) {
|
|
21
|
+
validateRunId(runId);
|
|
22
|
+
return parseRunResult(JSON.parse(fs.readFileSync(path.join(root, `${runId}.json`), "utf8")));
|
|
23
|
+
}
|
|
24
|
+
export function listRuns(root = benchHistoryDir()) {
|
|
25
|
+
if (!fs.existsSync(root))
|
|
26
|
+
return [];
|
|
27
|
+
return fs
|
|
28
|
+
.readdirSync(root)
|
|
29
|
+
.filter((name) => name.endsWith(".json"))
|
|
30
|
+
.map((name) => parseRunResult(JSON.parse(fs.readFileSync(path.join(root, name), "utf8"))))
|
|
31
|
+
.sort((a, b) => b.started_at.localeCompare(a.started_at));
|
|
32
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export type BenchPassCriterion = {
|
|
2
|
+
type: "file_exists";
|
|
3
|
+
path: string;
|
|
4
|
+
} | {
|
|
5
|
+
type: "exit_zero";
|
|
6
|
+
} | {
|
|
7
|
+
type: "stdout_contains";
|
|
8
|
+
value: string;
|
|
9
|
+
} | {
|
|
10
|
+
type: "command_succeeds";
|
|
11
|
+
command: string;
|
|
12
|
+
};
|
|
13
|
+
export interface BenchTask {
|
|
14
|
+
id: string;
|
|
15
|
+
prompt: string;
|
|
16
|
+
pass: BenchPassCriterion[];
|
|
17
|
+
fixtureDir: string;
|
|
18
|
+
sourcePath: string;
|
|
19
|
+
}
|
|
20
|
+
export interface BenchCell {
|
|
21
|
+
agent: string;
|
|
22
|
+
model?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface BenchCellResult extends BenchCell {
|
|
25
|
+
status: "passed" | "failed";
|
|
26
|
+
exit: number | null;
|
|
27
|
+
wall_ms: number;
|
|
28
|
+
tokens?: number;
|
|
29
|
+
stdout: string;
|
|
30
|
+
stderr: string;
|
|
31
|
+
}
|
|
32
|
+
export interface BenchRunResult {
|
|
33
|
+
schema_version: 1;
|
|
34
|
+
run_id: string;
|
|
35
|
+
task_id?: string;
|
|
36
|
+
prompt: string;
|
|
37
|
+
started_at: string;
|
|
38
|
+
finished_at: string;
|
|
39
|
+
cells: BenchCellResult[];
|
|
40
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
/** True only for a clean, single-line `sk-ant-oat01-…` token — see {@link SETUP_TOKEN_RE}. */
|
|
2
|
+
export declare function isValidClaudeSetupToken(value: string): boolean;
|
|
1
3
|
/** The per-account key an email maps to inside the `auth` bundle. */
|
|
2
4
|
export declare function claudeAccountTokenKey(account: string): string;
|
|
3
5
|
/** Signed-in account email for a version home, from `.claude.json` (no keychain). */
|
|
@@ -2,6 +2,8 @@ import * as fs from 'fs';
|
|
|
2
2
|
import * as os from 'os';
|
|
3
3
|
import * as path from 'path';
|
|
4
4
|
import { bundleBackend, bundleExists, readAndResolveBundleEnv } from './secrets/bundles.js';
|
|
5
|
+
import { fileStoreItemPath } from './secrets/filestore.js';
|
|
6
|
+
import { secretsKeychainItem } from './secrets/index.js';
|
|
5
7
|
/**
|
|
6
8
|
* Reserved FILE-BASED secrets bundle holding long-lived, non-rotating Claude
|
|
7
9
|
* setup-tokens. Usage/probe reads authenticate with these instead of Claude
|
|
@@ -11,6 +13,35 @@ import { bundleBackend, bundleExists, readAndResolveBundleEnv } from './secrets/
|
|
|
11
13
|
* multi-account fleet.
|
|
12
14
|
*/
|
|
13
15
|
const AUTH_BUNDLE = 'auth';
|
|
16
|
+
/**
|
|
17
|
+
* A well-formed Claude OAuth setup-token: the `sk-ant-oat01-` prefix followed by
|
|
18
|
+
* token-safe characters only, on a single line. `claude setup-token` mints exactly
|
|
19
|
+
* this; nothing else is a token.
|
|
20
|
+
*
|
|
21
|
+
* The provisioning capture bug behind #1767 stored the raw `claude setup-token`
|
|
22
|
+
* TTY stream — the welcome banner, ANSI control sequences, box-drawing art, and the
|
|
23
|
+
* token buried inside — under a per-account key instead of the parsed value. That
|
|
24
|
+
* blob is not a token: injected as `Authorization: Bearer <blob>` it made Anthropic
|
|
25
|
+
* reject every request and the run crash. agents-cli only *consumes* these tokens
|
|
26
|
+
* (the mint itself is the Rush Cloud / mint-auth path), so this is the boundary
|
|
27
|
+
* where a corrupt bundle entry must be caught before it reaches the auth header.
|
|
28
|
+
*/
|
|
29
|
+
const SETUP_TOKEN_RE = /^sk-ant-oat01-[A-Za-z0-9_-]+$/;
|
|
30
|
+
/** Process-local only: plaintext setup-tokens are never written to another cache. */
|
|
31
|
+
const setupTokenCache = new Map();
|
|
32
|
+
function credentialFingerprint(credentialPath) {
|
|
33
|
+
try {
|
|
34
|
+
const stat = fs.statSync(credentialPath, { bigint: true });
|
|
35
|
+
return `${stat.dev}:${stat.ino}:${stat.ctimeNs}:${stat.mtimeNs}:${stat.size}`;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return 'missing';
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** True only for a clean, single-line `sk-ant-oat01-…` token — see {@link SETUP_TOKEN_RE}. */
|
|
42
|
+
export function isValidClaudeSetupToken(value) {
|
|
43
|
+
return SETUP_TOKEN_RE.test(value);
|
|
44
|
+
}
|
|
14
45
|
/** The per-account key an email maps to inside the `auth` bundle. */
|
|
15
46
|
export function claudeAccountTokenKey(account) {
|
|
16
47
|
const slug = account
|
|
@@ -53,9 +84,31 @@ export function resolveClaudeSetupToken(home) {
|
|
|
53
84
|
return null;
|
|
54
85
|
if (!bundleExists(AUTH_BUNDLE) || bundleBackend(AUTH_BUNDLE) !== 'file')
|
|
55
86
|
return null;
|
|
56
|
-
const
|
|
57
|
-
const
|
|
58
|
-
|
|
87
|
+
const cacheKey = home ?? os.homedir();
|
|
88
|
+
const item = secretsKeychainItem(AUTH_BUNDLE, claudeAccountTokenKey(email));
|
|
89
|
+
const credentialPath = fileStoreItemPath(item);
|
|
90
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
91
|
+
const before = credentialFingerprint(credentialPath);
|
|
92
|
+
const cached = setupTokenCache.get(cacheKey);
|
|
93
|
+
if (cached?.credentialPath === credentialPath && cached.fingerprint === before) {
|
|
94
|
+
return cached.token;
|
|
95
|
+
}
|
|
96
|
+
if (before === 'missing') {
|
|
97
|
+
setupTokenCache.set(cacheKey, { credentialPath, fingerprint: before, token: null });
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
const { env } = readAndResolveBundleEnv(AUTH_BUNDLE, { caller: 'usage', agentOnly: true });
|
|
101
|
+
const v = (env[claudeAccountTokenKey(email)] ?? '').trim();
|
|
102
|
+
const token = v.length > 0 && isValidClaudeSetupToken(v) ? v : null;
|
|
103
|
+
const after = credentialFingerprint(credentialPath);
|
|
104
|
+
if (before === after) {
|
|
105
|
+
setupTokenCache.set(cacheKey, { credentialPath, fingerprint: after, token });
|
|
106
|
+
return token;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// The credential changed during both decrypt attempts. Fail closed rather
|
|
110
|
+
// than return a token whose current file fingerprint was never observed.
|
|
111
|
+
return null;
|
|
59
112
|
}
|
|
60
113
|
catch {
|
|
61
114
|
return null;
|
package/dist/lib/cloud/host.js
CHANGED
|
@@ -153,6 +153,7 @@ export class HostCloudProvider {
|
|
|
153
153
|
remoteExit: task.remoteExit,
|
|
154
154
|
taskId: task.id,
|
|
155
155
|
offset,
|
|
156
|
+
extraSshArgs: task.identityFile ? ['-i', task.identityFile, '-o', 'IdentitiesOnly=yes'] : [],
|
|
156
157
|
});
|
|
157
158
|
if (fetched) {
|
|
158
159
|
if (fetched.logChunk.length > 0) {
|
|
@@ -186,6 +186,8 @@ export interface CrabboxRunOptions extends CrabboxOptions {
|
|
|
186
186
|
onData?: (chunk: string) => void;
|
|
187
187
|
/** Force a full remote resync before running. */
|
|
188
188
|
fullResync?: boolean;
|
|
189
|
+
/** Refresh an existing lease with this idle window before running. */
|
|
190
|
+
renewIdleTimeoutSecs?: number;
|
|
189
191
|
}
|
|
190
192
|
/**
|
|
191
193
|
* Run `remoteCmd` on the leased box via `crabbox run` (crabbox syncs the dirty
|
package/dist/lib/crabbox/cli.js
CHANGED
|
@@ -485,6 +485,8 @@ export function crabboxRun(slug, remoteCmd, opts = {}) {
|
|
|
485
485
|
export function crabboxRunScript(slug, script, opts = {}) {
|
|
486
486
|
findCrabbox();
|
|
487
487
|
const args = ['run', '--id', slug, '--reclaim'];
|
|
488
|
+
if (opts.renewIdleTimeoutSecs !== undefined)
|
|
489
|
+
args.push('--idle-timeout', `${opts.renewIdleTimeoutSecs}s`);
|
|
488
490
|
if (opts.fullResync)
|
|
489
491
|
args.push('--full-resync');
|
|
490
492
|
args.push('--script-stdin');
|
|
@@ -233,7 +233,9 @@ export function isExpiredPoolStray(box, opts) {
|
|
|
233
233
|
return false;
|
|
234
234
|
if (box.expiresAt === null || box.expiresAt > nowSecs)
|
|
235
235
|
return false; // unexpired ⇒ reusable
|
|
236
|
-
if (box.lastTouchedAt
|
|
236
|
+
if (box.lastTouchedAt === null)
|
|
237
|
+
return false; // unknown age is never reap-safe
|
|
238
|
+
if (nowSecs - box.lastTouchedAt < graceSecs)
|
|
237
239
|
return false; // maybe active
|
|
238
240
|
return true;
|
|
239
241
|
}
|
|
@@ -329,9 +331,13 @@ export async function leaseAndRun(opts) {
|
|
|
329
331
|
let exitCode = null;
|
|
330
332
|
let toreDown = false;
|
|
331
333
|
try {
|
|
334
|
+
const renewIdleTimeoutSecs = reused && !opts.reuseBox && box.idleTimeoutSecs !== null
|
|
335
|
+
? box.idleTimeoutSecs
|
|
336
|
+
: undefined;
|
|
332
337
|
exitCode = await crabboxRunScript(box.slug, script, {
|
|
333
338
|
secretsBundle: opts.secretsBundle,
|
|
334
339
|
onData: opts.onData,
|
|
340
|
+
renewIdleTimeoutSecs,
|
|
335
341
|
});
|
|
336
342
|
}
|
|
337
343
|
finally {
|