@phnx-labs/agents-cli 1.22.25 → 1.22.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +427 -0
- package/README.md +35 -3
- package/dist/bin/agents +0 -0
- package/dist/browser.js +14 -4
- package/dist/commands/apply.js +52 -8
- package/dist/commands/bench.d.ts +2 -0
- package/dist/commands/bench.js +101 -0
- package/dist/commands/browser.js +35 -0
- package/dist/commands/doctor.js +32 -10
- package/dist/commands/exec.js +17 -16
- package/dist/commands/insights.d.ts +25 -19
- package/dist/commands/insights.js +146 -38
- package/dist/commands/output.js +100 -25
- package/dist/commands/reconnect.d.ts +46 -0
- package/dist/commands/reconnect.js +109 -0
- package/dist/commands/resume.d.ts +11 -0
- package/dist/commands/resume.js +51 -0
- package/dist/commands/routines.js +2 -2
- package/dist/commands/secrets.d.ts +2 -8
- package/dist/commands/secrets.js +29 -105
- package/dist/commands/sessions-picker.js +12 -7
- package/dist/commands/sessions.d.ts +47 -1
- package/dist/commands/sessions.js +349 -37
- package/dist/commands/setup-secrets.d.ts +1 -0
- package/dist/commands/setup-secrets.js +1 -1
- package/dist/commands/setup.d.ts +26 -3
- package/dist/commands/setup.js +105 -46
- package/dist/commands/ssh.js +184 -11
- package/dist/commands/teams.d.ts +6 -0
- package/dist/commands/teams.js +46 -3
- package/dist/commands/trends.d.ts +8 -0
- package/dist/commands/trends.js +10 -156
- package/dist/index.js +17 -21
- package/dist/lib/agents.d.ts +11 -0
- package/dist/lib/agents.js +29 -2
- package/dist/lib/analytics/dashboard.d.ts +10 -6
- package/dist/lib/analytics/dashboard.js +6 -4
- package/dist/lib/analytics/mix-commands.d.ts +53 -0
- package/dist/lib/analytics/mix-commands.js +229 -0
- package/dist/lib/analytics/recipes.d.ts +19 -14
- package/dist/lib/analytics/recipes.js +4 -2
- package/dist/lib/auth-health.d.ts +47 -3
- package/dist/lib/auth-health.js +74 -15
- package/dist/lib/bench/index.d.ts +4 -0
- package/dist/lib/bench/index.js +4 -0
- package/dist/lib/bench/runner.d.ts +16 -0
- package/dist/lib/bench/runner.js +111 -0
- package/dist/lib/bench/schema.d.ts +5 -0
- package/dist/lib/bench/schema.js +91 -0
- package/dist/lib/bench/storage.d.ts +5 -0
- package/dist/lib/bench/storage.js +32 -0
- package/dist/lib/bench/types.d.ts +40 -0
- package/dist/lib/bench/types.js +1 -0
- package/dist/lib/browser/ipc.d.ts +26 -0
- package/dist/lib/browser/ipc.js +139 -24
- package/dist/lib/browser/profiles.d.ts +11 -0
- package/dist/lib/browser/profiles.js +1 -1
- package/dist/lib/browser/stream.d.ts +14 -0
- package/dist/lib/browser/stream.js +71 -0
- package/dist/lib/channels/owner-sink.d.ts +27 -0
- package/dist/lib/channels/owner-sink.js +93 -0
- package/dist/lib/claude-account-token.d.ts +2 -0
- package/dist/lib/claude-account-token.js +27 -1
- package/dist/lib/crabbox/cli.d.ts +2 -0
- package/dist/lib/crabbox/cli.js +2 -0
- package/dist/lib/crabbox/lease.js +7 -1
- package/dist/lib/daemon.js +20 -0
- package/dist/lib/devices/connect.d.ts +2 -0
- package/dist/lib/devices/connect.js +7 -0
- package/dist/lib/devices/doctor-findings.d.ts +10 -1
- package/dist/lib/devices/doctor-findings.js +47 -1
- package/dist/lib/devices/harness-inventory.d.ts +97 -0
- package/dist/lib/devices/harness-inventory.js +0 -0
- package/dist/lib/devices/registry.d.ts +2 -0
- package/dist/lib/devices/ssh-config.js +3 -0
- package/dist/lib/devices/windows-ssh-enrollment.d.ts +20 -0
- package/dist/lib/devices/windows-ssh-enrollment.js +98 -0
- package/dist/lib/exec.bench.d.ts +1 -0
- package/dist/lib/exec.bench.js +186 -0
- package/dist/lib/exec.js +18 -8
- package/dist/lib/fleet/apply.d.ts +59 -3
- package/dist/lib/fleet/apply.js +191 -12
- package/dist/lib/fleet/remote-login.d.ts +4 -3
- package/dist/lib/fleet/remote-login.js +11 -9
- package/dist/lib/fleet/types.d.ts +21 -2
- package/dist/lib/gemini-settings.d.ts +0 -1
- package/dist/lib/gemini-settings.js +12 -7
- package/dist/lib/hooks/cache.js +15 -0
- package/dist/lib/hooks.d.ts +8 -5
- package/dist/lib/hooks.js +10 -6
- package/dist/lib/hosts/dispatch.js +7 -3
- package/dist/lib/hosts/passthrough.d.ts +45 -0
- package/dist/lib/hosts/passthrough.js +59 -8
- package/dist/lib/hosts/progress.d.ts +2 -1
- package/dist/lib/hosts/progress.js +4 -3
- package/dist/lib/hosts/providers/devices.js +1 -0
- package/dist/lib/hosts/ready.d.ts +6 -2
- package/dist/lib/hosts/ready.js +33 -8
- package/dist/lib/hosts/reconcile.d.ts +1 -1
- package/dist/lib/hosts/reconcile.js +16 -7
- package/dist/lib/hosts/reconnect.d.ts +56 -21
- package/dist/lib/hosts/reconnect.js +128 -71
- package/dist/lib/hosts/registry.d.ts +2 -2
- package/dist/lib/hosts/registry.js +3 -5
- package/dist/lib/hosts/remote-cmd.d.ts +17 -0
- package/dist/lib/hosts/remote-cmd.js +29 -0
- package/dist/lib/hosts/tasks.d.ts +2 -0
- package/dist/lib/hosts/types.d.ts +1 -0
- package/dist/lib/hosts/types.js +3 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/CodeResources +0 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/MacOS/MenubarHelper +0 -0
- package/dist/lib/menubar/install-menubar.d.ts +9 -6
- package/dist/lib/menubar/install-menubar.js +20 -9
- package/dist/lib/models.d.ts +30 -1
- package/dist/lib/models.js +42 -144
- package/dist/lib/pricing/cost.d.ts +9 -0
- package/dist/lib/pricing/cost.js +24 -0
- package/dist/lib/pricing/index.d.ts +1 -1
- package/dist/lib/pricing/index.js +1 -1
- package/dist/lib/redact.js +8 -3
- package/dist/lib/remote-agents-json.d.ts +32 -0
- package/dist/lib/remote-agents-json.js +47 -16
- package/dist/lib/resource-profiles.js +1 -2
- package/dist/lib/routine-notify-owner.d.ts +102 -0
- package/dist/lib/routine-notify-owner.js +232 -0
- package/dist/lib/routines.d.ts +10 -0
- package/dist/lib/routines.js +14 -2
- package/dist/lib/runner.d.ts +9 -4
- package/dist/lib/runner.js +182 -30
- package/dist/lib/sandbox.d.ts +0 -2
- package/dist/lib/sandbox.js +2 -19
- package/dist/lib/secrets/Agents CLI.app/Contents/CodeResources +0 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/MacOS/Agents CLI +0 -0
- package/dist/lib/secrets/agent.js +3 -2
- package/dist/lib/secrets/lease.d.ts +25 -0
- package/dist/lib/secrets/lease.js +44 -0
- package/dist/lib/secrets/push.d.ts +94 -0
- package/dist/lib/secrets/push.js +145 -0
- package/dist/lib/secrets/reaper.d.ts +15 -1
- package/dist/lib/secrets/reaper.js +30 -3
- package/dist/lib/self-update.d.ts +20 -5
- package/dist/lib/self-update.js +93 -16
- package/dist/lib/session/db.d.ts +38 -3
- package/dist/lib/session/db.js +273 -16
- package/dist/lib/session/discover.d.ts +22 -1
- package/dist/lib/session/discover.js +239 -51
- package/dist/lib/session/insights.d.ts +37 -0
- package/dist/lib/session/insights.js +219 -9
- package/dist/lib/session/recovery.d.ts +17 -1
- package/dist/lib/session/recovery.js +111 -4
- package/dist/lib/session/remote-list.d.ts +10 -9
- package/dist/lib/session/remote-list.js +25 -23
- package/dist/lib/session/resume-owner.d.ts +55 -0
- package/dist/lib/session/resume-owner.js +69 -0
- package/dist/lib/session/team-filter.d.ts +65 -0
- package/dist/lib/session/team-filter.js +98 -3
- package/dist/lib/session/tool-index.js +133 -22
- package/dist/lib/session/tool-store.d.ts +26 -2
- package/dist/lib/session/tool-store.js +36 -17
- package/dist/lib/session/types.d.ts +33 -0
- package/dist/lib/smart-launch.d.ts +31 -5
- package/dist/lib/smart-launch.js +43 -7
- package/dist/lib/ssh-exec.d.ts +27 -0
- package/dist/lib/ssh-exec.js +42 -3
- package/dist/lib/ssh-tunnel.d.ts +3 -2
- package/dist/lib/ssh-tunnel.js +25 -16
- package/dist/lib/startup/command-registry.d.ts +2 -0
- package/dist/lib/startup/command-registry.js +6 -0
- package/dist/lib/teams/agents.d.ts +13 -0
- package/dist/lib/teams/agents.js +75 -7
- package/dist/lib/teams/placement-probe.d.ts +21 -0
- package/dist/lib/teams/placement-probe.js +135 -0
- package/dist/lib/teams/scheduler.d.ts +74 -1
- package/dist/lib/teams/scheduler.js +187 -10
- package/dist/lib/tmux/session.d.ts +8 -0
- package/dist/lib/tmux/session.js +22 -0
- package/dist/lib/types.d.ts +2 -0
- package/package.json +1 -1
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { type UsageKind } from './usage-db.js';
|
|
2
|
-
|
|
2
|
+
/** Time window for mix recipes under `agents insights mix`. */
|
|
3
|
+
export interface AnalyticsWindow {
|
|
3
4
|
days: number;
|
|
4
5
|
sinceIso: string;
|
|
5
6
|
}
|
|
7
|
+
/** @deprecated Use AnalyticsWindow — kept as a type alias for external callers. */
|
|
8
|
+
export type TrendsWindow = AnalyticsWindow;
|
|
6
9
|
export interface RecipeSection {
|
|
7
10
|
id: string;
|
|
8
11
|
title: string;
|
|
@@ -10,23 +13,25 @@ export interface RecipeSection {
|
|
|
10
13
|
rows: Array<Record<string, string | number | null>>;
|
|
11
14
|
empty?: boolean;
|
|
12
15
|
}
|
|
13
|
-
export declare function
|
|
14
|
-
|
|
15
|
-
export declare
|
|
16
|
-
export declare function
|
|
17
|
-
export declare function
|
|
18
|
-
export declare function
|
|
19
|
-
export declare function
|
|
20
|
-
export declare function
|
|
21
|
-
export declare function
|
|
16
|
+
export declare function analyticsWindow(days: number): AnalyticsWindow;
|
|
17
|
+
/** @deprecated Use analyticsWindow. */
|
|
18
|
+
export declare const trendsWindow: typeof analyticsWindow;
|
|
19
|
+
export declare function recipeHarnessMix(win: AnalyticsWindow): RecipeSection;
|
|
20
|
+
export declare function recipeModelMix(win: AnalyticsWindow): RecipeSection;
|
|
21
|
+
export declare function recipeToolsPerSession(win: AnalyticsWindow): RecipeSection;
|
|
22
|
+
export declare function recipeTokenRatio(win: AnalyticsWindow): RecipeSection;
|
|
23
|
+
export declare function recipeSessionVolume(win: AnalyticsWindow): RecipeSection;
|
|
24
|
+
export declare function recipeSecretsHot(win: AnalyticsWindow): RecipeSection;
|
|
25
|
+
export declare function recipeBrowserActivity(win: AnalyticsWindow): RecipeSection;
|
|
26
|
+
export declare function recipeResourceMix(win: AnalyticsWindow): RecipeSection;
|
|
22
27
|
export type RecipeId = 'harness-mix' | 'model-mix' | 'tools-per-session' | 'token-ratio' | 'session-volume' | 'secrets-hot' | 'browser-activity' | 'resource-mix';
|
|
23
28
|
export declare const RECIPE_IDS: readonly RecipeId[];
|
|
24
|
-
export declare function runRecipe(id: RecipeId, win:
|
|
29
|
+
export declare function runRecipe(id: RecipeId, win: AnalyticsWindow): RecipeSection;
|
|
25
30
|
export declare function listRecipes(): Array<{
|
|
26
31
|
id: RecipeId;
|
|
27
32
|
title: string;
|
|
28
33
|
store: 'sessions' | 'usage';
|
|
29
34
|
}>;
|
|
30
|
-
export declare function hasSessionsInWindow(win:
|
|
31
|
-
export declare function usageKindsPresent(win:
|
|
32
|
-
export declare function usageEventCount(kind: UsageKind, win:
|
|
35
|
+
export declare function hasSessionsInWindow(win: AnalyticsWindow): boolean;
|
|
36
|
+
export declare function usageKindsPresent(win: AnalyticsWindow): UsageKind[];
|
|
37
|
+
export declare function usageEventCount(kind: UsageKind, win: AnalyticsWindow): number;
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import Database from '../sqlite.js';
|
|
2
2
|
import { getSessionsDbPath } from '../state.js';
|
|
3
3
|
import { topNamesByKind, kindMix, countUsage, listUsageKindsWithData, } from './usage-db.js';
|
|
4
|
-
export function
|
|
4
|
+
export function analyticsWindow(days) {
|
|
5
5
|
const d = Number.isFinite(days) && days > 0 ? days : 7;
|
|
6
6
|
const since = new Date(Date.now() - d * 24 * 60 * 60 * 1000);
|
|
7
7
|
return { days: d, sinceIso: since.toISOString() };
|
|
8
8
|
}
|
|
9
|
+
/** @deprecated Use analyticsWindow. */
|
|
10
|
+
export const trendsWindow = analyticsWindow;
|
|
9
11
|
function openSessions() {
|
|
10
12
|
try {
|
|
11
13
|
const db = new Database(getSessionsDbPath());
|
|
@@ -290,7 +292,7 @@ export function runRecipe(id, win) {
|
|
|
290
292
|
return RUNNERS[id](win);
|
|
291
293
|
}
|
|
292
294
|
export function listRecipes() {
|
|
293
|
-
const win =
|
|
295
|
+
const win = analyticsWindow(7);
|
|
294
296
|
return RECIPE_IDS.map((id) => {
|
|
295
297
|
const s = RUNNERS[id](win);
|
|
296
298
|
return { id, title: s.title, store: s.store };
|
|
@@ -156,10 +156,54 @@ export interface AuthProbeRow {
|
|
|
156
156
|
account?: string;
|
|
157
157
|
health: AuthHealth;
|
|
158
158
|
}
|
|
159
|
+
/** One installed (agent, version) home, tagged with its resolved account label. */
|
|
160
|
+
export interface FleetAuthInstall {
|
|
161
|
+
agent: AgentId;
|
|
162
|
+
version: string;
|
|
163
|
+
/** Human account label from {@link authAccountLabel}, or undefined when none resolves. */
|
|
164
|
+
account: string | undefined;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* A set of installs that share one provider account and MUST be probed once.
|
|
168
|
+
* The live probe runs against `probe` (the representative home); every entry in
|
|
169
|
+
* `members` — the representative included — then receives that one verdict.
|
|
170
|
+
*/
|
|
171
|
+
export interface FleetAuthProbeGroup<T extends FleetAuthInstall> {
|
|
172
|
+
probe: T;
|
|
173
|
+
members: T[];
|
|
174
|
+
}
|
|
159
175
|
/**
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
176
|
+
* Collapse installs so a live auth probe fires ONCE per (agent, account) rather
|
|
177
|
+
* than once per version home.
|
|
178
|
+
*
|
|
179
|
+
* Several version homes signed into the same provider account share one OAuth
|
|
180
|
+
* rate limit, so probing each of them concurrently — which the daemon did every
|
|
181
|
+
* three minutes across every installed home (RUSH-2111) — raced that limit into
|
|
182
|
+
* a 429 storm that then parked the whole box behind a `Retry-After` penalty (the
|
|
183
|
+
* incident {@link file:./usage-backoff.ts} was written to survive; this removes
|
|
184
|
+
* its cause). Grouping by account means N homes on one account issue ONE request.
|
|
185
|
+
*
|
|
186
|
+
* `isMergeable` scopes the dedup to the installs it actually helps: only agents
|
|
187
|
+
* that make a network probe ({@link LIVE_PROBE_AGENTS}) can 429, so only they are
|
|
188
|
+
* merged. A best-effort agent (its verdict is a cheap local file read, no rate
|
|
189
|
+
* limit) is left per-install — collapsing it would gain nothing and could
|
|
190
|
+
* silently override one home's local `signedIn` verdict with another's. Installs
|
|
191
|
+
* with no resolvable account label, or that `isMergeable` rejects, are never
|
|
192
|
+
* merged: each becomes its own group keyed by version, preserving the old
|
|
193
|
+
* per-install probe (also the `unconfigured` case dropped downstream). Pure: no
|
|
194
|
+
* fs, no network, so the dedup decision is unit-tested directly.
|
|
195
|
+
*/
|
|
196
|
+
export declare function groupFleetAuthInstalls<T extends FleetAuthInstall>(installs: readonly T[], isMergeable?: (install: T) => boolean): FleetAuthProbeGroup<T>[];
|
|
197
|
+
/**
|
|
198
|
+
* Enumerate every installed (agent, version) on THIS host and return one row per
|
|
199
|
+
* install (installs with no credential at all are dropped). Shared by
|
|
200
|
+
* `agents fleet ping --local` and the daemon refresh.
|
|
201
|
+
*
|
|
202
|
+
* The live network probe is deduped by account: homes sharing one provider
|
|
203
|
+
* account are probed ONCE and the verdict is fanned out to each home's row, so
|
|
204
|
+
* the daemon's every-3-minute refresh can no longer fire concurrent same-account
|
|
205
|
+
* requests and self-inflict a 429 (RUSH-2111). Each home still gets its own cache
|
|
206
|
+
* row (the cache key is per-version by design — see {@link authCacheKey}).
|
|
163
207
|
*/
|
|
164
208
|
export declare function probeLocalFleetAuth(opts?: {
|
|
165
209
|
cliVersion?: string | null;
|
package/dist/lib/auth-health.js
CHANGED
|
@@ -283,28 +283,87 @@ export async function probeAuthHealth(agent, home, opts) {
|
|
|
283
283
|
return { verdict: info?.signedIn ? 'unverified' : 'unconfigured', checkedAt };
|
|
284
284
|
}
|
|
285
285
|
/**
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
286
|
+
* Collapse installs so a live auth probe fires ONCE per (agent, account) rather
|
|
287
|
+
* than once per version home.
|
|
288
|
+
*
|
|
289
|
+
* Several version homes signed into the same provider account share one OAuth
|
|
290
|
+
* rate limit, so probing each of them concurrently — which the daemon did every
|
|
291
|
+
* three minutes across every installed home (RUSH-2111) — raced that limit into
|
|
292
|
+
* a 429 storm that then parked the whole box behind a `Retry-After` penalty (the
|
|
293
|
+
* incident {@link file:./usage-backoff.ts} was written to survive; this removes
|
|
294
|
+
* its cause). Grouping by account means N homes on one account issue ONE request.
|
|
295
|
+
*
|
|
296
|
+
* `isMergeable` scopes the dedup to the installs it actually helps: only agents
|
|
297
|
+
* that make a network probe ({@link LIVE_PROBE_AGENTS}) can 429, so only they are
|
|
298
|
+
* merged. A best-effort agent (its verdict is a cheap local file read, no rate
|
|
299
|
+
* limit) is left per-install — collapsing it would gain nothing and could
|
|
300
|
+
* silently override one home's local `signedIn` verdict with another's. Installs
|
|
301
|
+
* with no resolvable account label, or that `isMergeable` rejects, are never
|
|
302
|
+
* merged: each becomes its own group keyed by version, preserving the old
|
|
303
|
+
* per-install probe (also the `unconfigured` case dropped downstream). Pure: no
|
|
304
|
+
* fs, no network, so the dedup decision is unit-tested directly.
|
|
305
|
+
*/
|
|
306
|
+
export function groupFleetAuthInstalls(installs, isMergeable = () => true) {
|
|
307
|
+
const groups = new Map();
|
|
308
|
+
for (const inst of installs) {
|
|
309
|
+
// The `acct:` / `ver:` tokens make the two branches disjoint, so an account
|
|
310
|
+
// label can never collide with a version fallback key no matter its content.
|
|
311
|
+
const key = inst.account && isMergeable(inst)
|
|
312
|
+
? `${inst.agent} acct:${inst.account}`
|
|
313
|
+
: `${inst.agent} ver:${inst.version}`;
|
|
314
|
+
const existing = groups.get(key);
|
|
315
|
+
if (existing)
|
|
316
|
+
existing.members.push(inst);
|
|
317
|
+
else
|
|
318
|
+
groups.set(key, { probe: inst, members: [inst] });
|
|
319
|
+
}
|
|
320
|
+
return [...groups.values()];
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Enumerate every installed (agent, version) on THIS host and return one row per
|
|
324
|
+
* install (installs with no credential at all are dropped). Shared by
|
|
325
|
+
* `agents fleet ping --local` and the daemon refresh.
|
|
326
|
+
*
|
|
327
|
+
* The live network probe is deduped by account: homes sharing one provider
|
|
328
|
+
* account are probed ONCE and the verdict is fanned out to each home's row, so
|
|
329
|
+
* the daemon's every-3-minute refresh can no longer fire concurrent same-account
|
|
330
|
+
* requests and self-inflict a 429 (RUSH-2111). Each home still gets its own cache
|
|
331
|
+
* row (the cache key is per-version by design — see {@link authCacheKey}).
|
|
289
332
|
*/
|
|
290
333
|
export async function probeLocalFleetAuth(opts) {
|
|
291
334
|
const agentIds = opts?.agents ?? ALL_AGENT_IDS;
|
|
292
|
-
|
|
335
|
+
// Enumerate every install, then resolve its account label. getAccountInfo is a
|
|
336
|
+
// local credential-file read (no network), so this fan-out is cheap and cannot
|
|
337
|
+
// contribute to the rate limit the probe grouping below exists to avoid.
|
|
338
|
+
const installs = [];
|
|
293
339
|
for (const agent of agentIds) {
|
|
294
340
|
for (const version of listInstalledVersions(agent)) {
|
|
295
|
-
|
|
296
|
-
tasks.push((async () => {
|
|
297
|
-
const info = await getAccountInfo(agent, home).catch(() => null);
|
|
298
|
-
const health = await probeAuthHealth(agent, home, { cliVersion: opts?.cliVersion, info });
|
|
299
|
-
health.account = authAccountLabel(info);
|
|
300
|
-
if (health.verdict === 'unconfigured')
|
|
301
|
-
return null;
|
|
302
|
-
return { agent, version, account: health.account, health };
|
|
303
|
-
})());
|
|
341
|
+
installs.push({ agent, version, home: getVersionHomePath(agent, version), info: null, account: undefined });
|
|
304
342
|
}
|
|
305
343
|
}
|
|
306
|
-
|
|
307
|
-
|
|
344
|
+
await Promise.all(installs.map(async (inst) => {
|
|
345
|
+
inst.info = await getAccountInfo(inst.agent, inst.home).catch(() => null);
|
|
346
|
+
inst.account = authAccountLabel(inst.info);
|
|
347
|
+
}));
|
|
348
|
+
// Probe once per (agent, account) — but only for the network-probing agents
|
|
349
|
+
// that can actually 429; best-effort agents stay per-install (see
|
|
350
|
+
// groupFleetAuthInstalls). Groups run in parallel: they target distinct
|
|
351
|
+
// accounts, so no same-account concurrency is left to trip the throttle.
|
|
352
|
+
const perGroup = await Promise.all(groupFleetAuthInstalls(installs, (inst) => LIVE_PROBE_AGENTS.has(inst.agent)).map(async (group) => {
|
|
353
|
+
const rep = group.probe;
|
|
354
|
+
const health = await probeAuthHealth(rep.agent, rep.home, { cliVersion: opts?.cliVersion, info: rep.info });
|
|
355
|
+
health.account = authAccountLabel(rep.info);
|
|
356
|
+
if (health.verdict === 'unconfigured')
|
|
357
|
+
return [];
|
|
358
|
+
return group.members.map((inst) => ({
|
|
359
|
+
agent: inst.agent,
|
|
360
|
+
version: inst.version,
|
|
361
|
+
account: health.account,
|
|
362
|
+
// A distinct object per row so a later mutation of one can't bleed across.
|
|
363
|
+
health: { ...health },
|
|
364
|
+
}));
|
|
365
|
+
}));
|
|
366
|
+
return perGroup.flat();
|
|
308
367
|
}
|
|
309
368
|
/** Persist a host's probed rows into the cache (keyed by host+agent+version). */
|
|
310
369
|
export function writeFleetAuthRows(host, rows) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { BenchCell, BenchCellResult, BenchRunResult, BenchTask } from "./types.js";
|
|
2
|
+
export interface CellExecutionRequest extends BenchCell {
|
|
3
|
+
prompt: string;
|
|
4
|
+
cwd: string;
|
|
5
|
+
}
|
|
6
|
+
export type CellExecutor = (request: CellExecutionRequest) => Promise<Omit<BenchCellResult, keyof BenchCell | "wall_ms" | "status">>;
|
|
7
|
+
export declare function extractTokenCount(stdout: string, stderr: string): number | undefined;
|
|
8
|
+
export declare function executeCellViaAgentsRun(request: CellExecutionRequest): ReturnType<CellExecutor>;
|
|
9
|
+
export declare function runCells(options: {
|
|
10
|
+
task?: BenchTask;
|
|
11
|
+
prompt: string;
|
|
12
|
+
cells: BenchCell[];
|
|
13
|
+
concurrency: number;
|
|
14
|
+
execute?: CellExecutor;
|
|
15
|
+
tempRoot?: string;
|
|
16
|
+
}): Promise<BenchRunResult>;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { spawn } from "child_process";
|
|
2
|
+
import * as fs from "fs";
|
|
3
|
+
import * as os from "os";
|
|
4
|
+
import * as path from "path";
|
|
5
|
+
import { getCliLaunch } from "../cli-entry.js";
|
|
6
|
+
export function extractTokenCount(stdout, stderr) {
|
|
7
|
+
const matches = [
|
|
8
|
+
...`${stdout}\n${stderr}`.matchAll(/(?:tokens used|total tokens|token usage)\s*[:=]?\s*([\d,]+)/gi),
|
|
9
|
+
];
|
|
10
|
+
const raw = matches.at(-1)?.[1];
|
|
11
|
+
if (!raw)
|
|
12
|
+
return undefined;
|
|
13
|
+
const tokens = Number(raw.replaceAll(",", ""));
|
|
14
|
+
return Number.isSafeInteger(tokens) && tokens >= 0 ? tokens : undefined;
|
|
15
|
+
}
|
|
16
|
+
export async function executeCellViaAgentsRun(request) {
|
|
17
|
+
const args = [
|
|
18
|
+
"run",
|
|
19
|
+
request.agent,
|
|
20
|
+
request.prompt,
|
|
21
|
+
"--cwd",
|
|
22
|
+
request.cwd,
|
|
23
|
+
"--headless",
|
|
24
|
+
"--no-tmux",
|
|
25
|
+
];
|
|
26
|
+
if (request.model)
|
|
27
|
+
args.push("--model", request.model);
|
|
28
|
+
const launch = getCliLaunch(args);
|
|
29
|
+
return await new Promise((resolve, reject) => {
|
|
30
|
+
const child = spawn(launch.command, launch.args, {
|
|
31
|
+
cwd: request.cwd,
|
|
32
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
33
|
+
});
|
|
34
|
+
let stdout = "";
|
|
35
|
+
let stderr = "";
|
|
36
|
+
child.stdout.on("data", (chunk) => {
|
|
37
|
+
stdout += chunk.toString();
|
|
38
|
+
});
|
|
39
|
+
child.stderr.on("data", (chunk) => {
|
|
40
|
+
stderr += chunk.toString();
|
|
41
|
+
});
|
|
42
|
+
child.on("error", reject);
|
|
43
|
+
child.on("close", (code) => resolve({
|
|
44
|
+
exit: code,
|
|
45
|
+
stdout,
|
|
46
|
+
stderr,
|
|
47
|
+
tokens: extractTokenCount(stdout, stderr),
|
|
48
|
+
}));
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
function copyFixture(task, cellIndex, runRoot) {
|
|
52
|
+
const cwd = path.join(runRoot, `cell-${cellIndex}`);
|
|
53
|
+
fs.mkdirSync(cwd, { recursive: true });
|
|
54
|
+
if (fs.existsSync(task.fixtureDir))
|
|
55
|
+
fs.cpSync(task.fixtureDir, cwd, { recursive: true });
|
|
56
|
+
return cwd;
|
|
57
|
+
}
|
|
58
|
+
export async function runCells(options) {
|
|
59
|
+
if (!Number.isInteger(options.concurrency) || options.concurrency < 1)
|
|
60
|
+
throw new Error("concurrency must be a positive integer");
|
|
61
|
+
if (options.cells.length === 0)
|
|
62
|
+
throw new Error("at least one bench cell is required");
|
|
63
|
+
const runId = `${new Date().toISOString().replace(/[-:.TZ]/g, "")}-${process.pid}`;
|
|
64
|
+
const startedAt = new Date();
|
|
65
|
+
const runRoot = fs.mkdtempSync(path.join(options.tempRoot ?? os.tmpdir(), `agents-bench-${runId}-`));
|
|
66
|
+
const results = new Array(options.cells.length);
|
|
67
|
+
const execute = options.execute ?? executeCellViaAgentsRun;
|
|
68
|
+
let next = 0;
|
|
69
|
+
const worker = async () => {
|
|
70
|
+
for (;;) {
|
|
71
|
+
const index = next++;
|
|
72
|
+
if (index >= options.cells.length)
|
|
73
|
+
return;
|
|
74
|
+
const cell = options.cells[index];
|
|
75
|
+
const cwd = options.task
|
|
76
|
+
? copyFixture(options.task, index, runRoot)
|
|
77
|
+
: path.join(runRoot, `cell-${index}`);
|
|
78
|
+
fs.mkdirSync(cwd, { recursive: true });
|
|
79
|
+
const started = performance.now();
|
|
80
|
+
try {
|
|
81
|
+
const output = await execute({ ...cell, prompt: options.prompt, cwd });
|
|
82
|
+
results[index] = {
|
|
83
|
+
...cell,
|
|
84
|
+
...output,
|
|
85
|
+
wall_ms: Math.round(performance.now() - started),
|
|
86
|
+
status: output.exit === 0 ? "passed" : "failed",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
results[index] = {
|
|
91
|
+
...cell,
|
|
92
|
+
exit: null,
|
|
93
|
+
stdout: "",
|
|
94
|
+
stderr: error instanceof Error ? error.message : String(error),
|
|
95
|
+
wall_ms: Math.round(performance.now() - started),
|
|
96
|
+
status: "failed",
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
await Promise.all(Array.from({ length: Math.min(options.concurrency, options.cells.length) }, worker));
|
|
102
|
+
return {
|
|
103
|
+
schema_version: 1,
|
|
104
|
+
run_id: runId,
|
|
105
|
+
...(options.task ? { task_id: options.task.id } : {}),
|
|
106
|
+
prompt: options.prompt,
|
|
107
|
+
started_at: startedAt.toISOString(),
|
|
108
|
+
finished_at: new Date().toISOString(),
|
|
109
|
+
cells: results,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { BenchRunResult, BenchTask } from "./types.js";
|
|
2
|
+
export declare function parseTask(raw: unknown, sourcePath: string): BenchTask;
|
|
3
|
+
export declare function parseRunResult(raw: unknown): BenchRunResult;
|
|
4
|
+
export declare function validateRunId(runId: string): string;
|
|
5
|
+
export declare function loadTask(taskId: string, tasksRoot: string): BenchTask;
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
function object(value, label) {
|
|
4
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
5
|
+
throw new Error(`${label} must be an object`);
|
|
6
|
+
return value;
|
|
7
|
+
}
|
|
8
|
+
function string(value, label) {
|
|
9
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
10
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
export function parseTask(raw, sourcePath) {
|
|
14
|
+
const value = object(raw, "task.json");
|
|
15
|
+
const id = string(value.id, "task.id");
|
|
16
|
+
const prompt = string(value.prompt, "task.prompt");
|
|
17
|
+
if (!Array.isArray(value.pass) || value.pass.length === 0)
|
|
18
|
+
throw new Error("task.pass must be a non-empty array");
|
|
19
|
+
const pass = value.pass.map((item, index) => {
|
|
20
|
+
const criterion = object(item, `task.pass[${index}]`);
|
|
21
|
+
const type = string(criterion.type, `task.pass[${index}].type`);
|
|
22
|
+
if (type === "exit_zero")
|
|
23
|
+
return { type };
|
|
24
|
+
if (type === "file_exists")
|
|
25
|
+
return { type, path: string(criterion.path, `task.pass[${index}].path`) };
|
|
26
|
+
if (type === "stdout_contains")
|
|
27
|
+
return {
|
|
28
|
+
type,
|
|
29
|
+
value: string(criterion.value, `task.pass[${index}].value`),
|
|
30
|
+
};
|
|
31
|
+
if (type === "command_succeeds")
|
|
32
|
+
return {
|
|
33
|
+
type,
|
|
34
|
+
command: string(criterion.command, `task.pass[${index}].command`),
|
|
35
|
+
};
|
|
36
|
+
throw new Error(`task.pass[${index}].type is unsupported: ${type}`);
|
|
37
|
+
});
|
|
38
|
+
const taskDir = path.dirname(sourcePath);
|
|
39
|
+
return {
|
|
40
|
+
id,
|
|
41
|
+
prompt,
|
|
42
|
+
pass,
|
|
43
|
+
fixtureDir: path.join(taskDir, "fixture"),
|
|
44
|
+
sourcePath,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export function parseRunResult(raw) {
|
|
48
|
+
const value = object(raw, "bench result");
|
|
49
|
+
if (value.schema_version !== 1)
|
|
50
|
+
throw new Error("bench result schema_version must be 1");
|
|
51
|
+
string(value.run_id, "bench result run_id");
|
|
52
|
+
string(value.prompt, "bench result prompt");
|
|
53
|
+
string(value.started_at, "bench result started_at");
|
|
54
|
+
string(value.finished_at, "bench result finished_at");
|
|
55
|
+
if (value.task_id !== undefined)
|
|
56
|
+
string(value.task_id, "bench result task_id");
|
|
57
|
+
if (!Array.isArray(value.cells))
|
|
58
|
+
throw new Error("bench result cells must be an array");
|
|
59
|
+
value.cells.forEach((rawCell, index) => {
|
|
60
|
+
const cell = object(rawCell, `bench result cells[${index}]`);
|
|
61
|
+
string(cell.agent, `bench result cells[${index}].agent`);
|
|
62
|
+
if (cell.model !== undefined)
|
|
63
|
+
string(cell.model, `bench result cells[${index}].model`);
|
|
64
|
+
if (cell.status !== "passed" && cell.status !== "failed")
|
|
65
|
+
throw new Error(`bench result cells[${index}].status must be passed or failed`);
|
|
66
|
+
if (cell.exit !== null &&
|
|
67
|
+
(!Number.isInteger(cell.exit) || cell.exit < 0))
|
|
68
|
+
throw new Error(`bench result cells[${index}].exit must be a non-negative integer or null`);
|
|
69
|
+
if (typeof cell.wall_ms !== "number" ||
|
|
70
|
+
!Number.isFinite(cell.wall_ms) ||
|
|
71
|
+
cell.wall_ms < 0)
|
|
72
|
+
throw new Error(`bench result cells[${index}].wall_ms must be a non-negative number`);
|
|
73
|
+
if (cell.tokens !== undefined &&
|
|
74
|
+
(!Number.isSafeInteger(cell.tokens) || cell.tokens < 0))
|
|
75
|
+
throw new Error(`bench result cells[${index}].tokens must be a non-negative integer`);
|
|
76
|
+
if (typeof cell.stdout !== "string" || typeof cell.stderr !== "string")
|
|
77
|
+
throw new Error(`bench result cells[${index}] stdout and stderr must be strings`);
|
|
78
|
+
});
|
|
79
|
+
return value;
|
|
80
|
+
}
|
|
81
|
+
export function validateRunId(runId) {
|
|
82
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(runId))
|
|
83
|
+
throw new Error(`Invalid run id: ${runId}`);
|
|
84
|
+
return runId;
|
|
85
|
+
}
|
|
86
|
+
export function loadTask(taskId, tasksRoot) {
|
|
87
|
+
if (!/^[a-z0-9][a-z0-9-]*$/.test(taskId))
|
|
88
|
+
throw new Error(`Invalid task id: ${taskId}`);
|
|
89
|
+
const sourcePath = path.join(tasksRoot, taskId, "task.json");
|
|
90
|
+
return parseTask(JSON.parse(fs.readFileSync(sourcePath, "utf8")), sourcePath);
|
|
91
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { BenchRunResult } from "./types.js";
|
|
2
|
+
export declare function benchHistoryDir(): string;
|
|
3
|
+
export declare function saveRun(result: BenchRunResult, root?: string): string;
|
|
4
|
+
export declare function loadRun(runId: string, root?: string): BenchRunResult;
|
|
5
|
+
export declare function listRuns(root?: string): BenchRunResult[];
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import { getUserAgentsDir } from "../state.js";
|
|
4
|
+
import { parseRunResult, validateRunId } from "./schema.js";
|
|
5
|
+
export function benchHistoryDir() {
|
|
6
|
+
return path.join(getUserAgentsDir(), ".history", "bench");
|
|
7
|
+
}
|
|
8
|
+
export function saveRun(result, root = benchHistoryDir()) {
|
|
9
|
+
parseRunResult(result);
|
|
10
|
+
validateRunId(result.run_id);
|
|
11
|
+
fs.mkdirSync(root, { recursive: true });
|
|
12
|
+
const target = path.join(root, `${result.run_id}.json`);
|
|
13
|
+
const temporary = `${target}.${process.pid}.tmp`;
|
|
14
|
+
fs.writeFileSync(temporary, `${JSON.stringify(result, null, 2)}\n`, {
|
|
15
|
+
mode: 0o600,
|
|
16
|
+
});
|
|
17
|
+
fs.renameSync(temporary, target);
|
|
18
|
+
return target;
|
|
19
|
+
}
|
|
20
|
+
export function loadRun(runId, root = benchHistoryDir()) {
|
|
21
|
+
validateRunId(runId);
|
|
22
|
+
return parseRunResult(JSON.parse(fs.readFileSync(path.join(root, `${runId}.json`), "utf8")));
|
|
23
|
+
}
|
|
24
|
+
export function listRuns(root = benchHistoryDir()) {
|
|
25
|
+
if (!fs.existsSync(root))
|
|
26
|
+
return [];
|
|
27
|
+
return fs
|
|
28
|
+
.readdirSync(root)
|
|
29
|
+
.filter((name) => name.endsWith(".json"))
|
|
30
|
+
.map((name) => parseRunResult(JSON.parse(fs.readFileSync(path.join(root, name), "utf8"))))
|
|
31
|
+
.sort((a, b) => b.started_at.localeCompare(a.started_at));
|
|
32
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export type BenchPassCriterion = {
|
|
2
|
+
type: "file_exists";
|
|
3
|
+
path: string;
|
|
4
|
+
} | {
|
|
5
|
+
type: "exit_zero";
|
|
6
|
+
} | {
|
|
7
|
+
type: "stdout_contains";
|
|
8
|
+
value: string;
|
|
9
|
+
} | {
|
|
10
|
+
type: "command_succeeds";
|
|
11
|
+
command: string;
|
|
12
|
+
};
|
|
13
|
+
export interface BenchTask {
|
|
14
|
+
id: string;
|
|
15
|
+
prompt: string;
|
|
16
|
+
pass: BenchPassCriterion[];
|
|
17
|
+
fixtureDir: string;
|
|
18
|
+
sourcePath: string;
|
|
19
|
+
}
|
|
20
|
+
export interface BenchCell {
|
|
21
|
+
agent: string;
|
|
22
|
+
model?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface BenchCellResult extends BenchCell {
|
|
25
|
+
status: "passed" | "failed";
|
|
26
|
+
exit: number | null;
|
|
27
|
+
wall_ms: number;
|
|
28
|
+
tokens?: number;
|
|
29
|
+
stdout: string;
|
|
30
|
+
stderr: string;
|
|
31
|
+
}
|
|
32
|
+
export interface BenchRunResult {
|
|
33
|
+
schema_version: 1;
|
|
34
|
+
run_id: string;
|
|
35
|
+
task_id?: string;
|
|
36
|
+
prompt: string;
|
|
37
|
+
started_at: string;
|
|
38
|
+
finished_at: string;
|
|
39
|
+
cells: BenchCellResult[];
|
|
40
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as net from 'net';
|
|
1
2
|
import { BrowserService } from './service.js';
|
|
2
3
|
import type { IPCRequest, IPCResponse } from './types.js';
|
|
3
4
|
export interface IPCRequestOptions {
|
|
@@ -11,6 +12,25 @@ export declare function getSocketPath(): string;
|
|
|
11
12
|
/** Is the daemon reachable? A real connect probe on every platform — a socket
|
|
12
13
|
* file existing on disk is not proof a daemon is listening on it. */
|
|
13
14
|
export declare function isDaemonReachable(): Promise<boolean>;
|
|
15
|
+
/**
|
|
16
|
+
* One long-lived connection to the existing browser daemon. Requests are
|
|
17
|
+
* serialized so the daemon's newline-delimited responses always map to the
|
|
18
|
+
* caller that produced them, while the process and socket stay warm between
|
|
19
|
+
* actions.
|
|
20
|
+
*/
|
|
21
|
+
export declare class BrowserIPCConnection {
|
|
22
|
+
private readonly socket;
|
|
23
|
+
private buffer;
|
|
24
|
+
private pending;
|
|
25
|
+
private tail;
|
|
26
|
+
private closed;
|
|
27
|
+
constructor(socket: net.Socket);
|
|
28
|
+
request(request: IPCRequest): Promise<IPCResponse>;
|
|
29
|
+
close(): Promise<void>;
|
|
30
|
+
private requestOnce;
|
|
31
|
+
private handleData;
|
|
32
|
+
private fail;
|
|
33
|
+
}
|
|
14
34
|
export declare class BrowserIPCServer {
|
|
15
35
|
private server;
|
|
16
36
|
private service;
|
|
@@ -27,3 +47,9 @@ export declare class BrowserIPCServer {
|
|
|
27
47
|
*/
|
|
28
48
|
export declare function shouldRestartStaleDaemon(daemonVersion: string | undefined, clientVersion: string): boolean;
|
|
29
49
|
export declare function sendIPCRequest(request: IPCRequest, opts?: IPCRequestOptions): Promise<IPCResponse>;
|
|
50
|
+
/**
|
|
51
|
+
* Open one connection that can serve many browser operations. This uses the
|
|
52
|
+
* same daemon readiness and version-reconciliation path as sendIPCRequest;
|
|
53
|
+
* only the client process and IPC socket lifetime differ.
|
|
54
|
+
*/
|
|
55
|
+
export declare function connectBrowserIPC(opts?: IPCRequestOptions): Promise<BrowserIPCConnection>;
|