@prohost/cli 0.8.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/README.md +119 -2
- package/dist/agent/account_runtime.d.ts +19 -3
- package/dist/agent/account_runtime.js +28 -6
- package/dist/agent/accounts.d.ts +17 -1
- package/dist/agent/accounts.js +43 -2
- package/dist/agent/agent_commands.js +6 -3
- package/dist/agent/api.d.ts +6 -0
- package/dist/agent/api.js +10 -5
- package/dist/agent/claude.js +9 -3
- package/dist/agent/command.d.ts +9 -0
- package/dist/agent/command.js +64 -2
- package/dist/agent/daemon.d.ts +4 -0
- package/dist/agent/daemon.js +4 -0
- package/dist/agent/prompt.d.ts +27 -0
- package/dist/agent/prompt.js +21 -1
- package/dist/agent/run.d.ts +28 -3
- package/dist/agent/run.js +90 -18
- package/dist/agent/scheduler.d.ts +44 -0
- package/dist/agent/scheduler.js +93 -0
- package/dist/agent/worktrees.d.ts +136 -0
- package/dist/agent/worktrees.js +317 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +29 -2
- package/dist/usage/backoff.d.ts +42 -0
- package/dist/usage/backoff.js +85 -0
- package/dist/usage/command.d.ts +31 -0
- package/dist/usage/command.js +269 -0
- package/dist/usage/config.d.ts +38 -0
- package/dist/usage/config.js +75 -0
- package/dist/usage/discovery.d.ts +63 -0
- package/dist/usage/discovery.js +187 -0
- package/dist/usage/quota.d.ts +96 -0
- package/dist/usage/quota.js +145 -0
- package/dist/usage/scanner.d.ts +117 -0
- package/dist/usage/scanner.js +438 -0
- package/dist/usage/service.d.ts +122 -0
- package/dist/usage/service.js +299 -0
- package/dist/version.d.ts +2 -2
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/agent/command.js
CHANGED
|
@@ -16,6 +16,8 @@ import { acquireRunLock } from './lock.js';
|
|
|
16
16
|
import { PairError, runPair } from './pair.js';
|
|
17
17
|
import { CredentialsMissingError, ensureWorkspace, loadCredentials, prohostHome, tryLoadCredentials, } from './credentials.js';
|
|
18
18
|
import { DEFAULT_EXEC_TIMEOUT_MS, DEFAULT_IDLE_TIMEOUT_MS, runAgentHarness } from './run.js';
|
|
19
|
+
import { MAX_CONCURRENCY } from './scheduler.js';
|
|
20
|
+
import { ReposFlagError, isCheckout, parseReposFlag } from './worktrees.js';
|
|
19
21
|
import { announceUpgradeIfAvailable } from '../upgrade.js';
|
|
20
22
|
function flagString(flags, name) {
|
|
21
23
|
const value = flags[name];
|
|
@@ -119,6 +121,55 @@ function parseIdleTimeoutSeconds(flags) {
|
|
|
119
121
|
}
|
|
120
122
|
return parsed;
|
|
121
123
|
}
|
|
124
|
+
/** Returned by {@link parseConcurrency} for a value that isn't a run count. */
|
|
125
|
+
export const INVALID_CONCURRENCY = Symbol('invalid-concurrency');
|
|
126
|
+
/**
|
|
127
|
+
* Read `--concurrency`. Pure, like {@link parseSeconds}, and strict for the
|
|
128
|
+
* same reason: `--concurrency` with nothing after it arrives as `true`, and
|
|
129
|
+
* treating that as absent would leave someone believing their agent runs four
|
|
130
|
+
* things at once while it runs one.
|
|
131
|
+
*/
|
|
132
|
+
export function parseConcurrency(raw) {
|
|
133
|
+
if (raw === undefined)
|
|
134
|
+
return undefined;
|
|
135
|
+
if (typeof raw !== 'string' || !/^\d+$/.test(raw.trim()))
|
|
136
|
+
return INVALID_CONCURRENCY;
|
|
137
|
+
const count = Number(raw);
|
|
138
|
+
return count >= 1 && count <= MAX_CONCURRENCY ? count : INVALID_CONCURRENCY;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* What `--concurrency` and `--repos` asked for, or `null` after explaining why not.
|
|
142
|
+
*
|
|
143
|
+
* `installing` decides how a `--repos` path that is not a git checkout is
|
|
144
|
+
* treated — see {@link parseReposFlag}. `agent run` only warns about it.
|
|
145
|
+
*/
|
|
146
|
+
function parseParallelFlags(flags, installing) {
|
|
147
|
+
const concurrency = parseConcurrency(flags.concurrency);
|
|
148
|
+
if (concurrency === INVALID_CONCURRENCY) {
|
|
149
|
+
process.stderr.write(`--concurrency must be a whole number from 1 to ${MAX_CONCURRENCY}.\n`);
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
if (flags.repos === undefined)
|
|
153
|
+
return { concurrency };
|
|
154
|
+
const raw = flagString(flags, 'repos');
|
|
155
|
+
if (!raw) {
|
|
156
|
+
process.stderr.write('--repos needs a comma-separated list of git checkouts, e.g. --repos ~/code/app\n');
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
const repos = parseReposFlag(raw, { requireCheckouts: installing });
|
|
161
|
+
for (const repo of repos.filter((entry) => !isCheckout(entry))) {
|
|
162
|
+
process.stderr.write(`! --repos: ${repo} is not a git checkout right now. Runs will be told it is read-only until it is.\n`);
|
|
163
|
+
}
|
|
164
|
+
return { concurrency, repos };
|
|
165
|
+
}
|
|
166
|
+
catch (err) {
|
|
167
|
+
if (!(err instanceof ReposFlagError))
|
|
168
|
+
throw err;
|
|
169
|
+
process.stderr.write(`✗ ${err.message}\n`);
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
122
173
|
/**
|
|
123
174
|
* Environment the daemon must be given explicitly.
|
|
124
175
|
*
|
|
@@ -210,6 +261,10 @@ async function installDaemonCommand(flags) {
|
|
|
210
261
|
const idleTimeoutSeconds = parseIdleTimeoutSeconds(flags);
|
|
211
262
|
if (idleTimeoutSeconds === INVALID_DURATION)
|
|
212
263
|
return 1;
|
|
264
|
+
// Same reasoning: a bad value here would crash-loop the daemon, not print.
|
|
265
|
+
const parallel = parseParallelFlags(flags, true);
|
|
266
|
+
if (!parallel)
|
|
267
|
+
return 1;
|
|
213
268
|
// Absolute, and created now. A supervisor does not inherit the installing
|
|
214
269
|
// shell's current directory, so a relative `--workdir .` copied verbatim into
|
|
215
270
|
// the plist names a directory that either doesn't exist or isn't the one the
|
|
@@ -235,6 +290,8 @@ async function installDaemonCommand(flags) {
|
|
|
235
290
|
url: flagString(flags, 'url'),
|
|
236
291
|
allowUnverified: flags['allow-unverified'] === true,
|
|
237
292
|
safeTools: flags['safe-tools'] === true || flags['no-skip-permissions'] === true,
|
|
293
|
+
concurrency: parallel.concurrency,
|
|
294
|
+
repos: parallel.repos,
|
|
238
295
|
};
|
|
239
296
|
const spec = {
|
|
240
297
|
label: launchdLabel(prohostHome()),
|
|
@@ -332,6 +389,9 @@ async function runCommand(flags) {
|
|
|
332
389
|
const parsedIdleTimeout = parseIdleTimeoutSeconds(flags);
|
|
333
390
|
if (parsedIdleTimeout === INVALID_DURATION)
|
|
334
391
|
return 1;
|
|
392
|
+
const parallel = parseParallelFlags(flags, false);
|
|
393
|
+
if (!parallel)
|
|
394
|
+
return 1;
|
|
335
395
|
// One harness per home: two would both execute every run.
|
|
336
396
|
const lock = acquireRunLock();
|
|
337
397
|
if (!lock.ok) {
|
|
@@ -340,8 +400,8 @@ async function runCommand(flags) {
|
|
|
340
400
|
'`prohost agent uninstall-daemon` — or give this agent its own $PROHOST_HOME.\n');
|
|
341
401
|
return 1;
|
|
342
402
|
}
|
|
343
|
-
// First signal stops accepting work and lets the in-flight
|
|
344
|
-
// reporting
|
|
403
|
+
// First signal stops accepting work and lets the in-flight runs finish
|
|
404
|
+
// reporting themselves (so the server doesn't wait on a run we abandoned); a
|
|
345
405
|
// second one means the operator wants out now.
|
|
346
406
|
const controller = new AbortController();
|
|
347
407
|
const onSig = () => {
|
|
@@ -364,6 +424,8 @@ async function runCommand(flags) {
|
|
|
364
424
|
url: flagString(flags, 'url') ?? process.env.PROHOST_WS_URL,
|
|
365
425
|
machine: flagString(flags, 'machine'),
|
|
366
426
|
cwd: requestedWorkdir(flags),
|
|
427
|
+
concurrency: parallel.concurrency,
|
|
428
|
+
repos: parallel.repos,
|
|
367
429
|
timeoutMs: parsedTimeout === undefined ? DEFAULT_EXEC_TIMEOUT_MS : parsedTimeout * 1000,
|
|
368
430
|
idleTimeoutMs: parsedIdleTimeout === undefined ? DEFAULT_IDLE_TIMEOUT_MS : parsedIdleTimeout * 1000,
|
|
369
431
|
dryRun: flags['dry-run'] === true,
|
package/dist/agent/daemon.d.ts
CHANGED
|
@@ -90,6 +90,10 @@ export interface HarnessInvocation {
|
|
|
90
90
|
allowUnverified?: boolean;
|
|
91
91
|
/** Gate tools behind permission prompts (the `--safe-tools` opt-out). */
|
|
92
92
|
safeTools?: boolean;
|
|
93
|
+
/** Runs allowed at once; omitted for the default of one. */
|
|
94
|
+
concurrency?: number;
|
|
95
|
+
/** Absolute checkouts each conversation gets its own worktree of. */
|
|
96
|
+
repos?: string[];
|
|
93
97
|
}
|
|
94
98
|
export declare function harnessArguments(invocation: HarnessInvocation): string[];
|
|
95
99
|
/** Default log destination — alongside the credential and the run ledger. */
|
package/dist/agent/daemon.js
CHANGED
|
@@ -192,6 +192,10 @@ export function harnessArguments(invocation) {
|
|
|
192
192
|
args.push('--allow-unverified');
|
|
193
193
|
if (invocation.safeTools)
|
|
194
194
|
args.push('--safe-tools');
|
|
195
|
+
if (invocation.concurrency !== undefined)
|
|
196
|
+
args.push('--concurrency', String(invocation.concurrency));
|
|
197
|
+
if (invocation.repos && invocation.repos.length > 0)
|
|
198
|
+
args.push('--repos', invocation.repos.join(','));
|
|
195
199
|
return args;
|
|
196
200
|
}
|
|
197
201
|
/** Default log destination — alongside the credential and the run ledger. */
|
package/dist/agent/prompt.d.ts
CHANGED
|
@@ -72,6 +72,33 @@ export interface PromptContext {
|
|
|
72
72
|
* hour" is advice about how to work, not a request to hurry.
|
|
73
73
|
*/
|
|
74
74
|
idleTimeoutMs?: number;
|
|
75
|
+
/**
|
|
76
|
+
* Other runs may be executing on this machine at the same moment
|
|
77
|
+
* (`--concurrency` above 1).
|
|
78
|
+
*
|
|
79
|
+
* Said because it changes what is safe: an agent that believes it has the
|
|
80
|
+
* machine to itself switches branches in a shared checkout and writes
|
|
81
|
+
* `scratch.json` into the working directory, and the run beside it does the
|
|
82
|
+
* same. Unset, the prompt is byte-for-byte what it was.
|
|
83
|
+
*/
|
|
84
|
+
parallel?: boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Git checkouts private to this run's conversation (`--repos`).
|
|
87
|
+
*
|
|
88
|
+
* Named with the checkout they were made from, because the agent's own
|
|
89
|
+
* standing instructions usually name *that* path — and the point of this
|
|
90
|
+
* block is to move it off it.
|
|
91
|
+
*/
|
|
92
|
+
checkouts?: Array<{
|
|
93
|
+
name: string;
|
|
94
|
+
path: string;
|
|
95
|
+
source: string;
|
|
96
|
+
}>;
|
|
97
|
+
/** `--repos` entries no private checkout could be made of, and why. */
|
|
98
|
+
checkoutFailures?: Array<{
|
|
99
|
+
source: string;
|
|
100
|
+
reason: string;
|
|
101
|
+
}>;
|
|
75
102
|
}
|
|
76
103
|
/** Build the stdin prompt for one run. */
|
|
77
104
|
export declare function buildPrompt(run: AgentRunRequest, context: PromptContext): string;
|
package/dist/agent/prompt.js
CHANGED
|
@@ -7,6 +7,26 @@
|
|
|
7
7
|
* agent that would otherwise narrate its reasoning returns something sendable.
|
|
8
8
|
*/
|
|
9
9
|
import { NO_REPLY_TOKEN } from './contract.js';
|
|
10
|
+
/**
|
|
11
|
+
* What the agent is told about sharing the machine and where its checkouts are.
|
|
12
|
+
*
|
|
13
|
+
* Empty when neither `--concurrency` nor `--repos` is in play.
|
|
14
|
+
*/
|
|
15
|
+
function isolationLines(context) {
|
|
16
|
+
const lines = [];
|
|
17
|
+
const checkouts = context.checkouts ?? [];
|
|
18
|
+
const failures = context.checkoutFailures ?? [];
|
|
19
|
+
if (context.parallel) {
|
|
20
|
+
lines.push('- Other conversations may be running on this machine right now, as other', ' processes of you, in this same working directory. Give any scratch file', ' a name unique to this run, and do not assume a file or a git branch', ' you did not create in this conversation is yours to change.');
|
|
21
|
+
}
|
|
22
|
+
if (checkouts.length > 0) {
|
|
23
|
+
lines.push('- This conversation has its own git checkouts. Do all repository work', ' in them, and nowhere else:', ...checkouts.map((checkout) => ` ${checkout.name}: ${checkout.path}`), ' They are worktrees private to this conversation and they persist', ' between your runs here, so a branch or an uncommitted change you find', ' in one is your own earlier work in this thread. Wherever your other', ' instructions name one of these as the place to work, use the path', ' above instead:', ...checkouts.map((checkout) => ` ${checkout.source}`), ' Never switch branches, commit, or edit files in those originals —', ' other runs read them. Do not run `git worktree add` yourself.');
|
|
24
|
+
}
|
|
25
|
+
if (failures.length > 0) {
|
|
26
|
+
lines.push('- A private checkout could not be prepared for:', ...failures.map((failure) => ` ${failure.source} (${failure.reason})`), ' Treat that repository as read-only for this run: do not switch', ' branches, commit, or edit files in it, and say so if the request', ' needed a change there.');
|
|
27
|
+
}
|
|
28
|
+
return lines;
|
|
29
|
+
}
|
|
10
30
|
/**
|
|
11
31
|
* Say the run's budget in a unit a reader thinks in.
|
|
12
32
|
*
|
|
@@ -259,7 +279,7 @@ export function buildPrompt(run, context) {
|
|
|
259
279
|
// not buried under the tool advice: how long the agent has changes how it
|
|
260
280
|
// should approach the work, so it has to be read before any of the "go and
|
|
261
281
|
// look" bullets below.
|
|
262
|
-
...timeBudgetLines(context.timeoutMs, context.idleTimeoutMs),
|
|
282
|
+
...timeBudgetLines(context.timeoutMs, context.idleTimeoutMs), ...isolationLines(context),
|
|
263
283
|
// Claude Code lazy-loads part of its catalog — WebSearch/WebFetch among
|
|
264
284
|
// them — so a run that inspects its loaded tools concludes it cannot
|
|
265
285
|
// browse and says so to a customer, when one ToolSearch call away it
|
package/dist/agent/run.d.ts
CHANGED
|
@@ -10,9 +10,12 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Two deliberate safety properties:
|
|
12
12
|
*
|
|
13
|
-
* * **Runs are serialized.** One local agent, one run at a time;
|
|
14
|
-
* frames queue. Agents hold working directories and rate limits,
|
|
15
|
-
* several at once is a good way to corrupt both.
|
|
13
|
+
* * **Runs are serialized by default.** One local agent, one run at a time;
|
|
14
|
+
* concurrent frames queue. Agents hold working directories and rate limits,
|
|
15
|
+
* and running several at once is a good way to corrupt both. `--concurrency`
|
|
16
|
+
* raises the limit for runs in *different* conversations only — two turns of
|
|
17
|
+
* one conversation never overlap (see `scheduler.ts`) — and `--repos` gives
|
|
18
|
+
* each conversation its own git checkouts to do it in (see `worktrees.ts`).
|
|
16
19
|
* * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
|
|
17
20
|
* attempt is mirrored to this socket. Without dedupe a retry would execute
|
|
18
21
|
* the command a second time and post a second reply.
|
|
@@ -59,6 +62,15 @@ export declare const DEFAULT_EXEC_TIMEOUT_MS = 0;
|
|
|
59
62
|
* tokens and their agent's availability indefinitely.
|
|
60
63
|
*/
|
|
61
64
|
export declare const DEFAULT_IDLE_TIMEOUT_MS = 900000;
|
|
65
|
+
/**
|
|
66
|
+
* Set to `1` in the environment of every agent command this harness starts.
|
|
67
|
+
*
|
|
68
|
+
* Lets instructions the agent reads from disk tell a paired run — someone is
|
|
69
|
+
* waiting in a thread, and the run holds one of this agent's few slots — from
|
|
70
|
+
* the same file being read anywhere else. ProhostAI's own pull-request protocol
|
|
71
|
+
* uses it to stop a run from sitting on CI after the PR is open.
|
|
72
|
+
*/
|
|
73
|
+
export declare const PAIRED_RUN_ENV_VAR = "PROHOST_PAIRED_RUN";
|
|
62
74
|
export interface AgentRunOptions {
|
|
63
75
|
/** Shell command that runs the agent, e.g. ``claude -p``. */
|
|
64
76
|
exec: string;
|
|
@@ -95,6 +107,17 @@ export interface AgentRunOptions {
|
|
|
95
107
|
* ``$PROHOST_HOME/workspace`` — see {@link workspacePath} for why not "here".
|
|
96
108
|
*/
|
|
97
109
|
cwd?: string;
|
|
110
|
+
/**
|
|
111
|
+
* How many runs may execute at once. Defaults to 1 — one run at a time, as
|
|
112
|
+
* before. Above 1, runs in different conversations overlap; runs in the same
|
|
113
|
+
* conversation still wait for each other.
|
|
114
|
+
*/
|
|
115
|
+
concurrency?: number;
|
|
116
|
+
/**
|
|
117
|
+
* Git checkouts the agent works on (absolute paths). When set, every
|
|
118
|
+
* conversation gets a private worktree of each — see `worktrees.ts`.
|
|
119
|
+
*/
|
|
120
|
+
repos?: string[];
|
|
98
121
|
onLine?: (line: string) => void;
|
|
99
122
|
webSocketCtor?: typeof WebSocket;
|
|
100
123
|
fetchImpl?: typeof fetch;
|
|
@@ -186,6 +209,8 @@ interface PreparedMcp {
|
|
|
186
209
|
mcpWired: boolean;
|
|
187
210
|
/** A fail-closed preparation error. When set, the agent is never spawned. */
|
|
188
211
|
error?: string;
|
|
212
|
+
/** The account this run was resolved onto; hand it back when the run ends. */
|
|
213
|
+
accountLabel?: string;
|
|
189
214
|
}
|
|
190
215
|
/**
|
|
191
216
|
* Prepare a fresh Codex MCP override immediately before each invocation.
|
package/dist/agent/run.js
CHANGED
|
@@ -10,9 +10,12 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Two deliberate safety properties:
|
|
12
12
|
*
|
|
13
|
-
* * **Runs are serialized.** One local agent, one run at a time;
|
|
14
|
-
* frames queue. Agents hold working directories and rate limits,
|
|
15
|
-
* several at once is a good way to corrupt both.
|
|
13
|
+
* * **Runs are serialized by default.** One local agent, one run at a time;
|
|
14
|
+
* concurrent frames queue. Agents hold working directories and rate limits,
|
|
15
|
+
* and running several at once is a good way to corrupt both. `--concurrency`
|
|
16
|
+
* raises the limit for runs in *different* conversations only — two turns of
|
|
17
|
+
* one conversation never overlap (see `scheduler.ts`) — and `--repos` gives
|
|
18
|
+
* each conversation its own git checkouts to do it in (see `worktrees.ts`).
|
|
16
19
|
* * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
|
|
17
20
|
* attempt is mirrored to this socket. Without dedupe a retry would execute
|
|
18
21
|
* the command a second time and post a second reply.
|
|
@@ -40,9 +43,11 @@ import { HeartbeatLoop } from './heartbeat.js';
|
|
|
40
43
|
import { buildPrompt, replyFromStdout } from './prompt.js';
|
|
41
44
|
import { NO_CAPABILITIES, capabilityProbe } from './capabilities.js';
|
|
42
45
|
import { RunStore } from './runstore.js';
|
|
46
|
+
import { MAX_CONCURRENCY, RunScheduler } from './scheduler.js';
|
|
43
47
|
import { RUNTIME_STATUS_FEATURE, RuntimeStatusReporter, claudeQuotaProvider, codexConfiguredModel, codexQuotaProvider, modelFromExec, parseVersion, readCodexObservation, } from './runtime_status.js';
|
|
44
48
|
import { SessionStore, sessionKeyFor } from './sessionstore.js';
|
|
45
49
|
import { verifySignature } from './signature.js';
|
|
50
|
+
import { RUN_REPOS_ENV_VAR, WorktreeManager } from './worktrees.js';
|
|
46
51
|
import { isWebhookEvent } from '../forward.js';
|
|
47
52
|
import { DEFAULT_WS_URL } from '../listen.js';
|
|
48
53
|
import { ServerLiveness, enableTcpKeepAlive, seconds, terminateSocket } from '../liveness.js';
|
|
@@ -89,6 +94,15 @@ const RECONNECT_MAX_DELAY_MS = 30_000;
|
|
|
89
94
|
* would hang with neither `open` nor `close` to move the loop along.
|
|
90
95
|
*/
|
|
91
96
|
const HANDSHAKE_TIMEOUT_MS = 30_000;
|
|
97
|
+
/**
|
|
98
|
+
* Set to `1` in the environment of every agent command this harness starts.
|
|
99
|
+
*
|
|
100
|
+
* Lets instructions the agent reads from disk tell a paired run — someone is
|
|
101
|
+
* waiting in a thread, and the run holds one of this agent's few slots — from
|
|
102
|
+
* the same file being read anywhere else. ProhostAI's own pull-request protocol
|
|
103
|
+
* uses it to stop a run from sitting on CI after the PR is open.
|
|
104
|
+
*/
|
|
105
|
+
export const PAIRED_RUN_ENV_VAR = 'PROHOST_PAIRED_RUN';
|
|
92
106
|
/** Ack status codes reported back over the socket for each frame. */
|
|
93
107
|
const ACK_ACCEPTED = 200;
|
|
94
108
|
const ACK_IGNORED = 204;
|
|
@@ -228,6 +242,16 @@ function resolveWorkdir(options, log) {
|
|
|
228
242
|
return undefined;
|
|
229
243
|
}
|
|
230
244
|
}
|
|
245
|
+
/**
|
|
246
|
+
* The lane a run waits on: runs sharing one never overlap.
|
|
247
|
+
*
|
|
248
|
+
* The session key, because a session is exactly what two overlapping turns of
|
|
249
|
+
* one conversation would corrupt. A run with no surface resumes nothing and
|
|
250
|
+
* shares nothing, so it gets a lane of its own.
|
|
251
|
+
*/
|
|
252
|
+
function laneFor(run) {
|
|
253
|
+
return sessionKeyFor(run) ?? `run:${run.run_id}`;
|
|
254
|
+
}
|
|
231
255
|
/**
|
|
232
256
|
* Assemble the per-process runtime: workspace, session ledger, and the
|
|
233
257
|
* brain-specific MCP preparation used immediately before each run.
|
|
@@ -330,7 +354,11 @@ function buildRuntime(options, log) {
|
|
|
330
354
|
if (!resolved.ok)
|
|
331
355
|
return { mcpWired: false, error: resolved.error };
|
|
332
356
|
const prepared = prepareMcp();
|
|
333
|
-
return {
|
|
357
|
+
return {
|
|
358
|
+
...prepared,
|
|
359
|
+
execEnv: { ...account.spawnEnv(), ...prepared.execEnv },
|
|
360
|
+
accountLabel: resolved.record.label,
|
|
361
|
+
};
|
|
334
362
|
};
|
|
335
363
|
log(`${timestamp()} ⚙ account: ${account.label()}`);
|
|
336
364
|
return {
|
|
@@ -579,7 +607,9 @@ async function runTurn(run, prompt, options, runtime, mcp, control, log,
|
|
|
579
607
|
* Where the streaming brain's tool-call events go, if anywhere. Only the
|
|
580
608
|
* Claude Code reader produces any; absent, they are simply never collected.
|
|
581
609
|
*/
|
|
582
|
-
toolEvents
|
|
610
|
+
toolEvents,
|
|
611
|
+
/** Harness-set variables for this run's command, over the brain's own. */
|
|
612
|
+
runEnv) {
|
|
583
613
|
// `--timeout` is a ceiling on the run, not on each attempt: the retry below
|
|
584
614
|
// must not let one run take twice what the operator allowed. Both attempts
|
|
585
615
|
// share one deadline, which costs the retry nothing in practice — a rejected
|
|
@@ -667,7 +697,7 @@ toolEvents) {
|
|
|
667
697
|
timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
|
|
668
698
|
idleTimeoutMs: idleMs,
|
|
669
699
|
cwd: runtime.workdir,
|
|
670
|
-
env: mcp.execEnv,
|
|
700
|
+
env: { ...mcp.execEnv, ...runEnv },
|
|
671
701
|
spawnImpl: options.spawnImpl,
|
|
672
702
|
cancelSignal: control.signal,
|
|
673
703
|
onStdout: reader
|
|
@@ -842,11 +872,18 @@ toolEvents) {
|
|
|
842
872
|
* Every path except a dry run — including every failure — calls the completion
|
|
843
873
|
* endpoint before returning.
|
|
844
874
|
*/
|
|
845
|
-
async function handleRun(run, options, runtime, api, log, sleep, control
|
|
875
|
+
async function handleRun(run, options, runtime, api, log, sleep, control, isolation,
|
|
876
|
+
/** Prepared by the caller, which also hands the run's account back. */
|
|
877
|
+
mcp) {
|
|
846
878
|
const label = `run ${run.run_id.slice(0, 12)}`;
|
|
847
879
|
const surface = run.conversation_id ?? run.kind ?? 'no conversation';
|
|
848
880
|
log(`${timestamp()} ▶ ${label} started (${surface})`);
|
|
849
|
-
|
|
881
|
+
// Before anything is spawned: the prompt names these paths. A dry run makes
|
|
882
|
+
// no writes, and a worktree is one.
|
|
883
|
+
const checkouts = isolation.worktrees.enabled && !options.dryRun ? await isolation.worktrees.prepare(isolation.lane) : undefined;
|
|
884
|
+
for (const failure of checkouts?.failures ?? []) {
|
|
885
|
+
log(` ! no private checkout of ${failure.source} for ${label} (${failure.reason})`);
|
|
886
|
+
}
|
|
850
887
|
// Only worth asking when the tools it describes are wired into this run.
|
|
851
888
|
const capabilities = mcp.mcpWired ? await runtime.capabilities() : NO_CAPABILITIES;
|
|
852
889
|
const prompt = buildPrompt(run, {
|
|
@@ -865,6 +902,9 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
|
|
|
865
902
|
// The real numbers, not a vague "there is a limit" — see PromptContext.
|
|
866
903
|
timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
|
|
867
904
|
idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
|
|
905
|
+
parallel: isolation.parallel,
|
|
906
|
+
checkouts: checkouts?.checkouts,
|
|
907
|
+
checkoutFailures: checkouts?.failures,
|
|
868
908
|
});
|
|
869
909
|
// Tool-call narration for the thread. A dry run makes no writes, so it gets
|
|
870
910
|
// no path and every event is dropped; so does a server that advertised none.
|
|
@@ -874,7 +914,10 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
|
|
|
874
914
|
log,
|
|
875
915
|
label,
|
|
876
916
|
});
|
|
877
|
-
const
|
|
917
|
+
const runEnv = { [PAIRED_RUN_ENV_VAR]: '1' };
|
|
918
|
+
if (checkouts && checkouts.checkouts.length > 0)
|
|
919
|
+
runEnv[RUN_REPOS_ENV_VAR] = checkouts.dir;
|
|
920
|
+
const result = await runTurn(run, prompt, options, runtime, mcp, control, log, toolEvents, runEnv);
|
|
878
921
|
// Whatever the turn did, the timeline is whole BEFORE the reply is posted or
|
|
879
922
|
// the run is closed — a completion that beats its own last tool row would
|
|
880
923
|
// leave a `calling` row on a finished run. Never rejects, and after `stop`
|
|
@@ -1109,7 +1152,19 @@ export async function runAgentHarness(options) {
|
|
|
1109
1152
|
}
|
|
1110
1153
|
return true;
|
|
1111
1154
|
};
|
|
1112
|
-
|
|
1155
|
+
const concurrency = Math.min(MAX_CONCURRENCY, Math.max(1, Math.floor(options.concurrency ?? 1)));
|
|
1156
|
+
const scheduler = new RunScheduler(concurrency);
|
|
1157
|
+
const worktrees = new WorktreeManager({ repos: options.repos ?? [], env: options.env, log });
|
|
1158
|
+
if (concurrency > 1) {
|
|
1159
|
+
log(`${timestamp()} ⚙ up to ${concurrency} runs at once — runs in the same conversation still take turns` +
|
|
1160
|
+
(worktrees.enabled
|
|
1161
|
+
? ''
|
|
1162
|
+
: '. No --repos given: runs share every checkout on this machine, so name the ' +
|
|
1163
|
+
'repositories the agent edits to give each conversation its own'));
|
|
1164
|
+
}
|
|
1165
|
+
if (worktrees.enabled) {
|
|
1166
|
+
log(`${timestamp()} ⚙ each conversation gets its own checkout of: ${(options.repos ?? []).join(', ')}`);
|
|
1167
|
+
}
|
|
1113
1168
|
let fatal;
|
|
1114
1169
|
let attempt = 0;
|
|
1115
1170
|
/** Stop tracking a run — it has reached a terminal state, one way or another. */
|
|
@@ -1195,7 +1250,7 @@ export async function runAgentHarness(options) {
|
|
|
1195
1250
|
// Let an in-flight run finish reporting before we hand back metrics — and
|
|
1196
1251
|
// any cancellation the heartbeat loop started and deliberately did not wait
|
|
1197
1252
|
// for, so the harness never exits with a run half-closed.
|
|
1198
|
-
await
|
|
1253
|
+
await scheduler.onIdle();
|
|
1199
1254
|
status.detach();
|
|
1200
1255
|
// Stopped first, so no further tick can start work after the wait below has
|
|
1201
1256
|
// taken its snapshot of `background`.
|
|
@@ -1404,7 +1459,7 @@ export async function runAgentHarness(options) {
|
|
|
1404
1459
|
// completion. Retry only the completion: re-running the agent could
|
|
1405
1460
|
// repeat whatever it did with its credential the first time.
|
|
1406
1461
|
metrics.duplicates += 1;
|
|
1407
|
-
|
|
1462
|
+
scheduler.submit(laneFor(run), async () => {
|
|
1408
1463
|
log(`${timestamp()} ↻ run ${run.run_id.slice(0, 12)} was started before but never reported; ` +
|
|
1409
1464
|
'reporting it now without re-running the agent');
|
|
1410
1465
|
const recovered = await completeWithRetry(api, previous.completion_path || run.completion_path, { ok: false, error: 'the harness restarted before this run was reported' }, log, sleep);
|
|
@@ -1434,9 +1489,11 @@ export async function runAgentHarness(options) {
|
|
|
1434
1489
|
},
|
|
1435
1490
|
onResolved: () => resolveAccepted(entry),
|
|
1436
1491
|
});
|
|
1437
|
-
//
|
|
1438
|
-
//
|
|
1439
|
-
|
|
1492
|
+
// Queue execution but never block the socket reader: the ack and the
|
|
1493
|
+
// heartbeat have to keep flowing while the agent thinks. The lane is
|
|
1494
|
+
// what keeps two turns of one conversation from overlapping.
|
|
1495
|
+
const lane = laneFor(run);
|
|
1496
|
+
scheduler.submit(lane, async () => {
|
|
1440
1497
|
// Settled while it waited — cancelled, or closed server-side. Both
|
|
1441
1498
|
// were dealt with where they were decided; there is nothing to run
|
|
1442
1499
|
// and nothing left to report.
|
|
@@ -1465,17 +1522,32 @@ export async function runAgentHarness(options) {
|
|
|
1465
1522
|
entry.progress = activity;
|
|
1466
1523
|
},
|
|
1467
1524
|
};
|
|
1468
|
-
const
|
|
1525
|
+
const isolation = { lane, parallel: concurrency > 1, worktrees };
|
|
1526
|
+
// Resolved here rather than inside `handleRun` so the account this
|
|
1527
|
+
// run is on can be handed back however the run ends.
|
|
1528
|
+
let mcp;
|
|
1529
|
+
try {
|
|
1530
|
+
mcp = runtime.prepareMcp();
|
|
1531
|
+
}
|
|
1532
|
+
catch (err) {
|
|
1533
|
+
// Fails the run closed, and — unlike a throw from here — still
|
|
1534
|
+
// reports it, so the server is not left waiting on it.
|
|
1535
|
+
mcp = { mcpWired: false, error: `could not prepare the run (${String(err)})` };
|
|
1536
|
+
}
|
|
1537
|
+
const outcome = await handleRun(run, options, runtime, api, log, sleep, control, isolation, mcp).catch((err) => {
|
|
1469
1538
|
log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
|
|
1470
1539
|
return 'failed';
|
|
1471
1540
|
});
|
|
1541
|
+
if (mcp.accountLabel !== undefined)
|
|
1542
|
+
runtime.account?.finishSpawn(mcp.accountLabel);
|
|
1472
1543
|
// Held until here on purpose: heartbeats keep the row alive for the
|
|
1473
1544
|
// whole of `handleRun`, including the ten minutes of completion
|
|
1474
1545
|
// retries at the end of it.
|
|
1475
1546
|
release(run.run_id);
|
|
1476
1547
|
// An account picked mid-run was held back from the status frame
|
|
1477
|
-
// until this run (on the old account) was done; report it now.
|
|
1478
|
-
|
|
1548
|
+
// until this run (on the old account) was done; report it now. Asked
|
|
1549
|
+
// about *this* run's account: a newer run may already be on the pick.
|
|
1550
|
+
if (runtime.account?.changedSinceSpawn(mcp.accountLabel))
|
|
1479
1551
|
status.accountsChanged();
|
|
1480
1552
|
// Both terminal outcomes reached the completion endpoint; only then
|
|
1481
1553
|
// is the run settled and safe to skip on redelivery.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which accepted runs execute now, and which wait.
|
|
3
|
+
*
|
|
4
|
+
* `agent run` used to chain every run onto one promise: one local agent, one
|
|
5
|
+
* run at a time. That is still the default, and with a limit of 1 this
|
|
6
|
+
* scheduler is exactly that chain — strict arrival order, nothing overlapping.
|
|
7
|
+
*
|
|
8
|
+
* `--concurrency N` lifts the limit without lifting the rule that actually
|
|
9
|
+
* protects an answer: **two runs on the same lane never overlap, and they start
|
|
10
|
+
* in the order they arrived.** A lane is a conversation (see `sessionKeyFor`).
|
|
11
|
+
* Each turn resumes the session the previous turn left, so running two turns of
|
|
12
|
+
* one thread side by side would have both resume the same session and the
|
|
13
|
+
* second answer without having seen the first. Runs on different lanes share
|
|
14
|
+
* nothing but the machine, and those are the ones allowed to overlap.
|
|
15
|
+
*
|
|
16
|
+
* Among the runs free to start, the oldest goes first. A lane that is busy does
|
|
17
|
+
* not hold up the lanes behind it — which is the whole point: a quick question
|
|
18
|
+
* in one thread no longer waits out a two-hour job in another.
|
|
19
|
+
*/
|
|
20
|
+
/** Most runs one harness will execute at once, whatever the flag says. */
|
|
21
|
+
export declare const MAX_CONCURRENCY = 16;
|
|
22
|
+
export declare class RunScheduler {
|
|
23
|
+
private readonly limit;
|
|
24
|
+
private readonly pending;
|
|
25
|
+
private readonly busy;
|
|
26
|
+
private waiters;
|
|
27
|
+
constructor(limit?: number);
|
|
28
|
+
/** How many tasks are executing right now. */
|
|
29
|
+
get active(): number;
|
|
30
|
+
/** How many tasks are waiting for a slot or for their lane. */
|
|
31
|
+
get queued(): number;
|
|
32
|
+
/**
|
|
33
|
+
* Queue a task on a lane. It starts as soon as a slot is free and no earlier
|
|
34
|
+
* task on the same lane is still running or waiting.
|
|
35
|
+
*
|
|
36
|
+
* The task must not reject: a rejection is swallowed so one broken run can
|
|
37
|
+
* never wedge its lane, but nothing reports it either — callers catch inside.
|
|
38
|
+
*/
|
|
39
|
+
submit(lane: string, task: () => Promise<void>): void;
|
|
40
|
+
/** Resolves once nothing is executing and nothing is waiting. */
|
|
41
|
+
onIdle(): Promise<void>;
|
|
42
|
+
private pump;
|
|
43
|
+
private execute;
|
|
44
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which accepted runs execute now, and which wait.
|
|
3
|
+
*
|
|
4
|
+
* `agent run` used to chain every run onto one promise: one local agent, one
|
|
5
|
+
* run at a time. That is still the default, and with a limit of 1 this
|
|
6
|
+
* scheduler is exactly that chain — strict arrival order, nothing overlapping.
|
|
7
|
+
*
|
|
8
|
+
* `--concurrency N` lifts the limit without lifting the rule that actually
|
|
9
|
+
* protects an answer: **two runs on the same lane never overlap, and they start
|
|
10
|
+
* in the order they arrived.** A lane is a conversation (see `sessionKeyFor`).
|
|
11
|
+
* Each turn resumes the session the previous turn left, so running two turns of
|
|
12
|
+
* one thread side by side would have both resume the same session and the
|
|
13
|
+
* second answer without having seen the first. Runs on different lanes share
|
|
14
|
+
* nothing but the machine, and those are the ones allowed to overlap.
|
|
15
|
+
*
|
|
16
|
+
* Among the runs free to start, the oldest goes first. A lane that is busy does
|
|
17
|
+
* not hold up the lanes behind it — which is the whole point: a quick question
|
|
18
|
+
* in one thread no longer waits out a two-hour job in another.
|
|
19
|
+
*/
|
|
20
|
+
/** Most runs one harness will execute at once, whatever the flag says. */
|
|
21
|
+
export const MAX_CONCURRENCY = 16;
|
|
22
|
+
export class RunScheduler {
|
|
23
|
+
limit;
|
|
24
|
+
pending = [];
|
|
25
|
+
busy = new Set();
|
|
26
|
+
waiters = [];
|
|
27
|
+
constructor(limit = 1) {
|
|
28
|
+
this.limit = limit;
|
|
29
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
30
|
+
throw new RangeError(`concurrency must be a whole number of at least 1, got ${String(limit)}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/** How many tasks are executing right now. */
|
|
34
|
+
get active() {
|
|
35
|
+
return this.busy.size;
|
|
36
|
+
}
|
|
37
|
+
/** How many tasks are waiting for a slot or for their lane. */
|
|
38
|
+
get queued() {
|
|
39
|
+
return this.pending.length;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Queue a task on a lane. It starts as soon as a slot is free and no earlier
|
|
43
|
+
* task on the same lane is still running or waiting.
|
|
44
|
+
*
|
|
45
|
+
* The task must not reject: a rejection is swallowed so one broken run can
|
|
46
|
+
* never wedge its lane, but nothing reports it either — callers catch inside.
|
|
47
|
+
*/
|
|
48
|
+
submit(lane, task) {
|
|
49
|
+
this.pending.push({ lane, task });
|
|
50
|
+
this.pump();
|
|
51
|
+
}
|
|
52
|
+
/** Resolves once nothing is executing and nothing is waiting. */
|
|
53
|
+
onIdle() {
|
|
54
|
+
if (this.busy.size === 0 && this.pending.length === 0)
|
|
55
|
+
return Promise.resolve();
|
|
56
|
+
return new Promise((resolve) => {
|
|
57
|
+
this.waiters.push(resolve);
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
pump() {
|
|
61
|
+
while (this.busy.size < this.limit) {
|
|
62
|
+
// First in arrival order whose lane is free. An earlier task on a busy
|
|
63
|
+
// lane is skipped, not waited for; a later task on that same lane is
|
|
64
|
+
// skipped with it, which is what keeps a lane in order.
|
|
65
|
+
const index = this.pending.findIndex((entry) => !this.busy.has(entry.lane));
|
|
66
|
+
if (index === -1)
|
|
67
|
+
break;
|
|
68
|
+
const [next] = this.pending.splice(index, 1);
|
|
69
|
+
if (!next)
|
|
70
|
+
break;
|
|
71
|
+
this.busy.add(next.lane);
|
|
72
|
+
void this.execute(next);
|
|
73
|
+
}
|
|
74
|
+
if (this.busy.size === 0 && this.pending.length === 0) {
|
|
75
|
+
const waiters = this.waiters;
|
|
76
|
+
this.waiters = [];
|
|
77
|
+
for (const resolve of waiters)
|
|
78
|
+
resolve();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async execute(entry) {
|
|
82
|
+
try {
|
|
83
|
+
await entry.task();
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
/* see `submit` — a task reports its own failure */
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
this.busy.delete(entry.lane);
|
|
90
|
+
this.pump();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|