@prohost/cli 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -0
- package/README.md +24 -0
- package/dist/agent/api.d.ts +45 -2
- package/dist/agent/api.js +67 -8
- package/dist/agent/claude.d.ts +7 -0
- package/dist/agent/claude.js +13 -0
- package/dist/agent/command.js +4 -0
- package/dist/agent/contract.d.ts +2 -0
- package/dist/agent/contract.js +2 -0
- package/dist/agent/heartbeat.js +4 -3
- package/dist/agent/run.d.ts +19 -0
- package/dist/agent/run.js +214 -21
- package/dist/agent/runtime_status.d.ts +191 -0
- package/dist/agent/runtime_status.js +556 -0
- package/dist/listen.d.ts +7 -0
- package/dist/listen.js +25 -2
- package/dist/liveness.d.ts +107 -0
- package/dist/liveness.js +156 -0
- package/dist/upgrade.d.ts +49 -0
- package/dist/upgrade.js +99 -0
- package/dist/version.d.ts +2 -2
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/agent/run.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
import os from 'node:os';
|
|
26
26
|
import { spawnSync } from 'node:child_process';
|
|
27
27
|
import WebSocket from 'ws';
|
|
28
|
-
import { completeRun, postReply } from './api.js';
|
|
28
|
+
import { completeRun, describeFailure, isIdempotencyInFlight, postReply } from './api.js';
|
|
29
29
|
import { MAX_SALVAGE_CHARS, claudeCommand, isClaudeExec, looksLikeResumeFailure, readClaudeStream, writeMcpConfig, } from './claude.js';
|
|
30
30
|
import { APPS_FEATURE, MCP_API_KEY_ENV_VAR, PLUGINS_DISABLED_OVERRIDE, codexBinary, codexCommand, codexMcpInventoryFlag, codexProfileName, codexShellSyntax, extraMcpEntries, isCodexExec, isolateConfiguredServers, listMcpServers, mcpServersOverride, prohostMcpEntry, looksLikeCodexResumeFailure, parseCodexOutput, } from './codex.js';
|
|
31
31
|
import { EXTRA_MCP_FILENAME, loadExtraMcpServers } from './extras.js';
|
|
@@ -36,10 +36,13 @@ import { execAgent } from './exec.js';
|
|
|
36
36
|
import { HeartbeatLoop } from './heartbeat.js';
|
|
37
37
|
import { buildPrompt, replyFromStdout } from './prompt.js';
|
|
38
38
|
import { RunStore } from './runstore.js';
|
|
39
|
+
import { RUNTIME_STATUS_FEATURE, RuntimeStatusReporter, claudeQuotaProvider, codexConfiguredModel, codexQuotaProvider, modelFromExec, parseVersion, readCodexObservation, } from './runtime_status.js';
|
|
39
40
|
import { SessionStore, sessionKeyFor } from './sessionstore.js';
|
|
40
41
|
import { verifySignature } from './signature.js';
|
|
41
42
|
import { isWebhookEvent } from '../forward.js';
|
|
42
43
|
import { DEFAULT_WS_URL } from '../listen.js';
|
|
44
|
+
import { ServerLiveness, enableTcpKeepAlive, seconds, terminateSocket } from '../liveness.js';
|
|
45
|
+
import { CLI_VERSION, USER_AGENT } from '../version.js';
|
|
43
46
|
/**
|
|
44
47
|
* Default ceiling on one agent invocation: none.
|
|
45
48
|
*
|
|
@@ -74,6 +77,14 @@ export const DEFAULT_IDLE_TIMEOUT_MS = 900_000;
|
|
|
74
77
|
/** Reconnect backoff — full jitter between ``BASE`` and the current cap. */
|
|
75
78
|
const RECONNECT_BASE_DELAY_MS = 1_000;
|
|
76
79
|
const RECONNECT_MAX_DELAY_MS = 30_000;
|
|
80
|
+
/**
|
|
81
|
+
* Longest the websocket handshake may take before `ws` gives up on it.
|
|
82
|
+
*
|
|
83
|
+
* The liveness deadline only starts at `open`; without this, a connect that
|
|
84
|
+
* stalls mid-handshake — the same asleep-laptop shape, one step earlier —
|
|
85
|
+
* would hang with neither `open` nor `close` to move the loop along.
|
|
86
|
+
*/
|
|
87
|
+
const HANDSHAKE_TIMEOUT_MS = 30_000;
|
|
77
88
|
/** Ack status codes reported back over the socket for each frame. */
|
|
78
89
|
const ACK_ACCEPTED = 200;
|
|
79
90
|
const ACK_IGNORED = 204;
|
|
@@ -88,6 +99,7 @@ function newMetrics() {
|
|
|
88
99
|
signatureRejections: 0,
|
|
89
100
|
ignored: 0,
|
|
90
101
|
duplicates: 0,
|
|
102
|
+
livenessTimeouts: 0,
|
|
91
103
|
};
|
|
92
104
|
}
|
|
93
105
|
function timestamp() {
|
|
@@ -122,20 +134,70 @@ const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000, 30_000, 60_000, 120_000
|
|
|
122
134
|
* sweeper reaps it. A 409 (`already_resolved`) counts as done. A 4xx other
|
|
123
135
|
* than 429 is a decision, not a blip, so it isn't retried.
|
|
124
136
|
*/
|
|
125
|
-
async function completeWithRetry(api, path, outcome, log, sleep) {
|
|
126
|
-
let result = await completeRun(api, path, outcome);
|
|
137
|
+
async function completeWithRetry(api, path, outcome, log, sleep, model) {
|
|
138
|
+
let result = await completeRun(api, path, outcome, model);
|
|
127
139
|
for (const delay of COMPLETION_RETRY_DELAYS_MS) {
|
|
128
140
|
if (result.ok)
|
|
129
141
|
return result;
|
|
130
142
|
const retryable = result.status === 0 || result.status === 429 || result.status >= 500;
|
|
131
143
|
if (!retryable)
|
|
132
144
|
return result;
|
|
133
|
-
log(` … completion failed (
|
|
145
|
+
log(` … completion failed (${describeFailure(result)}); retrying in ${delay / 1000}s`);
|
|
134
146
|
await sleep(delay);
|
|
135
|
-
result = await completeRun(api, path, outcome);
|
|
147
|
+
result = await completeRun(api, path, outcome, model);
|
|
136
148
|
}
|
|
137
149
|
return result;
|
|
138
150
|
}
|
|
151
|
+
/**
|
|
152
|
+
* How long to wait before replaying a reply whose first attempt the server
|
|
153
|
+
* reports as still in flight. The original request is being processed at
|
|
154
|
+
* that moment; a few seconds is enough for it to finish and be cached.
|
|
155
|
+
*/
|
|
156
|
+
const REPLAY_IN_FLIGHT_DELAY_MS = 5_000;
|
|
157
|
+
/**
|
|
158
|
+
* Post the reply, and if the request dies in transit, replay it once with
|
|
159
|
+
* the same idempotency key before believing it failed.
|
|
160
|
+
*
|
|
161
|
+
* A request abandoned at its deadline is not a request that failed — it is
|
|
162
|
+
* a request whose answer we did not wait for. On 2026-09-01 the server took
|
|
163
|
+
* 51 s to accept a reply that the CLI had given up on at 30 s; the message
|
|
164
|
+
* was in the conversation, and the CLI reported the run as failed anyway.
|
|
165
|
+
* The key (``prohost-cli-run-<run_id>``) already made the send idempotent
|
|
166
|
+
* server-side, so the correct reaction to "no answer" is to ask again with
|
|
167
|
+
* the same key and let the server say what happened:
|
|
168
|
+
*
|
|
169
|
+
* - a 2xx (fresh, or the cached response of the attempt that did land) —
|
|
170
|
+
* the reply is durable, carry on as a success;
|
|
171
|
+
* - ``409 idempotency_key_in_flight`` — the first attempt is still being
|
|
172
|
+
* processed; wait, then replay once more;
|
|
173
|
+
* - anything else — a real decision or a second transport failure — is the
|
|
174
|
+
* answer, and the run is reported failed exactly once.
|
|
175
|
+
*/
|
|
176
|
+
async function postReplyWithReplay(api, send, label, log, sleep) {
|
|
177
|
+
let posted = await send();
|
|
178
|
+
// A real answer — success or a decision — is final. Only "no answer at all"
|
|
179
|
+
// earns a replay.
|
|
180
|
+
if (posted.ok || posted.status !== 0)
|
|
181
|
+
return posted;
|
|
182
|
+
const why = posted.failure === 'timeout'
|
|
183
|
+
? `reply POST ${posted.error}`
|
|
184
|
+
: `reply POST failed (${describeFailure(posted)})`;
|
|
185
|
+
log(`${timestamp()} … ${label} ${why} — replaying with the same idempotency key`);
|
|
186
|
+
posted = await send();
|
|
187
|
+
if (posted.ok) {
|
|
188
|
+
log(`${timestamp()} ✓ ${label} reply confirmed on replay`);
|
|
189
|
+
return posted;
|
|
190
|
+
}
|
|
191
|
+
if (isIdempotencyInFlight(posted)) {
|
|
192
|
+
log(`${timestamp()} … ${label} first reply attempt is still in flight server-side — ` +
|
|
193
|
+
`replaying again in ${REPLAY_IN_FLIGHT_DELAY_MS / 1000}s`);
|
|
194
|
+
await sleep(REPLAY_IN_FLIGHT_DELAY_MS);
|
|
195
|
+
posted = await send();
|
|
196
|
+
if (posted.ok)
|
|
197
|
+
log(`${timestamp()} ✓ ${label} reply confirmed on replay`);
|
|
198
|
+
}
|
|
199
|
+
return posted;
|
|
200
|
+
}
|
|
139
201
|
/**
|
|
140
202
|
* Decide where the agent command runs.
|
|
141
203
|
*
|
|
@@ -179,8 +241,21 @@ function buildRuntime(options, log) {
|
|
|
179
241
|
? 'codex'
|
|
180
242
|
: 'plain';
|
|
181
243
|
const skipPermissions = options.safeTools !== true;
|
|
244
|
+
const runtimeName = runtimeNameFor(brain, options.exec);
|
|
245
|
+
// Replaced by the harness once its status reporter exists.
|
|
246
|
+
const observe = () => { };
|
|
247
|
+
const model = () => undefined;
|
|
182
248
|
if (brain === 'plain') {
|
|
183
|
-
return {
|
|
249
|
+
return {
|
|
250
|
+
workdir,
|
|
251
|
+
sessions,
|
|
252
|
+
brain,
|
|
253
|
+
skipPermissions,
|
|
254
|
+
prepareMcp: () => ({ mcpWired: false }),
|
|
255
|
+
runtimeName,
|
|
256
|
+
observe,
|
|
257
|
+
model,
|
|
258
|
+
};
|
|
184
259
|
}
|
|
185
260
|
const mcpUrl = options.credentials.mcp_url;
|
|
186
261
|
let announced = '';
|
|
@@ -235,7 +310,30 @@ function buildRuntime(options, log) {
|
|
|
235
310
|
'access, on instructions from your workspace. Use --safe-tools to gate them.'
|
|
236
311
|
: `${timestamp()} ⚙ --safe-tools: tools are permission-gated; anything needing ` +
|
|
237
312
|
'approval will not run (nobody is at a terminal to approve it).');
|
|
238
|
-
return { workdir, sessions, brain, prepareMcp, skipPermissions };
|
|
313
|
+
return { workdir, sessions, brain, prepareMcp, skipPermissions, runtimeName, observe, model };
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* The `runtime` a status frame names. A plain command is `custom` unless it is
|
|
317
|
+
* recognisably goose — the one other agent CLI people pair today.
|
|
318
|
+
*/
|
|
319
|
+
function runtimeNameFor(brain, exec) {
|
|
320
|
+
if (brain === 'claude')
|
|
321
|
+
return 'claude_code';
|
|
322
|
+
if (brain === 'codex')
|
|
323
|
+
return 'codex';
|
|
324
|
+
const base = (exec.trim().split(/\s+/)[0] ?? '').split('/').pop() ?? '';
|
|
325
|
+
return base === 'goose' ? 'goose' : 'custom';
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Idle capacity source for the detected brain: the agent CLI itself, asked
|
|
329
|
+
* through its own protocol so its own login answers. `plain` has none.
|
|
330
|
+
*/
|
|
331
|
+
function defaultQuotaProvider(brain, exec) {
|
|
332
|
+
if (brain === 'claude')
|
|
333
|
+
return claudeQuotaProvider({ binary: exec.trim().split(/\s+/)[0] ?? 'claude' });
|
|
334
|
+
if (brain === 'codex')
|
|
335
|
+
return codexQuotaProvider({ binary: codexBinary(exec), clientVersion: CLI_VERSION });
|
|
336
|
+
return undefined;
|
|
239
337
|
}
|
|
240
338
|
/**
|
|
241
339
|
* Prepare a fresh Codex MCP override immediately before each invocation.
|
|
@@ -513,7 +611,7 @@ async function runTurn(run, prompt, options, runtime, mcp, control, log) {
|
|
|
513
611
|
const reader = runtime.brain === 'claude' ? readClaudeStream() : undefined;
|
|
514
612
|
stream = reader;
|
|
515
613
|
tail = '';
|
|
516
|
-
|
|
614
|
+
const running = execAgent({
|
|
517
615
|
command,
|
|
518
616
|
prompt,
|
|
519
617
|
timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
|
|
@@ -533,6 +631,9 @@ async function runTurn(run, prompt, options, runtime, mcp, control, log) {
|
|
|
533
631
|
}
|
|
534
632
|
: undefined,
|
|
535
633
|
});
|
|
634
|
+
// Whatever the stream announced — model, version, capacity — is true of
|
|
635
|
+
// this machine whether or not the turn succeeded.
|
|
636
|
+
return reader ? running.finally(() => runtime.observe(reader.observed())) : running;
|
|
536
637
|
};
|
|
537
638
|
// Wall clock for the whole turn, retry included — what the operator waited,
|
|
538
639
|
// rather than what the last attempt took.
|
|
@@ -643,6 +744,9 @@ async function runTurn(run, prompt, options, runtime, mcp, control, log) {
|
|
|
643
744
|
};
|
|
644
745
|
if (codex) {
|
|
645
746
|
const output = parseCodexOutput(result.stdout);
|
|
747
|
+
// Codex's `--json` stream carries neither model nor limits; its rollout
|
|
748
|
+
// file for this thread does.
|
|
749
|
+
runtime.observe(readCodexObservation(output.sessionId, { ...process.env, ...options.env }));
|
|
646
750
|
reportDenials(output.denials);
|
|
647
751
|
if (!output.ok) {
|
|
648
752
|
return {
|
|
@@ -726,9 +830,10 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
|
|
|
726
830
|
}
|
|
727
831
|
// The exact marker string, alone: the server words the thread notice from
|
|
728
832
|
// it, and anything wrapped around it reads as a crash instead of a stop.
|
|
729
|
-
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
|
|
833
|
+
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep, runtime.model());
|
|
730
834
|
if (!closed.ok) {
|
|
731
|
-
log(` ! could not report the cancellation (
|
|
835
|
+
log(` ! could not report the cancellation (${describeFailure(closed)})` +
|
|
836
|
+
`${closed.status !== 0 && closed.error ? ` ${closed.error}` : ''}`);
|
|
732
837
|
return 'unreported';
|
|
733
838
|
}
|
|
734
839
|
return 'cancelled';
|
|
@@ -743,9 +848,10 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
|
|
|
743
848
|
log(` [dry-run] would report the run as failed`);
|
|
744
849
|
return 'skipped';
|
|
745
850
|
}
|
|
746
|
-
const completed = await completeWithRetry(api, run.completion_path, { ok: false, error: detail ? `${reason}: ${detail}` : reason }, log, sleep);
|
|
851
|
+
const completed = await completeWithRetry(api, run.completion_path, { ok: false, error: detail ? `${reason}: ${detail}` : reason }, log, sleep, runtime.model());
|
|
747
852
|
if (!completed.ok) {
|
|
748
|
-
log(` ! could not report failure (
|
|
853
|
+
log(` ! could not report failure (${describeFailure(completed)})` +
|
|
854
|
+
`${completed.status !== 0 && completed.error ? ` ${completed.error}` : ''}`);
|
|
749
855
|
return 'unreported';
|
|
750
856
|
}
|
|
751
857
|
return 'failed';
|
|
@@ -770,9 +876,9 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
|
|
|
770
876
|
log(`${timestamp()} ✓ ${label} finished in ${seconds}s (${why})`);
|
|
771
877
|
for (const line of reply.split('\n'))
|
|
772
878
|
log(` │ ${line}`);
|
|
773
|
-
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep);
|
|
879
|
+
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep, runtime.model());
|
|
774
880
|
if (!completed.ok) {
|
|
775
|
-
log(` ! could not mark the run complete (
|
|
881
|
+
log(` ! could not mark the run complete (${describeFailure(completed)})`);
|
|
776
882
|
return 'unreported';
|
|
777
883
|
}
|
|
778
884
|
if (result.sessionId) {
|
|
@@ -780,16 +886,20 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
|
|
|
780
886
|
}
|
|
781
887
|
return 'succeeded';
|
|
782
888
|
}
|
|
783
|
-
const
|
|
889
|
+
const replyPath = run.reply_path;
|
|
890
|
+
const replyBodyKey = run.reply_body_key;
|
|
891
|
+
const posted = await postReplyWithReplay(api, () => postReply(api, replyPath, replyBodyKey, reply, run.run_id, run.reply_body_extra), label, log, sleep);
|
|
784
892
|
if (!posted.ok) {
|
|
785
|
-
|
|
893
|
+
const failure = describeFailure(posted);
|
|
894
|
+
const detail = posted.status === 0 ? '' : ` ${posted.error ?? ''}`;
|
|
895
|
+
log(`${timestamp()} ✗ ${label} reply rejected (${failure})${detail}`);
|
|
786
896
|
// Undo the mid-run recording. The session exists and the agent's transcript
|
|
787
897
|
// holds this turn, but the conversation never received its answer, so
|
|
788
898
|
// continuing it would skip reasoning nobody can see — the same rule that
|
|
789
899
|
// used to be expressed by simply not recording until the reply landed.
|
|
790
900
|
if (result.sessionRecorded)
|
|
791
901
|
runtime.sessions.forget(sessionKeyFor(run));
|
|
792
|
-
const reported = await completeWithRetry(api, run.completion_path, { ok: false, error: `posting the reply failed
|
|
902
|
+
const reported = await completeWithRetry(api, run.completion_path, { ok: false, error: `posting the reply failed: ${failure}` }, log, sleep, runtime.model());
|
|
793
903
|
return reported.ok ? 'failed' : 'unreported';
|
|
794
904
|
}
|
|
795
905
|
// Advance continuity only after the answer is durable on its reply surface.
|
|
@@ -798,13 +908,13 @@ async function handleRun(run, options, runtime, api, log, sleep, control) {
|
|
|
798
908
|
if (result.sessionId) {
|
|
799
909
|
runtime.sessions.remember(sessionKeyFor(run), result.sessionId);
|
|
800
910
|
}
|
|
801
|
-
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep);
|
|
911
|
+
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep, runtime.model());
|
|
802
912
|
if (!completed.ok) {
|
|
803
913
|
// The reply is already in the conversation, so this is a bookkeeping
|
|
804
914
|
// failure, not a lost answer — say so precisely. Returning false lets the
|
|
805
915
|
// caller forget the run so a redelivery can retry the completion; the
|
|
806
916
|
// reply's idempotency key stops that from posting twice.
|
|
807
|
-
log(`${timestamp()} ! ${label} replied, but marking it complete failed (
|
|
917
|
+
log(`${timestamp()} ! ${label} replied, but marking it complete failed (${describeFailure(completed)})`);
|
|
808
918
|
return 'unreported';
|
|
809
919
|
}
|
|
810
920
|
log(`${timestamp()} ✓ ${label} replied in ${seconds}s (${reply.length} chars)`);
|
|
@@ -847,6 +957,44 @@ export async function runAgentHarness(options) {
|
|
|
847
957
|
/** Accepted, not yet reported — the executing run and everything queued. */
|
|
848
958
|
const accepted = new Map();
|
|
849
959
|
const heartbeats = new HeartbeatLoop({ api, log, intervalMs: options.heartbeatIntervalMs });
|
|
960
|
+
const status = new RuntimeStatusReporter({
|
|
961
|
+
runtime: runtime.runtimeName,
|
|
962
|
+
provider: options.quotaProvider === undefined
|
|
963
|
+
? defaultQuotaProvider(runtime.brain, options.exec)
|
|
964
|
+
: (options.quotaProvider ?? undefined),
|
|
965
|
+
// A run in flight reports fresh numbers when it ends; don't probe beside it.
|
|
966
|
+
isBusy: () => [...accepted.values()].some((entry) => entry.executing),
|
|
967
|
+
log,
|
|
968
|
+
});
|
|
969
|
+
runtime.observe = (observation) => status.observe(observation);
|
|
970
|
+
runtime.model = () => (runtime.brain === 'plain' ? undefined : status.model());
|
|
971
|
+
let statusPrimed = false;
|
|
972
|
+
/**
|
|
973
|
+
* First-connect facts that need no run: the CLI's version and any model the
|
|
974
|
+
* exec string or Codex config pins. Done lazily — only once a server has
|
|
975
|
+
* asked for status — so nothing is probed for a server that can't use it.
|
|
976
|
+
*/
|
|
977
|
+
const primeStatus = () => {
|
|
978
|
+
if (statusPrimed || runtime.brain === 'plain')
|
|
979
|
+
return;
|
|
980
|
+
statusPrimed = true;
|
|
981
|
+
const binary = runtime.brain === 'codex'
|
|
982
|
+
? codexBinary(options.exec)
|
|
983
|
+
: (options.exec.trim().split(/\s+/)[0] ?? 'claude');
|
|
984
|
+
const probe = options.probeImpl ?? defaultProbe;
|
|
985
|
+
let version;
|
|
986
|
+
try {
|
|
987
|
+
version = parseVersion(probe(`${binary} --version`, runtime.workdir).stdout);
|
|
988
|
+
}
|
|
989
|
+
catch {
|
|
990
|
+
version = undefined;
|
|
991
|
+
}
|
|
992
|
+
status.observe({
|
|
993
|
+
runtime_version: version,
|
|
994
|
+
model: modelFromExec(options.exec) ??
|
|
995
|
+
(runtime.brain === 'codex' ? codexConfiguredModel({ ...process.env, ...options.env }) : undefined),
|
|
996
|
+
});
|
|
997
|
+
};
|
|
850
998
|
let queue = Promise.resolve();
|
|
851
999
|
let fatal;
|
|
852
1000
|
let attempt = 0;
|
|
@@ -934,6 +1082,7 @@ export async function runAgentHarness(options) {
|
|
|
934
1082
|
// any cancellation the heartbeat loop started and deliberately did not wait
|
|
935
1083
|
// for, so the harness never exits with a run half-closed.
|
|
936
1084
|
await queue;
|
|
1085
|
+
status.detach();
|
|
937
1086
|
// Stopped first, so no further tick can start work after the wait below has
|
|
938
1087
|
// taken its snapshot of `background`.
|
|
939
1088
|
heartbeats.stop();
|
|
@@ -945,7 +1094,11 @@ export async function runAgentHarness(options) {
|
|
|
945
1094
|
return metrics;
|
|
946
1095
|
/** One socket lifetime. Resolves ``true`` when the session ever authed. */
|
|
947
1096
|
function connectOnce() {
|
|
948
|
-
|
|
1097
|
+
// The `User-Agent` names the build server-side, the same as every HTTP
|
|
1098
|
+
// call this CLI makes; `client_version` in the auth frame is the copy the
|
|
1099
|
+
// server reads, because a header on an upgrade request is easy to lose in
|
|
1100
|
+
// proxies and logs and a frame field is not.
|
|
1101
|
+
const ws = new Ctor(url, { headers: { 'User-Agent': USER_AGENT }, handshakeTimeout: HANDSHAKE_TIMEOUT_MS });
|
|
949
1102
|
let authed = false;
|
|
950
1103
|
return new Promise((resolve) => {
|
|
951
1104
|
const close = () => {
|
|
@@ -962,7 +1115,33 @@ export async function runAgentHarness(options) {
|
|
|
962
1115
|
else
|
|
963
1116
|
options.signal.addEventListener('abort', close, { once: true });
|
|
964
1117
|
}
|
|
1118
|
+
// The server pings every 30 s and reaps after 60 s without a pong — but
|
|
1119
|
+
// that close never reaches a client whose kernel still believes the
|
|
1120
|
+
// connection is up (a laptop that slept through the FIN). Keep our own
|
|
1121
|
+
// deadline: no frame of any kind for the window means the socket is
|
|
1122
|
+
// dead, and tearing it down here is what makes the reconnect loop run.
|
|
1123
|
+
const liveness = new ServerLiveness({
|
|
1124
|
+
timeoutMs: options.livenessTimeoutMs,
|
|
1125
|
+
heartbeatIntervalMs: options.heartbeatLogIntervalMs,
|
|
1126
|
+
onSilence: (silentMs) => {
|
|
1127
|
+
metrics.livenessTimeouts += 1;
|
|
1128
|
+
log(`${timestamp()} … no server ping for ${seconds(silentMs)}s — reconnecting`);
|
|
1129
|
+
terminateSocket(ws);
|
|
1130
|
+
},
|
|
1131
|
+
onHeartbeat: (state) => {
|
|
1132
|
+
if (!authed)
|
|
1133
|
+
return;
|
|
1134
|
+
const since = state.sinceLastPingMs === undefined
|
|
1135
|
+
? `no server ping yet (connected ${seconds(state.sinceLastFrameMs)}s ago)`
|
|
1136
|
+
: `last server ping ${seconds(state.sinceLastPingMs)}s ago`;
|
|
1137
|
+
log(`${timestamp()} ● connected — ${since}`);
|
|
1138
|
+
},
|
|
1139
|
+
});
|
|
965
1140
|
ws.on('open', () => {
|
|
1141
|
+
// Kernel-level probe underneath the application-level one: a dead
|
|
1142
|
+
// peer is detected by the TCP stack even while the socket is idle.
|
|
1143
|
+
enableTcpKeepAlive(ws);
|
|
1144
|
+
liveness.start();
|
|
966
1145
|
// `subscription_id` is a presence signal, not a scoping one: the server
|
|
967
1146
|
// derives which subscription's events this socket receives from the
|
|
968
1147
|
// connection itself. Sending it stamps `connected_from` with this
|
|
@@ -972,6 +1151,7 @@ export async function runAgentHarness(options) {
|
|
|
972
1151
|
type: 'auth',
|
|
973
1152
|
api_key: options.credentials.api_key,
|
|
974
1153
|
machine,
|
|
1154
|
+
client_version: CLI_VERSION,
|
|
975
1155
|
...(options.credentials.subscription_id
|
|
976
1156
|
? { subscription_id: options.credentials.subscription_id }
|
|
977
1157
|
: {}),
|
|
@@ -983,11 +1163,16 @@ export async function runAgentHarness(options) {
|
|
|
983
1163
|
frame = JSON.parse(data.toString());
|
|
984
1164
|
}
|
|
985
1165
|
catch {
|
|
1166
|
+
// Unparseable, but the server sent it: still proof it is there.
|
|
1167
|
+
liveness.observe(undefined);
|
|
986
1168
|
return;
|
|
987
1169
|
}
|
|
988
|
-
if (!frame || typeof frame !== 'object')
|
|
1170
|
+
if (!frame || typeof frame !== 'object') {
|
|
1171
|
+
liveness.observe(undefined);
|
|
989
1172
|
return;
|
|
1173
|
+
}
|
|
990
1174
|
const f = frame;
|
|
1175
|
+
liveness.observe(f.type);
|
|
991
1176
|
if (f.type === 'auth_ok') {
|
|
992
1177
|
authed = true;
|
|
993
1178
|
const name = options.credentials.agent?.name ?? 'agent';
|
|
@@ -999,6 +1184,12 @@ export async function runAgentHarness(options) {
|
|
|
999
1184
|
// read mention code instead of the run dispatcher.
|
|
1000
1185
|
log(`${timestamp()} ● connected as "${name}" — waiting for ` +
|
|
1001
1186
|
`${EVENT_AGENT_RUN_REQUESTED} (exec: ${options.exec})`);
|
|
1187
|
+
// Only a server that says it understands the frame gets one; an
|
|
1188
|
+
// older server may treat an unknown frame type as a protocol error.
|
|
1189
|
+
if (Array.isArray(f.features) && f.features.includes(RUNTIME_STATUS_FEATURE)) {
|
|
1190
|
+
primeStatus();
|
|
1191
|
+
status.attach((frame) => ws.send(JSON.stringify(frame)));
|
|
1192
|
+
}
|
|
1002
1193
|
return;
|
|
1003
1194
|
}
|
|
1004
1195
|
if (f.type === 'auth_error') {
|
|
@@ -1161,6 +1352,8 @@ export async function runAgentHarness(options) {
|
|
|
1161
1352
|
});
|
|
1162
1353
|
});
|
|
1163
1354
|
ws.on('close', () => {
|
|
1355
|
+
liveness.stop();
|
|
1356
|
+
status.detach();
|
|
1164
1357
|
if (options.signal)
|
|
1165
1358
|
options.signal.removeEventListener('abort', close);
|
|
1166
1359
|
resolve(authed);
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime, model and remaining capacity — what `agent run` tells ProhostAI about itself.
|
|
3
|
+
*
|
|
4
|
+
* ProhostAI shows every paired AI employee with its runtime ("Claude Code"),
|
|
5
|
+
* model ("Opus 5.5"), machine and how much of the operator's Claude or Codex
|
|
6
|
+
* subscription is left. Only this machine can see those things, so the harness
|
|
7
|
+
* reports them in a `runtime_status` frame on the socket it already holds.
|
|
8
|
+
*
|
|
9
|
+
* **Credentials never leave this machine, and this module never reads them.**
|
|
10
|
+
* Every number comes from the agent CLI answering with its own login:
|
|
11
|
+
*
|
|
12
|
+
* - Claude Code: the `rate_limit_event` and `system/init` lines of a run's own
|
|
13
|
+
* `stream-json` output, and — when idle — the `get_usage` control request to
|
|
14
|
+
* an unmodified `claude -p --input-format stream-json` (no model turn).
|
|
15
|
+
* - Codex: the rollout file Codex writes for each run under
|
|
16
|
+
* `$CODEX_HOME/sessions`, and — when idle — `account/rateLimits/read` on
|
|
17
|
+
* `codex app-server`.
|
|
18
|
+
*
|
|
19
|
+
* Nothing here opens the Claude Code keychain item or `~/.codex/auth.json`, and
|
|
20
|
+
* nothing forwards an account id, email or org. Only labels, percentages and
|
|
21
|
+
* reset times go on the wire. Unknown or missing data is omitted, never sent as
|
|
22
|
+
* zero.
|
|
23
|
+
*
|
|
24
|
+
* Every parser here fails soft, in the same spirit as `readClaudeStream`: the
|
|
25
|
+
* shapes belong to other tools and grow without asking us.
|
|
26
|
+
*/
|
|
27
|
+
import { spawn as nodeSpawn } from 'node:child_process';
|
|
28
|
+
/** Feature name the server lists in `auth_ok.features` when it accepts the frame. */
|
|
29
|
+
export declare const RUNTIME_STATUS_FEATURE = "runtime_status";
|
|
30
|
+
/** Routine report cadence while connected. */
|
|
31
|
+
export declare const STATUS_INTERVAL_MS: number;
|
|
32
|
+
/**
|
|
33
|
+
* Minimum spacing between frames. The server accepts one per 30 s per socket
|
|
34
|
+
* and drops the rest; a second of margin keeps clock jitter from costing us
|
|
35
|
+
* the frame that carried a change.
|
|
36
|
+
*/
|
|
37
|
+
export declare const MIN_FRAME_SPACING_MS = 31000;
|
|
38
|
+
/** Ask the agent CLI for fresh numbers when the last ones are this old. */
|
|
39
|
+
export declare const QUOTA_REFRESH_AFTER_MS: number;
|
|
40
|
+
/** Ceiling on one idle quota probe. */
|
|
41
|
+
export declare const PROVIDER_TIMEOUT_MS = 10000;
|
|
42
|
+
export type QuotaKind = 'session' | 'weekly';
|
|
43
|
+
export interface QuotaWindow {
|
|
44
|
+
kind: QuotaKind;
|
|
45
|
+
/** 0–100. */
|
|
46
|
+
used_percent: number;
|
|
47
|
+
/** ISO-8601. */
|
|
48
|
+
resets_at: string;
|
|
49
|
+
}
|
|
50
|
+
/** What one source managed to learn. Every field is optional; absent means "didn't learn it". */
|
|
51
|
+
export interface RuntimeObservation {
|
|
52
|
+
runtime_version?: string;
|
|
53
|
+
model?: string;
|
|
54
|
+
plan_label?: string;
|
|
55
|
+
quota_windows?: QuotaWindow[];
|
|
56
|
+
}
|
|
57
|
+
/** A percentage on a 0–``scale`` scale, as 0–100 rounded to one decimal. */
|
|
58
|
+
export declare function percent(value: unknown, scale: 1 | 100): number | undefined;
|
|
59
|
+
/** Epoch seconds or an ISO string, as ISO-8601; anything else is `undefined`. */
|
|
60
|
+
export declare function isoTime(value: unknown): string | undefined;
|
|
61
|
+
/** First `X.Y.Z…` in a `--version` line, e.g. `2.1.282 (Claude Code)` or `codex-cli 0.146.0`. */
|
|
62
|
+
export declare function parseVersion(stdout: string): string | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Windows from a stream `rate_limit_event.rate_limit_info`.
|
|
65
|
+
*
|
|
66
|
+
* `unifiedWindows.*.utilization` is a FRACTION (0–1, and it can exceed 1) with
|
|
67
|
+
* epoch-second resets — unlike `get_usage`, which is 0–100 with ISO strings.
|
|
68
|
+
* The field is marked internal upstream, so an unexpected shape yields nothing.
|
|
69
|
+
*/
|
|
70
|
+
export declare function fromClaudeRateLimitInfo(info: unknown): QuotaWindow[];
|
|
71
|
+
/** Windows from a `get_usage` control response (0–100, ISO strings). */
|
|
72
|
+
export declare function fromClaudeGetUsage(usage: unknown): QuotaWindow[];
|
|
73
|
+
/** `get_usage.subscription_type` → display label; `undefined` for anything else. */
|
|
74
|
+
export declare function claudePlanLabel(subscriptionType: unknown): string | undefined;
|
|
75
|
+
/**
|
|
76
|
+
* Windows from Codex `rate_limits` (rollout, snake_case) or `rateLimits`
|
|
77
|
+
* (app-server, camelCase).
|
|
78
|
+
*
|
|
79
|
+
* Classified by window LENGTH, never by slot: on some plans `primary` is the
|
|
80
|
+
* weekly window and there is no 5-hour one at all.
|
|
81
|
+
*/
|
|
82
|
+
export declare function fromCodexRateLimits(rateLimits: unknown, lineTimestamp?: string): QuotaWindow[];
|
|
83
|
+
/** `plan_type` / `planType` → `ChatGPT Pro`; `undefined` when there is no plan. */
|
|
84
|
+
export declare function codexPlanLabel(planType: unknown): string | undefined;
|
|
85
|
+
/** Model named in the exec string itself (`-m gpt-5` / `--model gpt-5` / `--model=gpt-5`). */
|
|
86
|
+
export declare function modelFromExec(exec: string): string | undefined;
|
|
87
|
+
/** `$CODEX_HOME`, defaulting to `~/.codex` exactly as Codex does. */
|
|
88
|
+
export declare function codexHome(env?: NodeJS.ProcessEnv): string;
|
|
89
|
+
/**
|
|
90
|
+
* The top-level `model = "…"` in `$CODEX_HOME/config.toml` — Codex's default.
|
|
91
|
+
*
|
|
92
|
+
* Only the top level, before the first `[table]`: a `model` inside a profile
|
|
93
|
+
* table is not the default. A first guess only; the rollout's `turn_context`
|
|
94
|
+
* replaces it after the first run.
|
|
95
|
+
*/
|
|
96
|
+
export declare function codexConfiguredModel(env?: NodeJS.ProcessEnv): string | undefined;
|
|
97
|
+
/**
|
|
98
|
+
* Locate the rollout for one thread: `sessions/YYYY/MM/DD/rollout-*-<thread_id>.jsonl`.
|
|
99
|
+
*
|
|
100
|
+
* The date directory is local time and a resumed thread lives under the day it
|
|
101
|
+
* started, so the date is never guessed — days are scanned newest first.
|
|
102
|
+
*/
|
|
103
|
+
export declare function findCodexRollout(threadId: string, env?: NodeJS.ProcessEnv): string | undefined;
|
|
104
|
+
/**
|
|
105
|
+
* What a Codex rollout says: the model the turn really used (after `-m`,
|
|
106
|
+
* profiles and `-c`), the CLI version, and the last rate-limit snapshot.
|
|
107
|
+
*/
|
|
108
|
+
export declare function parseCodexRollout(content: string): RuntimeObservation;
|
|
109
|
+
/** Read the rollout for a finished Codex thread, if it can be found. */
|
|
110
|
+
export declare function readCodexObservation(threadId: string | undefined, env?: NodeJS.ProcessEnv): RuntimeObservation | undefined;
|
|
111
|
+
/**
|
|
112
|
+
* Where fresh numbers come from when no run has produced any lately.
|
|
113
|
+
*
|
|
114
|
+
* Implementations ask the unmodified agent CLI through its own protocol, so
|
|
115
|
+
* the CLI answers with its own login and this process never holds a
|
|
116
|
+
* credential. Resolve `undefined` on any failure — the reporter keeps what it
|
|
117
|
+
* had and omits what it doesn't know.
|
|
118
|
+
*/
|
|
119
|
+
export interface QuotaProvider {
|
|
120
|
+
refresh(): Promise<RuntimeObservation | undefined>;
|
|
121
|
+
}
|
|
122
|
+
type SpawnFn = typeof nodeSpawn;
|
|
123
|
+
/**
|
|
124
|
+
* Claude Code's `get_usage` control request. No model turn is made, so no
|
|
125
|
+
* quota is spent. The `initialize` response carries account identity; it is
|
|
126
|
+
* never read.
|
|
127
|
+
*/
|
|
128
|
+
export declare function claudeQuotaProvider(options: {
|
|
129
|
+
binary: string;
|
|
130
|
+
spawnImpl?: SpawnFn;
|
|
131
|
+
timeoutMs?: number;
|
|
132
|
+
}): QuotaProvider;
|
|
133
|
+
/** Codex's `account/rateLimits/read` over `codex app-server` stdio JSON-RPC. */
|
|
134
|
+
export declare function codexQuotaProvider(options: {
|
|
135
|
+
binary: string;
|
|
136
|
+
clientVersion: string;
|
|
137
|
+
spawnImpl?: SpawnFn;
|
|
138
|
+
timeoutMs?: number;
|
|
139
|
+
}): QuotaProvider;
|
|
140
|
+
export interface RuntimeStatusReporterOptions {
|
|
141
|
+
/** `claude_code` | `codex` | `goose` | `custom`. */
|
|
142
|
+
runtime: string;
|
|
143
|
+
/** Idle quota source; omit for a runtime that has none. */
|
|
144
|
+
provider?: QuotaProvider;
|
|
145
|
+
/** A run is executing — it will bring fresh numbers, so don't probe beside it. */
|
|
146
|
+
isBusy?: () => boolean;
|
|
147
|
+
now?: () => number;
|
|
148
|
+
/** Timer seam for tests. Returns a cancel function. */
|
|
149
|
+
schedule?: (fn: () => void, ms: number) => () => void;
|
|
150
|
+
intervalMs?: number;
|
|
151
|
+
log?: (line: string) => void;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Holds the latest runtime snapshot and decides when to send it.
|
|
155
|
+
*
|
|
156
|
+
* Sends only while attached — i.e. only on a socket whose server advertised
|
|
157
|
+
* {@link RUNTIME_STATUS_FEATURE}. Cadence: once on attach, every
|
|
158
|
+
* {@link STATUS_INTERVAL_MS}, and promptly when the model, plan or runtime
|
|
159
|
+
* version changes or capacity crosses a band — but never closer together than
|
|
160
|
+
* {@link MIN_FRAME_SPACING_MS}.
|
|
161
|
+
*/
|
|
162
|
+
export declare class RuntimeStatusReporter {
|
|
163
|
+
private readonly options;
|
|
164
|
+
private readonly now;
|
|
165
|
+
private readonly schedule;
|
|
166
|
+
private state;
|
|
167
|
+
private quotaObservedAt;
|
|
168
|
+
private send;
|
|
169
|
+
private lastSentAt;
|
|
170
|
+
private cancelPending;
|
|
171
|
+
private cancelTick;
|
|
172
|
+
private refreshing;
|
|
173
|
+
constructor(options: RuntimeStatusReporterOptions);
|
|
174
|
+
/** Merge what a source learned. Absent fields keep their previous value. */
|
|
175
|
+
observe(observation: RuntimeObservation | undefined): void;
|
|
176
|
+
/** The model to report on a run's completion, when nothing more specific is known. */
|
|
177
|
+
model(): string | undefined;
|
|
178
|
+
/** Start reporting on a socket that advertised the feature. */
|
|
179
|
+
attach(send: (frame: Record<string, unknown>) => void): void;
|
|
180
|
+
/** Stop reporting — the socket is gone. */
|
|
181
|
+
detach(): void;
|
|
182
|
+
/** The frame as it would be sent now. */
|
|
183
|
+
frame(): Record<string, unknown>;
|
|
184
|
+
/** Routine tick: refresh idle quota if it is stale, send, and re-arm. */
|
|
185
|
+
private tick;
|
|
186
|
+
private refresh;
|
|
187
|
+
/** Send now if spacing allows, else once it does. Coalesces. */
|
|
188
|
+
private sendSoon;
|
|
189
|
+
private flush;
|
|
190
|
+
}
|
|
191
|
+
export {};
|