@prohost/cli 0.12.4 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +27 -3
- package/dist/agent/api.d.ts +15 -7
- package/dist/agent/api.js +6 -1
- package/dist/agent/claude.d.ts +7 -0
- package/dist/agent/claude.js +4 -0
- package/dist/agent/contract.d.ts +37 -0
- package/dist/agent/contract.js +18 -0
- package/dist/agent/exec.d.ts +6 -0
- package/dist/agent/exec.js +5 -1
- package/dist/agent/heartbeat.d.ts +13 -0
- package/dist/agent/heartbeat.js +9 -1
- package/dist/agent/run.js +407 -151
- package/dist/agent/runtime_status.d.ts +6 -0
- package/dist/agent/runtime_status.js +5 -0
- package/dist/agent/venue.d.ts +163 -0
- package/dist/agent/venue.js +272 -0
- package/dist/version.d.ts +2 -2
- package/dist/version.js +1 -1
- package/package.json +2 -2
package/dist/agent/run.js
CHANGED
|
@@ -48,6 +48,7 @@ import { DEFAULT_CONCURRENCY, MAX_CONCURRENCY, RunScheduler } from './scheduler.
|
|
|
48
48
|
import { RUNTIME_STATUS_FEATURE, RuntimeStatusReporter, claudeQuotaProvider, codexConfiguredModel, codexQuotaProvider, modelFromExec, parseVersion, readCodexObservation, } from './runtime_status.js';
|
|
49
49
|
import { SessionStore, sessionKeyFor } from './sessionstore.js';
|
|
50
50
|
import { verifySignature } from './signature.js';
|
|
51
|
+
import { VENUE_RUN_TOKEN_CAPABILITY, apiForRun, claudeSupportsVenueSessions, claudeVenueCommand, VENUE_PROGRESS, VENUE_SESSION_FEATURE, VENUE_SESSION_FRAME, VENUE_SESSION_OPEN_FRAME, VENUE_GRANT_CHECK_MS, VENUE_EXPIRY_MARGIN_MS, VENUE_GRANT_ENDED_ERROR, VENUE_GRANT_EXPIRED_ERROR, isVenueRun, venueEnv, venueGrantLive, venueGrantRemainingMs, venueRunFromSession, venueWorkdir, } from './venue.js';
|
|
51
52
|
import { RUN_REPOS_ENV_VAR, WorktreeManager } from './worktrees.js';
|
|
52
53
|
import { isWebhookEvent } from '../forward.js';
|
|
53
54
|
import { DEFAULT_WS_URL } from '../listen.js';
|
|
@@ -108,6 +109,10 @@ export const PAIRED_RUN_ENV_VAR = 'PROHOST_PAIRED_RUN';
|
|
|
108
109
|
const ACK_ACCEPTED = 200;
|
|
109
110
|
const ACK_IGNORED = 204;
|
|
110
111
|
const ACK_REJECTED = 401;
|
|
112
|
+
/** Received, but the run could not be taken on yet: the server keeps retrying the delivery. */
|
|
113
|
+
const ACK_RETRY = 503;
|
|
114
|
+
/** How long the harness waits for the server to answer a venue-session open. */
|
|
115
|
+
const VENUE_SESSION_TIMEOUT_MS = 30_000;
|
|
111
116
|
function newMetrics() {
|
|
112
117
|
return {
|
|
113
118
|
connections: 0,
|
|
@@ -379,6 +384,26 @@ function buildRuntime(options, log) {
|
|
|
379
384
|
account,
|
|
380
385
|
};
|
|
381
386
|
}
|
|
387
|
+
/**
|
|
388
|
+
* The preparation a venue run gets in place of {@link HarnessRuntime.prepareMcp}:
|
|
389
|
+
* the account that runs the brain, and nothing else — see `venue.ts`.
|
|
390
|
+
*
|
|
391
|
+
* Fails the run closed on a harness that cannot hold a venue session to that.
|
|
392
|
+
* The server sends venue runs only to a harness that declared it can, so this
|
|
393
|
+
* is reached only by a run that should never have arrived.
|
|
394
|
+
*/
|
|
395
|
+
function prepareVenueSession(runtime) {
|
|
396
|
+
if (runtime.brain !== 'claude' || !runtime.venueSessions || !runtime.account) {
|
|
397
|
+
return {
|
|
398
|
+
mcpWired: false,
|
|
399
|
+
error: "this harness cannot answer in a connected team's thread (it needs a Claude Code with --tools and --safe-mode)",
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
const resolved = runtime.account.resolveForSpawn();
|
|
403
|
+
if (!resolved.ok)
|
|
404
|
+
return { mcpWired: false, error: resolved.error };
|
|
405
|
+
return { mcpWired: false, execEnv: runtime.account.spawnEnv(), accountLabel: resolved.record.label };
|
|
406
|
+
}
|
|
382
407
|
/**
|
|
383
408
|
* The `runtime` a status frame names. A plain command is `custom` unless it is
|
|
384
409
|
* recognisably goose — the one other agent CLI people pair today.
|
|
@@ -611,7 +636,9 @@ async function runTurn(run, prompt, options, runtime, mcp, control, log,
|
|
|
611
636
|
*/
|
|
612
637
|
toolEvents,
|
|
613
638
|
/** Harness-set variables for this run's command, over the brain's own. */
|
|
614
|
-
runEnv
|
|
639
|
+
runEnv,
|
|
640
|
+
/** A venue run's own empty working directory; marks the turn as a venue session. */
|
|
641
|
+
venueCwd) {
|
|
615
642
|
// `--timeout` is a ceiling on the run, not on each attempt: the retry below
|
|
616
643
|
// must not let one run take twice what the operator allowed. Both attempts
|
|
617
644
|
// share one deadline, which costs the retry nothing in practice — a rejected
|
|
@@ -635,7 +662,7 @@ runEnv) {
|
|
|
635
662
|
// remembering the session would leave the next *real* run continuing from an
|
|
636
663
|
// answer nobody saw. Resolved once, up here, because the streaming reader
|
|
637
664
|
// records it mid-run rather than at the end of the turn.
|
|
638
|
-
const sessionKey = options.dryRun ? undefined : sessionKeyFor(run);
|
|
665
|
+
const sessionKey = options.dryRun || venueCwd !== undefined ? undefined : sessionKeyFor(run);
|
|
639
666
|
let recordedSession;
|
|
640
667
|
/** The reader for the attempt currently running, if this brain streams. */
|
|
641
668
|
let stream;
|
|
@@ -687,6 +714,10 @@ runEnv) {
|
|
|
687
714
|
runtime.sessions.forget(sessionKey);
|
|
688
715
|
recordedSession = undefined;
|
|
689
716
|
};
|
|
717
|
+
// A venue session (`venueCwd` set): its own empty directory, an environment
|
|
718
|
+
// with nothing of ProhostAI's in it, and no session to resume or remember —
|
|
719
|
+
// see `venue.ts`.
|
|
720
|
+
const venue = venueCwd !== undefined;
|
|
690
721
|
const exec = (command) => {
|
|
691
722
|
// A fresh reader per attempt: a retry without `--resume` is a different
|
|
692
723
|
// session, and inheriting the failed attempt's id would resume nothing.
|
|
@@ -698,8 +729,9 @@ runEnv) {
|
|
|
698
729
|
prompt,
|
|
699
730
|
timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
|
|
700
731
|
idleTimeoutMs: idleMs,
|
|
701
|
-
cwd: runtime.workdir,
|
|
702
|
-
env: { ...mcp.execEnv, ...runEnv },
|
|
732
|
+
cwd: venue ? venueCwd : runtime.workdir,
|
|
733
|
+
env: venue ? venueEnv({ ...process.env, ...options.env, ...mcp.execEnv }) : { ...mcp.execEnv, ...runEnv },
|
|
734
|
+
...(venue ? { inheritEnv: false } : {}),
|
|
703
735
|
spawnImpl: options.spawnImpl,
|
|
704
736
|
cancelSignal: control.signal,
|
|
705
737
|
onStdout: reader
|
|
@@ -727,14 +759,19 @@ runEnv) {
|
|
|
727
759
|
* A turn the Claude subscription's usage limit stopped, worded for the
|
|
728
760
|
* thread: whose limit (this machine's login) and, in Claude Code's own
|
|
729
761
|
* words, when it resets. The only fix is waiting, so the reason says that
|
|
730
|
-
* rather than "exited 1"
|
|
762
|
+
* rather than "exited 1" — and the completion says when, so the server can
|
|
763
|
+
* run the work again then.
|
|
731
764
|
*/
|
|
732
|
-
const usageLimitFailure = (said, stderr = '') =>
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
765
|
+
const usageLimitFailure = (said, stderr = '') => {
|
|
766
|
+
const resetsAt = stream?.usageLimitResetsAt();
|
|
767
|
+
return {
|
|
768
|
+
ok: false,
|
|
769
|
+
reason: `Claude usage limit reached on ${options.machine ?? os.hostname()}`,
|
|
770
|
+
detail: [said, stderr.includes(said) ? '' : stderr].filter((part) => part.length > 0).join('\n'),
|
|
771
|
+
durationMs: elapsed(),
|
|
772
|
+
usageLimit: resetsAt === undefined ? {} : { retryAfter: new Date(resetsAt * 1000).toISOString() },
|
|
773
|
+
};
|
|
774
|
+
};
|
|
738
775
|
const failure = (result) => {
|
|
739
776
|
// A killed run has no reply, but it usually has most of one. Reporting the
|
|
740
777
|
// tail of what the agent was writing turns "no output for 15 minutes" into
|
|
@@ -799,19 +836,22 @@ runEnv) {
|
|
|
799
836
|
const sessionId = runtime.sessions.get(sessionKey);
|
|
800
837
|
const codex = runtime.brain === 'codex';
|
|
801
838
|
// Same three questions for both brains — resume this thread, wire these
|
|
802
|
-
// tools, gate or don't — asked with each one's own flags.
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
839
|
+
// tools, gate or don't — asked with each one's own flags. A venue session
|
|
840
|
+
// answers none of them the operator's way: see `claudeVenueCommand`.
|
|
841
|
+
const build = (resume) => venue
|
|
842
|
+
? claudeVenueCommand(options.exec, modelFromExec(options.exec))
|
|
843
|
+
: codex
|
|
844
|
+
? codexCommand(options.exec, {
|
|
845
|
+
sessionId: resume,
|
|
846
|
+
mcpEntries: mcp.codexMcpEntries,
|
|
847
|
+
configOverrides: mcp.codexConfigOverrides,
|
|
848
|
+
skipPermissions: runtime.skipPermissions,
|
|
849
|
+
})
|
|
850
|
+
: claudeCommand(options.exec, {
|
|
851
|
+
sessionId: resume,
|
|
852
|
+
mcpConfigPath: mcp.mcpConfigPath,
|
|
853
|
+
skipPermissions: runtime.skipPermissions,
|
|
854
|
+
});
|
|
815
855
|
let result = await exec(build(sessionId));
|
|
816
856
|
// A remembered session may simply be gone — transcripts are cleaned up and
|
|
817
857
|
// the ledger can't know. One retry without it, because the alternative is a
|
|
@@ -913,10 +953,22 @@ async function handleRun(run, options, runtime, api, log, sleep, control, isolat
|
|
|
913
953
|
mcp) {
|
|
914
954
|
const label = `run ${run.run_id.slice(0, 12)}`;
|
|
915
955
|
const surface = run.conversation_id ?? run.kind ?? 'no conversation';
|
|
916
|
-
|
|
956
|
+
// Answered with the run's own token, by a session with no tools and none of
|
|
957
|
+
// this machine's ProhostAI credentials — see `venue.ts`.
|
|
958
|
+
const venue = isVenueRun(run);
|
|
959
|
+
log(`${timestamp()} ▶ ${label} started (${surface}${venue ? ", a connected team's thread" : ''})`);
|
|
960
|
+
if (venue && !run.venue_run_token) {
|
|
961
|
+
// Nothing this harness could do for it is allowed: the server refuses this
|
|
962
|
+
// machine's own key for a venue run, and running the agent without the
|
|
963
|
+
// token would hand a connected team's prompt to a session that holds it.
|
|
964
|
+
log(` ! ${label} is in a connected team's thread but carries no run token — not running it`);
|
|
965
|
+
return 'unreported';
|
|
966
|
+
}
|
|
917
967
|
// Before anything is spawned: the prompt names these paths. A dry run makes
|
|
918
|
-
// no writes, and a worktree is one.
|
|
919
|
-
const checkouts = isolation.worktrees.enabled && !options.dryRun
|
|
968
|
+
// no writes, and a worktree is one. A venue session gets none.
|
|
969
|
+
const checkouts = isolation.worktrees.enabled && !options.dryRun && !venue
|
|
970
|
+
? await isolation.worktrees.prepare(isolation.lane)
|
|
971
|
+
: undefined;
|
|
920
972
|
for (const failure of checkouts?.failures ?? []) {
|
|
921
973
|
log(` ! no private checkout of ${failure.source} for ${label} (${failure.reason})`);
|
|
922
974
|
}
|
|
@@ -934,7 +986,7 @@ mcp) {
|
|
|
934
986
|
// it is. See PromptContext.mcpToolPattern.
|
|
935
987
|
mcpToolPattern: runtime.brain !== 'plain' && mcp.mcpWired ? MCP_TOOL_PATTERN : undefined,
|
|
936
988
|
backofficeRead: capabilities.backofficeRead,
|
|
937
|
-
toolsGated: !runtime.skipPermissions,
|
|
989
|
+
toolsGated: venue || !runtime.skipPermissions,
|
|
938
990
|
// The real numbers, not a vague "there is a limit" — see PromptContext.
|
|
939
991
|
timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
|
|
940
992
|
idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
|
|
@@ -953,7 +1005,37 @@ mcp) {
|
|
|
953
1005
|
const runEnv = { [PAIRED_RUN_ENV_VAR]: '1' };
|
|
954
1006
|
if (checkouts && checkouts.checkouts.length > 0)
|
|
955
1007
|
runEnv[RUN_REPOS_ENV_VAR] = checkouts.dir;
|
|
956
|
-
|
|
1008
|
+
// The last moment before the agent starts: a venue run that waited in the
|
|
1009
|
+
// queue past its grant is not started at all, and nothing is posted.
|
|
1010
|
+
if (venue && !venueGrantLive(run)) {
|
|
1011
|
+
await toolEvents.flush();
|
|
1012
|
+
toolEvents.stop();
|
|
1013
|
+
log(`${timestamp()} ⏹ ${label}'s connected-team grant expired before it could start — not running it`);
|
|
1014
|
+
if (options.dryRun) {
|
|
1015
|
+
log(' [dry-run] would report the run as failed');
|
|
1016
|
+
return 'skipped';
|
|
1017
|
+
}
|
|
1018
|
+
// Closed now, while the margin still leaves the token good: left open, the
|
|
1019
|
+
// row would keep its thinking indicator until the stale-run sweep. No
|
|
1020
|
+
// model: none ran.
|
|
1021
|
+
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: VENUE_GRANT_EXPIRED_ERROR }, log, sleep);
|
|
1022
|
+
return closed.ok ? 'failed' : 'unreported';
|
|
1023
|
+
}
|
|
1024
|
+
const venueDir = venue && !mcp.error ? venueWorkdir() : undefined;
|
|
1025
|
+
// What a venue run says about itself lands on its row in this agent's home
|
|
1026
|
+
// workspace, and on this machine's log: never the agent's words about the
|
|
1027
|
+
// connected team's thread (see `VENUE_PROGRESS`).
|
|
1028
|
+
const turnControl = venue ? { ...control, report: () => control.report(VENUE_PROGRESS) } : control;
|
|
1029
|
+
let result;
|
|
1030
|
+
try {
|
|
1031
|
+
result = await runTurn(run, prompt, options, runtime, mcp, turnControl, log, toolEvents, runEnv, venueDir?.dir);
|
|
1032
|
+
}
|
|
1033
|
+
finally {
|
|
1034
|
+
venueDir?.remove();
|
|
1035
|
+
}
|
|
1036
|
+
// A venue session is never continued: the next turn there starts fresh from
|
|
1037
|
+
// the thread the server sends, so nothing of this one is remembered.
|
|
1038
|
+
const sessionId = venue ? undefined : result.ok ? result.sessionId : undefined;
|
|
957
1039
|
// Whatever the turn did, the timeline is whole BEFORE the reply is posted or
|
|
958
1040
|
// the run is closed — a completion that beats its own last tool row would
|
|
959
1041
|
// leave a `calling` row on a finished run. Never rejects, and after `stop`
|
|
@@ -972,6 +1054,18 @@ mcp) {
|
|
|
972
1054
|
log(`${timestamp()} ⏹ ${label} was already closed server-side (${seconds}s) — nothing was posted`);
|
|
973
1055
|
return 'resolved';
|
|
974
1056
|
}
|
|
1057
|
+
if (stopped === 'expired') {
|
|
1058
|
+
// Stopped while the margin still leaves the token good, so it closes its
|
|
1059
|
+
// own run rather than leaving the row to the stale-run sweep. Nothing of
|
|
1060
|
+
// what it was saying is posted.
|
|
1061
|
+
log(`${timestamp()} ⏹ ${label}'s connected-team grant ran out after ${seconds}s — nothing was posted`);
|
|
1062
|
+
if (options.dryRun) {
|
|
1063
|
+
log(' [dry-run] would report the run as failed');
|
|
1064
|
+
return 'skipped';
|
|
1065
|
+
}
|
|
1066
|
+
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: VENUE_GRANT_ENDED_ERROR }, log, sleep, runtime.model());
|
|
1067
|
+
return closed.ok ? 'failed' : 'unreported';
|
|
1068
|
+
}
|
|
975
1069
|
if (stopped === 'cancelled') {
|
|
976
1070
|
log(`${timestamp()} ⏹ ${label} cancelled after ${seconds}s`);
|
|
977
1071
|
if (options.dryRun) {
|
|
@@ -990,7 +1084,9 @@ mcp) {
|
|
|
990
1084
|
}
|
|
991
1085
|
if (!result.ok) {
|
|
992
1086
|
const { reason } = result;
|
|
993
|
-
|
|
1087
|
+
// A venue run's detail is stderr and the agent's unfinished answer — the
|
|
1088
|
+
// connected team's thread, retold. It stays out of the home run and the log.
|
|
1089
|
+
const detail = venue ? '' : (result.detail ?? '');
|
|
994
1090
|
log(`${timestamp()} ✗ ${label} failed after ${seconds}s — ${reason}`);
|
|
995
1091
|
if (detail)
|
|
996
1092
|
log(` stderr: ${detail}`);
|
|
@@ -998,7 +1094,11 @@ mcp) {
|
|
|
998
1094
|
log(` [dry-run] would report the run as failed`);
|
|
999
1095
|
return 'skipped';
|
|
1000
1096
|
}
|
|
1001
|
-
const completed = await completeWithRetry(api, run.completion_path, {
|
|
1097
|
+
const completed = await completeWithRetry(api, run.completion_path, {
|
|
1098
|
+
ok: false,
|
|
1099
|
+
error: detail ? `${reason}: ${detail}` : reason,
|
|
1100
|
+
...(result.usageLimit ? { usageLimit: result.usageLimit } : {}),
|
|
1101
|
+
}, log, sleep, runtime.model());
|
|
1002
1102
|
if (!completed.ok) {
|
|
1003
1103
|
log(` ! could not report failure (${describeFailure(completed)})` +
|
|
1004
1104
|
`${completed.status !== 0 && completed.error ? ` ${completed.error}` : ''}`);
|
|
@@ -1010,8 +1110,9 @@ mcp) {
|
|
|
1010
1110
|
if (options.dryRun) {
|
|
1011
1111
|
const destination = run.reply_surface === 'none' ? '(no reply surface)' : `${run.reply_surface} ${run.reply_path}`;
|
|
1012
1112
|
log(`${timestamp()} ◦ ${label} dry-run — would post to ${destination}:`);
|
|
1013
|
-
|
|
1014
|
-
|
|
1113
|
+
if (!venue)
|
|
1114
|
+
for (const line of reply.split('\n'))
|
|
1115
|
+
log(` │ ${line}`);
|
|
1015
1116
|
log(` [dry-run] nothing was sent and the run was left open`);
|
|
1016
1117
|
return 'skipped';
|
|
1017
1118
|
}
|
|
@@ -1027,8 +1128,8 @@ mcp) {
|
|
|
1027
1128
|
log(` ! could not mark the run complete (${describeFailure(completed)})`);
|
|
1028
1129
|
return 'unreported';
|
|
1029
1130
|
}
|
|
1030
|
-
if (
|
|
1031
|
-
runtime.sessions.remember(sessionKeyFor(run),
|
|
1131
|
+
if (sessionId) {
|
|
1132
|
+
runtime.sessions.remember(sessionKeyFor(run), sessionId);
|
|
1032
1133
|
}
|
|
1033
1134
|
return 'succeeded';
|
|
1034
1135
|
}
|
|
@@ -1041,15 +1142,16 @@ mcp) {
|
|
|
1041
1142
|
? 'no reply surface'
|
|
1042
1143
|
: `${run.reply_surface} runs have nowhere to post yet`;
|
|
1043
1144
|
log(`${timestamp()} ✓ ${label} finished in ${seconds}s (${why})`);
|
|
1044
|
-
|
|
1045
|
-
|
|
1145
|
+
if (!venue)
|
|
1146
|
+
for (const line of reply.split('\n'))
|
|
1147
|
+
log(` │ ${line}`);
|
|
1046
1148
|
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep, runtime.model());
|
|
1047
1149
|
if (!completed.ok) {
|
|
1048
1150
|
log(` ! could not mark the run complete (${describeFailure(completed)})`);
|
|
1049
1151
|
return 'unreported';
|
|
1050
1152
|
}
|
|
1051
|
-
if (
|
|
1052
|
-
runtime.sessions.remember(sessionKeyFor(run),
|
|
1153
|
+
if (sessionId) {
|
|
1154
|
+
runtime.sessions.remember(sessionKeyFor(run), sessionId);
|
|
1053
1155
|
}
|
|
1054
1156
|
return 'succeeded';
|
|
1055
1157
|
}
|
|
@@ -1058,7 +1160,9 @@ mcp) {
|
|
|
1058
1160
|
const posted = await postReplyWithReplay(api, () => postReply(api, replyPath, replyBodyKey, reply, run.run_id, run.reply_body_extra), label, log, sleep);
|
|
1059
1161
|
if (!posted.ok) {
|
|
1060
1162
|
const failure = describeFailure(posted);
|
|
1061
|
-
|
|
1163
|
+
// The server's reason can quote the thread (a reply-race refusal previews
|
|
1164
|
+
// the other reply), so a venue run logs the status alone.
|
|
1165
|
+
const detail = posted.status === 0 || venue ? '' : ` ${posted.error ?? ''}`;
|
|
1062
1166
|
log(`${timestamp()} ✗ ${label} reply rejected (${failure})${detail}`);
|
|
1063
1167
|
// Undo the mid-run recording. The session exists and the agent's transcript
|
|
1064
1168
|
// holds this turn, but the conversation never received its answer, so
|
|
@@ -1072,8 +1176,8 @@ mcp) {
|
|
|
1072
1176
|
// Advance continuity only after the answer is durable on its reply surface.
|
|
1073
1177
|
// Remembering at process exit would strand the conversation behind an answer
|
|
1074
1178
|
// it never received whenever this POST is rejected.
|
|
1075
|
-
if (
|
|
1076
|
-
runtime.sessions.remember(sessionKeyFor(run),
|
|
1179
|
+
if (sessionId) {
|
|
1180
|
+
runtime.sessions.remember(sessionKeyFor(run), sessionId);
|
|
1077
1181
|
}
|
|
1078
1182
|
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep, runtime.model());
|
|
1079
1183
|
if (!completed.ok) {
|
|
@@ -1124,6 +1228,12 @@ export async function runAgentHarness(options) {
|
|
|
1124
1228
|
// duplicate — the run is in flight right here. Only a run in the ledger that
|
|
1125
1229
|
// this process never handled is a candidate for restart recovery.
|
|
1126
1230
|
const handledThisSession = new Set();
|
|
1231
|
+
/**
|
|
1232
|
+
* Venue runs whose session is still being opened, and whether it opened. A
|
|
1233
|
+
* duplicate delivery in that window is acknowledged the way the original
|
|
1234
|
+
* will be — never as accepted before a session exists.
|
|
1235
|
+
*/
|
|
1236
|
+
const venueOpenings = new Map();
|
|
1127
1237
|
/** Accepted, not yet reported — the executing run and everything queued. */
|
|
1128
1238
|
const accepted = new Map();
|
|
1129
1239
|
const heartbeats = new HeartbeatLoop({ api, log, intervalMs: options.heartbeatIntervalMs });
|
|
@@ -1138,6 +1248,7 @@ export async function runAgentHarness(options) {
|
|
|
1138
1248
|
accountKeys: runtime.account ? () => runtime.account.keys() : undefined,
|
|
1139
1249
|
// A run in flight reports fresh numbers when it ends; don't probe beside it.
|
|
1140
1250
|
isBusy: () => [...accepted.values()].some((entry) => entry.executing),
|
|
1251
|
+
capabilities: () => (runtime.venueSessions ? [VENUE_RUN_TOKEN_CAPABILITY] : []),
|
|
1141
1252
|
log,
|
|
1142
1253
|
});
|
|
1143
1254
|
runtime.observe = (observation) => status.observe(observation);
|
|
@@ -1163,6 +1274,14 @@ export async function runAgentHarness(options) {
|
|
|
1163
1274
|
catch {
|
|
1164
1275
|
version = undefined;
|
|
1165
1276
|
}
|
|
1277
|
+
if (runtime.brain === 'claude') {
|
|
1278
|
+
try {
|
|
1279
|
+
runtime.venueSessions = claudeSupportsVenueSessions(probe(`${binary} --help`, runtime.workdir).stdout);
|
|
1280
|
+
}
|
|
1281
|
+
catch {
|
|
1282
|
+
runtime.venueSessions = false;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1166
1285
|
status.observe({
|
|
1167
1286
|
runtime_version: version,
|
|
1168
1287
|
model: modelFromExec(options.exec) ??
|
|
@@ -1247,7 +1366,7 @@ export async function runAgentHarness(options) {
|
|
|
1247
1366
|
log(' [dry-run] would report the run as cancelled');
|
|
1248
1367
|
return;
|
|
1249
1368
|
}
|
|
1250
|
-
const closed = await completeWithRetry(api, entry.run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
|
|
1369
|
+
const closed = await completeWithRetry(apiForRun(api, entry.run), entry.run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
|
|
1251
1370
|
if (closed.ok)
|
|
1252
1371
|
runs.markCompleted(entry.run.run_id);
|
|
1253
1372
|
metrics.runsFailed += 1;
|
|
@@ -1271,6 +1390,30 @@ export async function runAgentHarness(options) {
|
|
|
1271
1390
|
runs.markCompleted(entry.run.run_id);
|
|
1272
1391
|
log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was closed server-side before it started`);
|
|
1273
1392
|
};
|
|
1393
|
+
/**
|
|
1394
|
+
* A venue run's grant is about to end. Stopped while the margin still
|
|
1395
|
+
* leaves its token good, so the run is closed with it: an executing run is
|
|
1396
|
+
* aborted and `handleRun` completes it; a queued one is closed out here.
|
|
1397
|
+
*/
|
|
1398
|
+
const expireAccepted = async (entry) => {
|
|
1399
|
+
if (entry.disposition)
|
|
1400
|
+
return;
|
|
1401
|
+
entry.disposition = 'expired';
|
|
1402
|
+
if (entry.executing) {
|
|
1403
|
+
entry.controller.abort();
|
|
1404
|
+
return;
|
|
1405
|
+
}
|
|
1406
|
+
release(entry.run.run_id);
|
|
1407
|
+
log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)}'s connected-team grant expired before it started`);
|
|
1408
|
+
if (options.dryRun) {
|
|
1409
|
+
log(' [dry-run] would report the run as failed');
|
|
1410
|
+
return;
|
|
1411
|
+
}
|
|
1412
|
+
const closed = await completeWithRetry(apiForRun(api, entry.run), entry.run.completion_path, { ok: false, error: VENUE_GRANT_EXPIRED_ERROR }, log, sleep);
|
|
1413
|
+
if (closed.ok)
|
|
1414
|
+
runs.markCompleted(entry.run.run_id);
|
|
1415
|
+
metrics.runsFailed += 1;
|
|
1416
|
+
};
|
|
1274
1417
|
const exhausted = () => options.maxConnections !== undefined && metrics.connections >= options.maxConnections;
|
|
1275
1418
|
while (!options.signal?.aborted && fatal === undefined && !exhausted()) {
|
|
1276
1419
|
metrics.connections += 1;
|
|
@@ -1343,6 +1486,36 @@ export async function runAgentHarness(options) {
|
|
|
1343
1486
|
log(`${timestamp()} ● connected — ${since}`);
|
|
1344
1487
|
},
|
|
1345
1488
|
});
|
|
1489
|
+
/** This server opens venue sessions over the socket (`auth_ok.features`). */
|
|
1490
|
+
let venueSessionsOffered = false;
|
|
1491
|
+
/** Venue-session opens awaiting the server's answer, by run. */
|
|
1492
|
+
const venueWaiters = new Map();
|
|
1493
|
+
/**
|
|
1494
|
+
* Open a run's venue session over this socket. `null` means no answer —
|
|
1495
|
+
* the socket went, or the server took too long — and the open can be
|
|
1496
|
+
* tried again on a redelivery; any answer is final.
|
|
1497
|
+
*/
|
|
1498
|
+
const askVenueSession = (runId) => {
|
|
1499
|
+
if (!venueSessionsOffered)
|
|
1500
|
+
return Promise.resolve({ ok: false });
|
|
1501
|
+
return new Promise((resolve) => {
|
|
1502
|
+
const timer = setTimeout(() => {
|
|
1503
|
+
venueWaiters.delete(runId);
|
|
1504
|
+
resolve(null);
|
|
1505
|
+
}, VENUE_SESSION_TIMEOUT_MS);
|
|
1506
|
+
venueWaiters.set(runId, (answer) => {
|
|
1507
|
+
clearTimeout(timer);
|
|
1508
|
+
resolve(answer);
|
|
1509
|
+
});
|
|
1510
|
+
try {
|
|
1511
|
+
ws.send(JSON.stringify({ type: VENUE_SESSION_OPEN_FRAME, run_id: runId }));
|
|
1512
|
+
}
|
|
1513
|
+
catch {
|
|
1514
|
+
venueWaiters.get(runId)?.(null);
|
|
1515
|
+
venueWaiters.delete(runId);
|
|
1516
|
+
}
|
|
1517
|
+
});
|
|
1518
|
+
};
|
|
1346
1519
|
ws.on('open', () => {
|
|
1347
1520
|
// Kernel-level probe underneath the application-level one: a dead
|
|
1348
1521
|
// peer is detected by the TCP stack even while the socket is idle.
|
|
@@ -1404,6 +1577,7 @@ export async function runAgentHarness(options) {
|
|
|
1404
1577
|
}
|
|
1405
1578
|
runtime.account?.noteRequestedAt(pending.requested_at);
|
|
1406
1579
|
}
|
|
1580
|
+
venueSessionsOffered = Array.isArray(f.features) && f.features.includes(VENUE_SESSION_FEATURE);
|
|
1407
1581
|
if (Array.isArray(f.features) && f.features.includes(RUNTIME_STATUS_FEATURE)) {
|
|
1408
1582
|
primeStatus();
|
|
1409
1583
|
status.attach((frame) => ws.send(JSON.stringify(frame)));
|
|
@@ -1427,6 +1601,13 @@ export async function runAgentHarness(options) {
|
|
|
1427
1601
|
ws.send(JSON.stringify({ type: 'pong' }));
|
|
1428
1602
|
return;
|
|
1429
1603
|
}
|
|
1604
|
+
if (f.type === VENUE_SESSION_FRAME) {
|
|
1605
|
+
const runId = String(f.run_id);
|
|
1606
|
+
const waiter = venueWaiters.get(runId);
|
|
1607
|
+
venueWaiters.delete(runId);
|
|
1608
|
+
waiter?.(f);
|
|
1609
|
+
return;
|
|
1610
|
+
}
|
|
1430
1611
|
if (!isWebhookEvent(frame))
|
|
1431
1612
|
return;
|
|
1432
1613
|
metrics.eventsReceived += 1;
|
|
@@ -1483,129 +1664,204 @@ export async function runAgentHarness(options) {
|
|
|
1483
1664
|
return;
|
|
1484
1665
|
}
|
|
1485
1666
|
const previous = runs.get(run.run_id);
|
|
1486
|
-
|
|
1667
|
+
const opening = venueOpenings.get(run.run_id);
|
|
1668
|
+
if (opening) {
|
|
1487
1669
|
metrics.duplicates += 1;
|
|
1488
|
-
ack(ACK_ACCEPTED);
|
|
1670
|
+
void opening.then((opened) => ack(opened ? ACK_ACCEPTED : ACK_RETRY));
|
|
1489
1671
|
return;
|
|
1490
1672
|
}
|
|
1491
|
-
|
|
1492
|
-
handledThisSession.add(run.run_id);
|
|
1493
|
-
if (previous) {
|
|
1494
|
-
// We ran this one before but never got a terminal state accepted —
|
|
1495
|
-
// most likely the process died between the command and the
|
|
1496
|
-
// completion. Retry only the completion: re-running the agent could
|
|
1497
|
-
// repeat whatever it did with its credential the first time.
|
|
1673
|
+
if (previous?.completed || handledThisSession.has(run.run_id)) {
|
|
1498
1674
|
metrics.duplicates += 1;
|
|
1499
|
-
|
|
1500
|
-
log(`${timestamp()} ↻ run ${run.run_id.slice(0, 12)} was started before but never reported; ` +
|
|
1501
|
-
'reporting it now without re-running the agent');
|
|
1502
|
-
const recovered = await completeWithRetry(api, previous.completion_path || run.completion_path, { ok: false, error: 'the harness restarted before this run was reported' }, log, sleep);
|
|
1503
|
-
if (recovered.ok)
|
|
1504
|
-
runs.markCompleted(run.run_id);
|
|
1505
|
-
});
|
|
1675
|
+
ack(ACK_ACCEPTED);
|
|
1506
1676
|
return;
|
|
1507
1677
|
}
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
log(`${timestamp()} ✗ closing cancelled run ${run.run_id} failed: ${String(err)}`);
|
|
1525
|
-
}));
|
|
1526
|
-
},
|
|
1527
|
-
onResolved: () => resolveAccepted(entry),
|
|
1528
|
-
});
|
|
1529
|
-
// Queue execution but never block the socket reader: the ack and the
|
|
1530
|
-
// heartbeat have to keep flowing while the agent thinks. The lane is
|
|
1531
|
-
// what keeps two turns of one conversation from overlapping.
|
|
1532
|
-
const lane = laneFor(run);
|
|
1533
|
-
scheduler.submit(lane, async () => {
|
|
1534
|
-
// Settled while it waited — cancelled, or closed server-side. Both
|
|
1535
|
-
// were dealt with where they were decided; there is nothing to run
|
|
1536
|
-
// and nothing left to report.
|
|
1537
|
-
if (entry.disposition)
|
|
1538
|
-
return;
|
|
1539
|
-
// Shutdown was requested while this sat in the queue. Don't start a
|
|
1540
|
-
// fresh command the operator is waiting to be rid of — but do close
|
|
1541
|
-
// the run out, so the server isn't left waiting on it either.
|
|
1542
|
-
if (options.signal?.aborted) {
|
|
1543
|
-
release(run.run_id);
|
|
1544
|
-
log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
|
|
1545
|
-
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
|
|
1546
|
-
if (closed.ok)
|
|
1547
|
-
runs.markCompleted(run.run_id);
|
|
1548
|
-
metrics.runsFailed += 1;
|
|
1678
|
+
handledThisSession.add(run.run_id);
|
|
1679
|
+
/** Take the run on: track it, heartbeat it, queue it. */
|
|
1680
|
+
const accept = (run) => {
|
|
1681
|
+
if (previous) {
|
|
1682
|
+
// We ran this one before but never got a terminal state accepted —
|
|
1683
|
+
// most likely the process died between the command and the
|
|
1684
|
+
// completion. Retry only the completion: re-running the agent could
|
|
1685
|
+
// repeat whatever it did with its credential the first time.
|
|
1686
|
+
metrics.duplicates += 1;
|
|
1687
|
+
scheduler.submit(laneFor(run), async () => {
|
|
1688
|
+
log(`${timestamp()} ↻ run ${run.run_id.slice(0, 12)} was started before but never reported; ` +
|
|
1689
|
+
'reporting it now without re-running the agent');
|
|
1690
|
+
const recovered = await completeWithRetry(apiForRun(api, run), previous.completion_path || run.completion_path, { ok: false, error: 'the harness restarted before this run was reported' }, log, sleep);
|
|
1691
|
+
if (recovered.ok)
|
|
1692
|
+
runs.markCompleted(run.run_id);
|
|
1693
|
+
});
|
|
1549
1694
|
return;
|
|
1550
1695
|
}
|
|
1551
|
-
//
|
|
1552
|
-
//
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1696
|
+
// Recorded BEFORE the command runs — a crash mid-run must not let a
|
|
1697
|
+
// redelivery execute the agent a second time.
|
|
1698
|
+
runs.markStarted(run.run_id, run.completion_path);
|
|
1699
|
+
metrics.runsStarted += 1;
|
|
1700
|
+
// Heartbeat from the moment it is accepted, not from the moment it
|
|
1701
|
+
// starts. The row is `running` server-side either way, and a run that
|
|
1702
|
+
// waits an hour behind a long job is reaped exactly like one that has
|
|
1703
|
+
// been working for an hour.
|
|
1704
|
+
const entry = { run, controller: new AbortController(), executing: false };
|
|
1705
|
+
accepted.set(run.run_id, entry);
|
|
1706
|
+
heartbeats.track(run.run_id, {
|
|
1707
|
+
path: heartbeatPathFor(run),
|
|
1708
|
+
api: apiForRun(api, run),
|
|
1709
|
+
venue: isVenueRun(run),
|
|
1710
|
+
progress: () => entry.progress,
|
|
1711
|
+
// Deliberately not awaited here — see `background`.
|
|
1712
|
+
onCancelRequested: () => {
|
|
1713
|
+
background.push(cancelAccepted(entry).catch((err) => {
|
|
1714
|
+
log(`${timestamp()} ✗ closing cancelled run ${run.run_id} failed: ${String(err)}`);
|
|
1715
|
+
}));
|
|
1560
1716
|
},
|
|
1561
|
-
|
|
1562
|
-
const isolation = { lane, parallel: concurrency > 1, worktrees };
|
|
1563
|
-
// Resolved here rather than inside `handleRun` so the account this
|
|
1564
|
-
// run is on can be handed back however the run ends.
|
|
1565
|
-
let mcp;
|
|
1566
|
-
try {
|
|
1567
|
-
mcp = runtime.prepareMcp();
|
|
1568
|
-
}
|
|
1569
|
-
catch (err) {
|
|
1570
|
-
// Fails the run closed, and — unlike a throw from here — still
|
|
1571
|
-
// reports it, so the server is not left waiting on it.
|
|
1572
|
-
mcp = { mcpWired: false, error: `could not prepare the run (${String(err)})` };
|
|
1573
|
-
}
|
|
1574
|
-
const outcome = await handleRun(run, options, runtime, api, log, sleep, control, isolation, mcp).catch((err) => {
|
|
1575
|
-
log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
|
|
1576
|
-
return 'failed';
|
|
1717
|
+
onResolved: () => resolveAccepted(entry),
|
|
1577
1718
|
});
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
//
|
|
1581
|
-
//
|
|
1582
|
-
//
|
|
1583
|
-
|
|
1584
|
-
//
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1719
|
+
// A venue run's grant ends at its expiry whether or not the server
|
|
1720
|
+
// can be reached to say so: stop it just before then, while the
|
|
1721
|
+
// margin still lets its token close the run (heartbeats stop it
|
|
1722
|
+
// sooner on any refusal). Re-checked at least every
|
|
1723
|
+
// VENUE_GRANT_CHECK_MS rather than timed once: a timer counts on the
|
|
1724
|
+
// monotonic clock, which can stand still while the machine sleeps,
|
|
1725
|
+
// and the grant does not.
|
|
1726
|
+
if (run.venue_deadline_ms !== undefined) {
|
|
1727
|
+
const checkGrant = () => {
|
|
1728
|
+
if (!accepted.has(run.run_id) || entry.disposition)
|
|
1729
|
+
return;
|
|
1730
|
+
const left = venueGrantRemainingMs(run) - VENUE_EXPIRY_MARGIN_MS;
|
|
1731
|
+
if (left > 0) {
|
|
1732
|
+
setTimeout(checkGrant, Math.min(VENUE_GRANT_CHECK_MS, left)).unref?.();
|
|
1733
|
+
return;
|
|
1734
|
+
}
|
|
1735
|
+
log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)}'s connected-team grant is ending — stopping it`);
|
|
1736
|
+
background.push(expireAccepted(entry).catch((err) => {
|
|
1737
|
+
log(`${timestamp()} ✗ closing expired run ${run.run_id} failed: ${String(err)}`);
|
|
1738
|
+
}));
|
|
1739
|
+
};
|
|
1740
|
+
setTimeout(checkGrant, 0).unref?.();
|
|
1597
1741
|
}
|
|
1598
|
-
//
|
|
1599
|
-
//
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1742
|
+
// Queue execution but never block the socket reader: the ack and the
|
|
1743
|
+
// heartbeat have to keep flowing while the agent thinks. The lane is
|
|
1744
|
+
// what keeps two turns of one conversation from overlapping.
|
|
1745
|
+
const lane = laneFor(run);
|
|
1746
|
+
scheduler.submit(lane, async () => {
|
|
1747
|
+
// Settled while it waited — cancelled, or closed server-side. Both
|
|
1748
|
+
// were dealt with where they were decided; there is nothing to run
|
|
1749
|
+
// and nothing left to report.
|
|
1750
|
+
if (entry.disposition)
|
|
1751
|
+
return;
|
|
1752
|
+
// Shutdown was requested while this sat in the queue. Don't start a
|
|
1753
|
+
// fresh command the operator is waiting to be rid of — but do close
|
|
1754
|
+
// the run out, so the server isn't left waiting on it either.
|
|
1755
|
+
if (options.signal?.aborted) {
|
|
1756
|
+
release(run.run_id);
|
|
1757
|
+
log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
|
|
1758
|
+
const closed = await completeWithRetry(apiForRun(api, run), run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
|
|
1759
|
+
if (closed.ok)
|
|
1760
|
+
runs.markCompleted(run.run_id);
|
|
1761
|
+
metrics.runsFailed += 1;
|
|
1762
|
+
return;
|
|
1763
|
+
}
|
|
1764
|
+
// Set before the first `await`, so a cancellation arriving from the
|
|
1765
|
+
// heartbeat either sees a queued run (and closes it out itself) or an
|
|
1766
|
+
// executing one (and aborts it) — never both, and never neither.
|
|
1767
|
+
entry.executing = true;
|
|
1768
|
+
const control = {
|
|
1769
|
+
signal: entry.controller.signal,
|
|
1770
|
+
disposition: () => entry.disposition,
|
|
1771
|
+
report: (activity) => {
|
|
1772
|
+
entry.progress = activity;
|
|
1773
|
+
},
|
|
1774
|
+
};
|
|
1775
|
+
const isolation = { lane, parallel: concurrency > 1, worktrees };
|
|
1776
|
+
// Resolved here rather than inside `handleRun` so the account this
|
|
1777
|
+
// run is on can be handed back however the run ends.
|
|
1778
|
+
let mcp;
|
|
1779
|
+
try {
|
|
1780
|
+
// A venue session gets no MCP at all — not the ProhostAI server, not
|
|
1781
|
+
// the operator's extras — only the account that runs the brain.
|
|
1782
|
+
mcp = isVenueRun(run) ? prepareVenueSession(runtime) : runtime.prepareMcp();
|
|
1783
|
+
}
|
|
1784
|
+
catch (err) {
|
|
1785
|
+
// Fails the run closed, and — unlike a throw from here — still
|
|
1786
|
+
// reports it, so the server is not left waiting on it.
|
|
1787
|
+
mcp = { mcpWired: false, error: `could not prepare the run (${String(err)})` };
|
|
1788
|
+
}
|
|
1789
|
+
const outcome = await handleRun(run, options, runtime, apiForRun(api, run), log, sleep, control, isolation, mcp).catch((err) => {
|
|
1790
|
+
log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
|
|
1791
|
+
return 'failed';
|
|
1792
|
+
});
|
|
1793
|
+
if (mcp.accountLabel !== undefined)
|
|
1794
|
+
runtime.account?.finishSpawn(mcp.accountLabel);
|
|
1795
|
+
// Held until here on purpose: heartbeats keep the row alive for the
|
|
1796
|
+
// whole of `handleRun`, including the ten minutes of completion
|
|
1797
|
+
// retries at the end of it.
|
|
1798
|
+
release(run.run_id);
|
|
1799
|
+
// An account picked mid-run was held back from the status frame
|
|
1800
|
+
// until this run (on the old account) was done; report it now. Asked
|
|
1801
|
+
// about *this* run's account: a newer run may already be on the pick.
|
|
1802
|
+
if (runtime.account?.changedSinceSpawn(mcp.accountLabel))
|
|
1803
|
+
status.accountsChanged();
|
|
1804
|
+
// Both terminal outcomes reached the completion endpoint; only then
|
|
1805
|
+
// is the run settled and safe to skip on redelivery.
|
|
1806
|
+
if (outcome === 'succeeded')
|
|
1807
|
+
metrics.runsSucceeded += 1;
|
|
1808
|
+
// A cancellation is reported to the server as `failed`, and counted
|
|
1809
|
+
// the same way here — the run did not produce the answer it was for.
|
|
1810
|
+
if (outcome === 'failed' || outcome === 'unreported' || outcome === 'cancelled') {
|
|
1811
|
+
metrics.runsFailed += 1;
|
|
1812
|
+
}
|
|
1813
|
+
// Only a run the server acknowledged as terminal is settled; an
|
|
1814
|
+
// unreported one stays open so a redelivery can report it.
|
|
1815
|
+
if (outcome === 'succeeded' ||
|
|
1816
|
+
outcome === 'failed' ||
|
|
1817
|
+
outcome === 'cancelled' ||
|
|
1818
|
+
outcome === 'resolved') {
|
|
1819
|
+
runs.markCompleted(run.run_id);
|
|
1820
|
+
}
|
|
1821
|
+
});
|
|
1822
|
+
};
|
|
1823
|
+
if (!isVenueRun(run)) {
|
|
1824
|
+
ack(ACK_ACCEPTED);
|
|
1825
|
+
accept(run);
|
|
1826
|
+
return;
|
|
1827
|
+
}
|
|
1828
|
+
// A connected team's thread: the event carries none of it. Open the
|
|
1829
|
+
// run's venue session over this socket for the thread and the run's
|
|
1830
|
+
// own token before anything is tracked, so every request the run makes
|
|
1831
|
+
// — heartbeats included — goes out with the token and never with this
|
|
1832
|
+
// machine's key. The delivery is acknowledged only once the server has
|
|
1833
|
+
// answered: with no answer it stays retryable, so a dropped socket or
|
|
1834
|
+
// a slow server does not strand the run.
|
|
1835
|
+
const runId = run.run_id;
|
|
1836
|
+
let settleOpening = () => { };
|
|
1837
|
+
venueOpenings.set(runId, new Promise((resolve) => {
|
|
1838
|
+
settleOpening = resolve;
|
|
1839
|
+
}));
|
|
1840
|
+
const openedAt = { openedAtMono: performance.now(), openedAtWall: Date.now() };
|
|
1841
|
+
void askVenueSession(runId).then((answer) => {
|
|
1842
|
+
const opened = answer === null ? null : venueRunFromSession(run, eventData(frame.payload), answer, openedAt);
|
|
1843
|
+
venueOpenings.delete(runId);
|
|
1844
|
+
settleOpening(opened !== null && opened.ok);
|
|
1845
|
+
if (opened === null || !opened.ok) {
|
|
1846
|
+
// Not taken: the delivery stays retryable, and a redelivery comes
|
|
1847
|
+
// back through the server's socket check — which either hands the
|
|
1848
|
+
// run to this socket after all or ends it there. Acknowledging a
|
|
1849
|
+
// run this harness never took would leave it open with nobody on it.
|
|
1850
|
+
handledThisSession.delete(runId);
|
|
1851
|
+
ack(ACK_RETRY);
|
|
1852
|
+
log(`${timestamp()} … run ${runId.slice(0, 12)} is in a connected team's thread, but its venue ` +
|
|
1853
|
+
`session could not be opened (${opened === null ? 'no answer' : opened.error}) — not running it; ` +
|
|
1854
|
+
'the server will retry or end it');
|
|
1855
|
+
return;
|
|
1605
1856
|
}
|
|
1857
|
+
ack(ACK_ACCEPTED);
|
|
1858
|
+
accept(opened.run);
|
|
1606
1859
|
});
|
|
1607
1860
|
});
|
|
1608
1861
|
ws.on('close', () => {
|
|
1862
|
+
for (const waiter of venueWaiters.values())
|
|
1863
|
+
waiter(null);
|
|
1864
|
+
venueWaiters.clear();
|
|
1609
1865
|
liveness.stop();
|
|
1610
1866
|
status.detach();
|
|
1611
1867
|
if (options.signal)
|