@prohost/cli 0.12.3 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/README.md +21 -0
- package/dist/agent/api.js +1 -0
- package/dist/agent/contract.d.ts +30 -0
- package/dist/agent/contract.js +11 -0
- package/dist/agent/exec.d.ts +6 -0
- package/dist/agent/exec.js +5 -1
- package/dist/agent/heartbeat.d.ts +13 -0
- package/dist/agent/heartbeat.js +9 -1
- package/dist/agent/pair.js +1 -0
- package/dist/agent/run.js +390 -143
- package/dist/agent/runtime_status.d.ts +6 -0
- package/dist/agent/runtime_status.js +5 -0
- package/dist/agent/venue.d.ts +163 -0
- package/dist/agent/venue.js +272 -0
- package/dist/usage/command.js +6 -6
- package/dist/version.d.ts +2 -2
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/agent/run.js
CHANGED
|
@@ -48,6 +48,7 @@ import { DEFAULT_CONCURRENCY, MAX_CONCURRENCY, RunScheduler } from './scheduler.
|
|
|
48
48
|
import { RUNTIME_STATUS_FEATURE, RuntimeStatusReporter, claudeQuotaProvider, codexConfiguredModel, codexQuotaProvider, modelFromExec, parseVersion, readCodexObservation, } from './runtime_status.js';
|
|
49
49
|
import { SessionStore, sessionKeyFor } from './sessionstore.js';
|
|
50
50
|
import { verifySignature } from './signature.js';
|
|
51
|
+
import { VENUE_RUN_TOKEN_CAPABILITY, apiForRun, claudeSupportsVenueSessions, claudeVenueCommand, VENUE_PROGRESS, VENUE_SESSION_FEATURE, VENUE_SESSION_FRAME, VENUE_SESSION_OPEN_FRAME, VENUE_GRANT_CHECK_MS, VENUE_EXPIRY_MARGIN_MS, VENUE_GRANT_ENDED_ERROR, VENUE_GRANT_EXPIRED_ERROR, isVenueRun, venueEnv, venueGrantLive, venueGrantRemainingMs, venueRunFromSession, venueWorkdir, } from './venue.js';
|
|
51
52
|
import { RUN_REPOS_ENV_VAR, WorktreeManager } from './worktrees.js';
|
|
52
53
|
import { isWebhookEvent } from '../forward.js';
|
|
53
54
|
import { DEFAULT_WS_URL } from '../listen.js';
|
|
@@ -108,6 +109,10 @@ export const PAIRED_RUN_ENV_VAR = 'PROHOST_PAIRED_RUN';
|
|
|
108
109
|
const ACK_ACCEPTED = 200;
|
|
109
110
|
const ACK_IGNORED = 204;
|
|
110
111
|
const ACK_REJECTED = 401;
|
|
112
|
+
/** Received, but the run could not be taken on yet: the server keeps retrying the delivery. */
|
|
113
|
+
const ACK_RETRY = 503;
|
|
114
|
+
/** How long the harness waits for the server to answer a venue-session open. */
|
|
115
|
+
const VENUE_SESSION_TIMEOUT_MS = 30_000;
|
|
111
116
|
function newMetrics() {
|
|
112
117
|
return {
|
|
113
118
|
connections: 0,
|
|
@@ -379,6 +384,26 @@ function buildRuntime(options, log) {
|
|
|
379
384
|
account,
|
|
380
385
|
};
|
|
381
386
|
}
|
|
387
|
+
/**
|
|
388
|
+
* The preparation a venue run gets in place of {@link HarnessRuntime.prepareMcp}:
|
|
389
|
+
* the account that runs the brain, and nothing else — see `venue.ts`.
|
|
390
|
+
*
|
|
391
|
+
* Fails the run closed on a harness that cannot hold a venue session to that.
|
|
392
|
+
* The server sends venue runs only to a harness that declared it can, so this
|
|
393
|
+
* is reached only by a run that should never have arrived.
|
|
394
|
+
*/
|
|
395
|
+
function prepareVenueSession(runtime) {
|
|
396
|
+
if (runtime.brain !== 'claude' || !runtime.venueSessions || !runtime.account) {
|
|
397
|
+
return {
|
|
398
|
+
mcpWired: false,
|
|
399
|
+
error: "this harness cannot answer in a connected team's thread (it needs a Claude Code with --tools and --safe-mode)",
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
const resolved = runtime.account.resolveForSpawn();
|
|
403
|
+
if (!resolved.ok)
|
|
404
|
+
return { mcpWired: false, error: resolved.error };
|
|
405
|
+
return { mcpWired: false, execEnv: runtime.account.spawnEnv(), accountLabel: resolved.record.label };
|
|
406
|
+
}
|
|
382
407
|
/**
|
|
383
408
|
* The `runtime` a status frame names. A plain command is `custom` unless it is
|
|
384
409
|
* recognisably goose — the one other agent CLI people pair today.
|
|
@@ -611,7 +636,9 @@ async function runTurn(run, prompt, options, runtime, mcp, control, log,
|
|
|
611
636
|
*/
|
|
612
637
|
toolEvents,
|
|
613
638
|
/** Harness-set variables for this run's command, over the brain's own. */
|
|
614
|
-
runEnv
|
|
639
|
+
runEnv,
|
|
640
|
+
/** A venue run's own empty working directory; marks the turn as a venue session. */
|
|
641
|
+
venueCwd) {
|
|
615
642
|
// `--timeout` is a ceiling on the run, not on each attempt: the retry below
|
|
616
643
|
// must not let one run take twice what the operator allowed. Both attempts
|
|
617
644
|
// share one deadline, which costs the retry nothing in practice — a rejected
|
|
@@ -635,7 +662,7 @@ runEnv) {
|
|
|
635
662
|
// remembering the session would leave the next *real* run continuing from an
|
|
636
663
|
// answer nobody saw. Resolved once, up here, because the streaming reader
|
|
637
664
|
// records it mid-run rather than at the end of the turn.
|
|
638
|
-
const sessionKey = options.dryRun ? undefined : sessionKeyFor(run);
|
|
665
|
+
const sessionKey = options.dryRun || venueCwd !== undefined ? undefined : sessionKeyFor(run);
|
|
639
666
|
let recordedSession;
|
|
640
667
|
/** The reader for the attempt currently running, if this brain streams. */
|
|
641
668
|
let stream;
|
|
@@ -687,6 +714,10 @@ runEnv) {
|
|
|
687
714
|
runtime.sessions.forget(sessionKey);
|
|
688
715
|
recordedSession = undefined;
|
|
689
716
|
};
|
|
717
|
+
// A venue session (`venueCwd` set): its own empty directory, an environment
|
|
718
|
+
// with nothing of ProhostAI's in it, and no session to resume or remember —
|
|
719
|
+
// see `venue.ts`.
|
|
720
|
+
const venue = venueCwd !== undefined;
|
|
690
721
|
const exec = (command) => {
|
|
691
722
|
// A fresh reader per attempt: a retry without `--resume` is a different
|
|
692
723
|
// session, and inheriting the failed attempt's id would resume nothing.
|
|
@@ -698,8 +729,9 @@ runEnv) {
|
|
|
698
729
|
prompt,
|
|
699
730
|
timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
|
|
700
731
|
idleTimeoutMs: idleMs,
|
|
701
|
-
cwd: runtime.workdir,
|
|
702
|
-
env: { ...mcp.execEnv, ...runEnv },
|
|
732
|
+
cwd: venue ? venueCwd : runtime.workdir,
|
|
733
|
+
env: venue ? venueEnv({ ...process.env, ...options.env, ...mcp.execEnv }) : { ...mcp.execEnv, ...runEnv },
|
|
734
|
+
...(venue ? { inheritEnv: false } : {}),
|
|
703
735
|
spawnImpl: options.spawnImpl,
|
|
704
736
|
cancelSignal: control.signal,
|
|
705
737
|
onStdout: reader
|
|
@@ -799,19 +831,22 @@ runEnv) {
|
|
|
799
831
|
const sessionId = runtime.sessions.get(sessionKey);
|
|
800
832
|
const codex = runtime.brain === 'codex';
|
|
801
833
|
// Same three questions for both brains — resume this thread, wire these
|
|
802
|
-
// tools, gate or don't — asked with each one's own flags.
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
834
|
+
// tools, gate or don't — asked with each one's own flags. A venue session
|
|
835
|
+
// answers none of them the operator's way: see `claudeVenueCommand`.
|
|
836
|
+
const build = (resume) => venue
|
|
837
|
+
? claudeVenueCommand(options.exec, modelFromExec(options.exec))
|
|
838
|
+
: codex
|
|
839
|
+
? codexCommand(options.exec, {
|
|
840
|
+
sessionId: resume,
|
|
841
|
+
mcpEntries: mcp.codexMcpEntries,
|
|
842
|
+
configOverrides: mcp.codexConfigOverrides,
|
|
843
|
+
skipPermissions: runtime.skipPermissions,
|
|
844
|
+
})
|
|
845
|
+
: claudeCommand(options.exec, {
|
|
846
|
+
sessionId: resume,
|
|
847
|
+
mcpConfigPath: mcp.mcpConfigPath,
|
|
848
|
+
skipPermissions: runtime.skipPermissions,
|
|
849
|
+
});
|
|
815
850
|
let result = await exec(build(sessionId));
|
|
816
851
|
// A remembered session may simply be gone — transcripts are cleaned up and
|
|
817
852
|
// the ledger can't know. One retry without it, because the alternative is a
|
|
@@ -913,10 +948,22 @@ async function handleRun(run, options, runtime, api, log, sleep, control, isolat
|
|
|
913
948
|
mcp) {
|
|
914
949
|
const label = `run ${run.run_id.slice(0, 12)}`;
|
|
915
950
|
const surface = run.conversation_id ?? run.kind ?? 'no conversation';
|
|
916
|
-
|
|
951
|
+
// Answered with the run's own token, by a session with no tools and none of
|
|
952
|
+
// this machine's ProhostAI credentials — see `venue.ts`.
|
|
953
|
+
const venue = isVenueRun(run);
|
|
954
|
+
log(`${timestamp()} ▶ ${label} started (${surface}${venue ? ", a connected team's thread" : ''})`);
|
|
955
|
+
if (venue && !run.venue_run_token) {
|
|
956
|
+
// Nothing this harness could do for it is allowed: the server refuses this
|
|
957
|
+
// machine's own key for a venue run, and running the agent without the
|
|
958
|
+
// token would hand a connected team's prompt to a session that holds it.
|
|
959
|
+
log(` ! ${label} is in a connected team's thread but carries no run token — not running it`);
|
|
960
|
+
return 'unreported';
|
|
961
|
+
}
|
|
917
962
|
// Before anything is spawned: the prompt names these paths. A dry run makes
|
|
918
|
-
// no writes, and a worktree is one.
|
|
919
|
-
const checkouts = isolation.worktrees.enabled && !options.dryRun
|
|
963
|
+
// no writes, and a worktree is one. A venue session gets none.
|
|
964
|
+
const checkouts = isolation.worktrees.enabled && !options.dryRun && !venue
|
|
965
|
+
? await isolation.worktrees.prepare(isolation.lane)
|
|
966
|
+
: undefined;
|
|
920
967
|
for (const failure of checkouts?.failures ?? []) {
|
|
921
968
|
log(` ! no private checkout of ${failure.source} for ${label} (${failure.reason})`);
|
|
922
969
|
}
|
|
@@ -934,7 +981,7 @@ mcp) {
|
|
|
934
981
|
// it is. See PromptContext.mcpToolPattern.
|
|
935
982
|
mcpToolPattern: runtime.brain !== 'plain' && mcp.mcpWired ? MCP_TOOL_PATTERN : undefined,
|
|
936
983
|
backofficeRead: capabilities.backofficeRead,
|
|
937
|
-
toolsGated: !runtime.skipPermissions,
|
|
984
|
+
toolsGated: venue || !runtime.skipPermissions,
|
|
938
985
|
// The real numbers, not a vague "there is a limit" — see PromptContext.
|
|
939
986
|
timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
|
|
940
987
|
idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
|
|
@@ -953,7 +1000,37 @@ mcp) {
|
|
|
953
1000
|
const runEnv = { [PAIRED_RUN_ENV_VAR]: '1' };
|
|
954
1001
|
if (checkouts && checkouts.checkouts.length > 0)
|
|
955
1002
|
runEnv[RUN_REPOS_ENV_VAR] = checkouts.dir;
|
|
956
|
-
|
|
1003
|
+
// The last moment before the agent starts: a venue run that waited in the
|
|
1004
|
+
// queue past its grant is not started at all, and nothing is posted.
|
|
1005
|
+
if (venue && !venueGrantLive(run)) {
|
|
1006
|
+
await toolEvents.flush();
|
|
1007
|
+
toolEvents.stop();
|
|
1008
|
+
log(`${timestamp()} ⏹ ${label}'s connected-team grant expired before it could start — not running it`);
|
|
1009
|
+
if (options.dryRun) {
|
|
1010
|
+
log(' [dry-run] would report the run as failed');
|
|
1011
|
+
return 'skipped';
|
|
1012
|
+
}
|
|
1013
|
+
// Closed now, while the margin still leaves the token good: left open, the
|
|
1014
|
+
// row would keep its thinking indicator until the stale-run sweep. No
|
|
1015
|
+
// model: none ran.
|
|
1016
|
+
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: VENUE_GRANT_EXPIRED_ERROR }, log, sleep);
|
|
1017
|
+
return closed.ok ? 'failed' : 'unreported';
|
|
1018
|
+
}
|
|
1019
|
+
const venueDir = venue && !mcp.error ? venueWorkdir() : undefined;
|
|
1020
|
+
// What a venue run says about itself lands on its row in this agent's home
|
|
1021
|
+
// workspace, and on this machine's log: never the agent's words about the
|
|
1022
|
+
// connected team's thread (see `VENUE_PROGRESS`).
|
|
1023
|
+
const turnControl = venue ? { ...control, report: () => control.report(VENUE_PROGRESS) } : control;
|
|
1024
|
+
let result;
|
|
1025
|
+
try {
|
|
1026
|
+
result = await runTurn(run, prompt, options, runtime, mcp, turnControl, log, toolEvents, runEnv, venueDir?.dir);
|
|
1027
|
+
}
|
|
1028
|
+
finally {
|
|
1029
|
+
venueDir?.remove();
|
|
1030
|
+
}
|
|
1031
|
+
// A venue session is never continued: the next turn there starts fresh from
|
|
1032
|
+
// the thread the server sends, so nothing of this one is remembered.
|
|
1033
|
+
const sessionId = venue ? undefined : result.ok ? result.sessionId : undefined;
|
|
957
1034
|
// Whatever the turn did, the timeline is whole BEFORE the reply is posted or
|
|
958
1035
|
// the run is closed — a completion that beats its own last tool row would
|
|
959
1036
|
// leave a `calling` row on a finished run. Never rejects, and after `stop`
|
|
@@ -972,6 +1049,18 @@ mcp) {
|
|
|
972
1049
|
log(`${timestamp()} ⏹ ${label} was already closed server-side (${seconds}s) — nothing was posted`);
|
|
973
1050
|
return 'resolved';
|
|
974
1051
|
}
|
|
1052
|
+
if (stopped === 'expired') {
|
|
1053
|
+
// Stopped while the margin still leaves the token good, so it closes its
|
|
1054
|
+
// own run rather than leaving the row to the stale-run sweep. Nothing of
|
|
1055
|
+
// what it was saying is posted.
|
|
1056
|
+
log(`${timestamp()} ⏹ ${label}'s connected-team grant ran out after ${seconds}s — nothing was posted`);
|
|
1057
|
+
if (options.dryRun) {
|
|
1058
|
+
log(' [dry-run] would report the run as failed');
|
|
1059
|
+
return 'skipped';
|
|
1060
|
+
}
|
|
1061
|
+
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: VENUE_GRANT_ENDED_ERROR }, log, sleep, runtime.model());
|
|
1062
|
+
return closed.ok ? 'failed' : 'unreported';
|
|
1063
|
+
}
|
|
975
1064
|
if (stopped === 'cancelled') {
|
|
976
1065
|
log(`${timestamp()} ⏹ ${label} cancelled after ${seconds}s`);
|
|
977
1066
|
if (options.dryRun) {
|
|
@@ -990,7 +1079,9 @@ mcp) {
|
|
|
990
1079
|
}
|
|
991
1080
|
if (!result.ok) {
|
|
992
1081
|
const { reason } = result;
|
|
993
|
-
|
|
1082
|
+
// A venue run's detail is stderr and the agent's unfinished answer — the
|
|
1083
|
+
// connected team's thread, retold. It stays out of the home run and the log.
|
|
1084
|
+
const detail = venue ? '' : (result.detail ?? '');
|
|
994
1085
|
log(`${timestamp()} ✗ ${label} failed after ${seconds}s — ${reason}`);
|
|
995
1086
|
if (detail)
|
|
996
1087
|
log(` stderr: ${detail}`);
|
|
@@ -1010,8 +1101,9 @@ mcp) {
|
|
|
1010
1101
|
if (options.dryRun) {
|
|
1011
1102
|
const destination = run.reply_surface === 'none' ? '(no reply surface)' : `${run.reply_surface} ${run.reply_path}`;
|
|
1012
1103
|
log(`${timestamp()} ◦ ${label} dry-run — would post to ${destination}:`);
|
|
1013
|
-
|
|
1014
|
-
|
|
1104
|
+
if (!venue)
|
|
1105
|
+
for (const line of reply.split('\n'))
|
|
1106
|
+
log(` │ ${line}`);
|
|
1015
1107
|
log(` [dry-run] nothing was sent and the run was left open`);
|
|
1016
1108
|
return 'skipped';
|
|
1017
1109
|
}
|
|
@@ -1027,8 +1119,8 @@ mcp) {
|
|
|
1027
1119
|
log(` ! could not mark the run complete (${describeFailure(completed)})`);
|
|
1028
1120
|
return 'unreported';
|
|
1029
1121
|
}
|
|
1030
|
-
if (
|
|
1031
|
-
runtime.sessions.remember(sessionKeyFor(run),
|
|
1122
|
+
if (sessionId) {
|
|
1123
|
+
runtime.sessions.remember(sessionKeyFor(run), sessionId);
|
|
1032
1124
|
}
|
|
1033
1125
|
return 'succeeded';
|
|
1034
1126
|
}
|
|
@@ -1041,15 +1133,16 @@ mcp) {
|
|
|
1041
1133
|
? 'no reply surface'
|
|
1042
1134
|
: `${run.reply_surface} runs have nowhere to post yet`;
|
|
1043
1135
|
log(`${timestamp()} ✓ ${label} finished in ${seconds}s (${why})`);
|
|
1044
|
-
|
|
1045
|
-
|
|
1136
|
+
if (!venue)
|
|
1137
|
+
for (const line of reply.split('\n'))
|
|
1138
|
+
log(` │ ${line}`);
|
|
1046
1139
|
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep, runtime.model());
|
|
1047
1140
|
if (!completed.ok) {
|
|
1048
1141
|
log(` ! could not mark the run complete (${describeFailure(completed)})`);
|
|
1049
1142
|
return 'unreported';
|
|
1050
1143
|
}
|
|
1051
|
-
if (
|
|
1052
|
-
runtime.sessions.remember(sessionKeyFor(run),
|
|
1144
|
+
if (sessionId) {
|
|
1145
|
+
runtime.sessions.remember(sessionKeyFor(run), sessionId);
|
|
1053
1146
|
}
|
|
1054
1147
|
return 'succeeded';
|
|
1055
1148
|
}
|
|
@@ -1058,7 +1151,9 @@ mcp) {
|
|
|
1058
1151
|
const posted = await postReplyWithReplay(api, () => postReply(api, replyPath, replyBodyKey, reply, run.run_id, run.reply_body_extra), label, log, sleep);
|
|
1059
1152
|
if (!posted.ok) {
|
|
1060
1153
|
const failure = describeFailure(posted);
|
|
1061
|
-
|
|
1154
|
+
// The server's reason can quote the thread (a reply-race refusal previews
|
|
1155
|
+
// the other reply), so a venue run logs the status alone.
|
|
1156
|
+
const detail = posted.status === 0 || venue ? '' : ` ${posted.error ?? ''}`;
|
|
1062
1157
|
log(`${timestamp()} ✗ ${label} reply rejected (${failure})${detail}`);
|
|
1063
1158
|
// Undo the mid-run recording. The session exists and the agent's transcript
|
|
1064
1159
|
// holds this turn, but the conversation never received its answer, so
|
|
@@ -1072,8 +1167,8 @@ mcp) {
|
|
|
1072
1167
|
// Advance continuity only after the answer is durable on its reply surface.
|
|
1073
1168
|
// Remembering at process exit would strand the conversation behind an answer
|
|
1074
1169
|
// it never received whenever this POST is rejected.
|
|
1075
|
-
if (
|
|
1076
|
-
runtime.sessions.remember(sessionKeyFor(run),
|
|
1170
|
+
if (sessionId) {
|
|
1171
|
+
runtime.sessions.remember(sessionKeyFor(run), sessionId);
|
|
1077
1172
|
}
|
|
1078
1173
|
const completed = await completeWithRetry(api, run.completion_path, { ok: true }, log, sleep, runtime.model());
|
|
1079
1174
|
if (!completed.ok) {
|
|
@@ -1124,6 +1219,12 @@ export async function runAgentHarness(options) {
|
|
|
1124
1219
|
// duplicate — the run is in flight right here. Only a run in the ledger that
|
|
1125
1220
|
// this process never handled is a candidate for restart recovery.
|
|
1126
1221
|
const handledThisSession = new Set();
|
|
1222
|
+
/**
|
|
1223
|
+
* Venue runs whose session is still being opened, and whether it opened. A
|
|
1224
|
+
* duplicate delivery in that window is acknowledged the way the original
|
|
1225
|
+
* will be — never as accepted before a session exists.
|
|
1226
|
+
*/
|
|
1227
|
+
const venueOpenings = new Map();
|
|
1127
1228
|
/** Accepted, not yet reported — the executing run and everything queued. */
|
|
1128
1229
|
const accepted = new Map();
|
|
1129
1230
|
const heartbeats = new HeartbeatLoop({ api, log, intervalMs: options.heartbeatIntervalMs });
|
|
@@ -1138,6 +1239,7 @@ export async function runAgentHarness(options) {
|
|
|
1138
1239
|
accountKeys: runtime.account ? () => runtime.account.keys() : undefined,
|
|
1139
1240
|
// A run in flight reports fresh numbers when it ends; don't probe beside it.
|
|
1140
1241
|
isBusy: () => [...accepted.values()].some((entry) => entry.executing),
|
|
1242
|
+
capabilities: () => (runtime.venueSessions ? [VENUE_RUN_TOKEN_CAPABILITY] : []),
|
|
1141
1243
|
log,
|
|
1142
1244
|
});
|
|
1143
1245
|
runtime.observe = (observation) => status.observe(observation);
|
|
@@ -1163,6 +1265,14 @@ export async function runAgentHarness(options) {
|
|
|
1163
1265
|
catch {
|
|
1164
1266
|
version = undefined;
|
|
1165
1267
|
}
|
|
1268
|
+
if (runtime.brain === 'claude') {
|
|
1269
|
+
try {
|
|
1270
|
+
runtime.venueSessions = claudeSupportsVenueSessions(probe(`${binary} --help`, runtime.workdir).stdout);
|
|
1271
|
+
}
|
|
1272
|
+
catch {
|
|
1273
|
+
runtime.venueSessions = false;
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1166
1276
|
status.observe({
|
|
1167
1277
|
runtime_version: version,
|
|
1168
1278
|
model: modelFromExec(options.exec) ??
|
|
@@ -1247,7 +1357,7 @@ export async function runAgentHarness(options) {
|
|
|
1247
1357
|
log(' [dry-run] would report the run as cancelled');
|
|
1248
1358
|
return;
|
|
1249
1359
|
}
|
|
1250
|
-
const closed = await completeWithRetry(api, entry.run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
|
|
1360
|
+
const closed = await completeWithRetry(apiForRun(api, entry.run), entry.run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
|
|
1251
1361
|
if (closed.ok)
|
|
1252
1362
|
runs.markCompleted(entry.run.run_id);
|
|
1253
1363
|
metrics.runsFailed += 1;
|
|
@@ -1271,6 +1381,30 @@ export async function runAgentHarness(options) {
|
|
|
1271
1381
|
runs.markCompleted(entry.run.run_id);
|
|
1272
1382
|
log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was closed server-side before it started`);
|
|
1273
1383
|
};
|
|
1384
|
+
/**
|
|
1385
|
+
* A venue run's grant is about to end. Stopped while the margin still
|
|
1386
|
+
* leaves its token good, so the run is closed with it: an executing run is
|
|
1387
|
+
* aborted and `handleRun` completes it; a queued one is closed out here.
|
|
1388
|
+
*/
|
|
1389
|
+
const expireAccepted = async (entry) => {
|
|
1390
|
+
if (entry.disposition)
|
|
1391
|
+
return;
|
|
1392
|
+
entry.disposition = 'expired';
|
|
1393
|
+
if (entry.executing) {
|
|
1394
|
+
entry.controller.abort();
|
|
1395
|
+
return;
|
|
1396
|
+
}
|
|
1397
|
+
release(entry.run.run_id);
|
|
1398
|
+
log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)}'s connected-team grant expired before it started`);
|
|
1399
|
+
if (options.dryRun) {
|
|
1400
|
+
log(' [dry-run] would report the run as failed');
|
|
1401
|
+
return;
|
|
1402
|
+
}
|
|
1403
|
+
const closed = await completeWithRetry(apiForRun(api, entry.run), entry.run.completion_path, { ok: false, error: VENUE_GRANT_EXPIRED_ERROR }, log, sleep);
|
|
1404
|
+
if (closed.ok)
|
|
1405
|
+
runs.markCompleted(entry.run.run_id);
|
|
1406
|
+
metrics.runsFailed += 1;
|
|
1407
|
+
};
|
|
1274
1408
|
const exhausted = () => options.maxConnections !== undefined && metrics.connections >= options.maxConnections;
|
|
1275
1409
|
while (!options.signal?.aborted && fatal === undefined && !exhausted()) {
|
|
1276
1410
|
metrics.connections += 1;
|
|
@@ -1343,6 +1477,36 @@ export async function runAgentHarness(options) {
|
|
|
1343
1477
|
log(`${timestamp()} ● connected — ${since}`);
|
|
1344
1478
|
},
|
|
1345
1479
|
});
|
|
1480
|
+
/** This server opens venue sessions over the socket (`auth_ok.features`). */
|
|
1481
|
+
let venueSessionsOffered = false;
|
|
1482
|
+
/** Venue-session opens awaiting the server's answer, by run. */
|
|
1483
|
+
const venueWaiters = new Map();
|
|
1484
|
+
/**
|
|
1485
|
+
* Open a run's venue session over this socket. `null` means no answer —
|
|
1486
|
+
* the socket went, or the server took too long — and the open can be
|
|
1487
|
+
* tried again on a redelivery; any answer is final.
|
|
1488
|
+
*/
|
|
1489
|
+
const askVenueSession = (runId) => {
|
|
1490
|
+
if (!venueSessionsOffered)
|
|
1491
|
+
return Promise.resolve({ ok: false });
|
|
1492
|
+
return new Promise((resolve) => {
|
|
1493
|
+
const timer = setTimeout(() => {
|
|
1494
|
+
venueWaiters.delete(runId);
|
|
1495
|
+
resolve(null);
|
|
1496
|
+
}, VENUE_SESSION_TIMEOUT_MS);
|
|
1497
|
+
venueWaiters.set(runId, (answer) => {
|
|
1498
|
+
clearTimeout(timer);
|
|
1499
|
+
resolve(answer);
|
|
1500
|
+
});
|
|
1501
|
+
try {
|
|
1502
|
+
ws.send(JSON.stringify({ type: VENUE_SESSION_OPEN_FRAME, run_id: runId }));
|
|
1503
|
+
}
|
|
1504
|
+
catch {
|
|
1505
|
+
venueWaiters.get(runId)?.(null);
|
|
1506
|
+
venueWaiters.delete(runId);
|
|
1507
|
+
}
|
|
1508
|
+
});
|
|
1509
|
+
};
|
|
1346
1510
|
ws.on('open', () => {
|
|
1347
1511
|
// Kernel-level probe underneath the application-level one: a dead
|
|
1348
1512
|
// peer is detected by the TCP stack even while the socket is idle.
|
|
@@ -1404,6 +1568,7 @@ export async function runAgentHarness(options) {
|
|
|
1404
1568
|
}
|
|
1405
1569
|
runtime.account?.noteRequestedAt(pending.requested_at);
|
|
1406
1570
|
}
|
|
1571
|
+
venueSessionsOffered = Array.isArray(f.features) && f.features.includes(VENUE_SESSION_FEATURE);
|
|
1407
1572
|
if (Array.isArray(f.features) && f.features.includes(RUNTIME_STATUS_FEATURE)) {
|
|
1408
1573
|
primeStatus();
|
|
1409
1574
|
status.attach((frame) => ws.send(JSON.stringify(frame)));
|
|
@@ -1427,6 +1592,13 @@ export async function runAgentHarness(options) {
|
|
|
1427
1592
|
ws.send(JSON.stringify({ type: 'pong' }));
|
|
1428
1593
|
return;
|
|
1429
1594
|
}
|
|
1595
|
+
if (f.type === VENUE_SESSION_FRAME) {
|
|
1596
|
+
const runId = String(f.run_id);
|
|
1597
|
+
const waiter = venueWaiters.get(runId);
|
|
1598
|
+
venueWaiters.delete(runId);
|
|
1599
|
+
waiter?.(f);
|
|
1600
|
+
return;
|
|
1601
|
+
}
|
|
1430
1602
|
if (!isWebhookEvent(frame))
|
|
1431
1603
|
return;
|
|
1432
1604
|
metrics.eventsReceived += 1;
|
|
@@ -1483,129 +1655,204 @@ export async function runAgentHarness(options) {
|
|
|
1483
1655
|
return;
|
|
1484
1656
|
}
|
|
1485
1657
|
const previous = runs.get(run.run_id);
|
|
1486
|
-
|
|
1658
|
+
const opening = venueOpenings.get(run.run_id);
|
|
1659
|
+
if (opening) {
|
|
1487
1660
|
metrics.duplicates += 1;
|
|
1488
|
-
ack(ACK_ACCEPTED);
|
|
1661
|
+
void opening.then((opened) => ack(opened ? ACK_ACCEPTED : ACK_RETRY));
|
|
1489
1662
|
return;
|
|
1490
1663
|
}
|
|
1491
|
-
|
|
1492
|
-
handledThisSession.add(run.run_id);
|
|
1493
|
-
if (previous) {
|
|
1494
|
-
// We ran this one before but never got a terminal state accepted —
|
|
1495
|
-
// most likely the process died between the command and the
|
|
1496
|
-
// completion. Retry only the completion: re-running the agent could
|
|
1497
|
-
// repeat whatever it did with its credential the first time.
|
|
1664
|
+
if (previous?.completed || handledThisSession.has(run.run_id)) {
|
|
1498
1665
|
metrics.duplicates += 1;
|
|
1499
|
-
|
|
1500
|
-
log(`${timestamp()} ↻ run ${run.run_id.slice(0, 12)} was started before but never reported; ` +
|
|
1501
|
-
'reporting it now without re-running the agent');
|
|
1502
|
-
const recovered = await completeWithRetry(api, previous.completion_path || run.completion_path, { ok: false, error: 'the harness restarted before this run was reported' }, log, sleep);
|
|
1503
|
-
if (recovered.ok)
|
|
1504
|
-
runs.markCompleted(run.run_id);
|
|
1505
|
-
});
|
|
1666
|
+
ack(ACK_ACCEPTED);
|
|
1506
1667
|
return;
|
|
1507
1668
|
}
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
log(`${timestamp()} ✗ closing cancelled run ${run.run_id} failed: ${String(err)}`);
|
|
1525
|
-
}));
|
|
1526
|
-
},
|
|
1527
|
-
onResolved: () => resolveAccepted(entry),
|
|
1528
|
-
});
|
|
1529
|
-
// Queue execution but never block the socket reader: the ack and the
|
|
1530
|
-
// heartbeat have to keep flowing while the agent thinks. The lane is
|
|
1531
|
-
// what keeps two turns of one conversation from overlapping.
|
|
1532
|
-
const lane = laneFor(run);
|
|
1533
|
-
scheduler.submit(lane, async () => {
|
|
1534
|
-
// Settled while it waited — cancelled, or closed server-side. Both
|
|
1535
|
-
// were dealt with where they were decided; there is nothing to run
|
|
1536
|
-
// and nothing left to report.
|
|
1537
|
-
if (entry.disposition)
|
|
1538
|
-
return;
|
|
1539
|
-
// Shutdown was requested while this sat in the queue. Don't start a
|
|
1540
|
-
// fresh command the operator is waiting to be rid of — but do close
|
|
1541
|
-
// the run out, so the server isn't left waiting on it either.
|
|
1542
|
-
if (options.signal?.aborted) {
|
|
1543
|
-
release(run.run_id);
|
|
1544
|
-
log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
|
|
1545
|
-
const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
|
|
1546
|
-
if (closed.ok)
|
|
1547
|
-
runs.markCompleted(run.run_id);
|
|
1548
|
-
metrics.runsFailed += 1;
|
|
1669
|
+
handledThisSession.add(run.run_id);
|
|
1670
|
+
/** Take the run on: track it, heartbeat it, queue it. */
|
|
1671
|
+
const accept = (run) => {
|
|
1672
|
+
if (previous) {
|
|
1673
|
+
// We ran this one before but never got a terminal state accepted —
|
|
1674
|
+
// most likely the process died between the command and the
|
|
1675
|
+
// completion. Retry only the completion: re-running the agent could
|
|
1676
|
+
// repeat whatever it did with its credential the first time.
|
|
1677
|
+
metrics.duplicates += 1;
|
|
1678
|
+
scheduler.submit(laneFor(run), async () => {
|
|
1679
|
+
log(`${timestamp()} ↻ run ${run.run_id.slice(0, 12)} was started before but never reported; ` +
|
|
1680
|
+
'reporting it now without re-running the agent');
|
|
1681
|
+
const recovered = await completeWithRetry(apiForRun(api, run), previous.completion_path || run.completion_path, { ok: false, error: 'the harness restarted before this run was reported' }, log, sleep);
|
|
1682
|
+
if (recovered.ok)
|
|
1683
|
+
runs.markCompleted(run.run_id);
|
|
1684
|
+
});
|
|
1549
1685
|
return;
|
|
1550
1686
|
}
|
|
1551
|
-
//
|
|
1552
|
-
//
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1687
|
+
// Recorded BEFORE the command runs — a crash mid-run must not let a
|
|
1688
|
+
// redelivery execute the agent a second time.
|
|
1689
|
+
runs.markStarted(run.run_id, run.completion_path);
|
|
1690
|
+
metrics.runsStarted += 1;
|
|
1691
|
+
// Heartbeat from the moment it is accepted, not from the moment it
|
|
1692
|
+
// starts. The row is `running` server-side either way, and a run that
|
|
1693
|
+
// waits an hour behind a long job is reaped exactly like one that has
|
|
1694
|
+
// been working for an hour.
|
|
1695
|
+
const entry = { run, controller: new AbortController(), executing: false };
|
|
1696
|
+
accepted.set(run.run_id, entry);
|
|
1697
|
+
heartbeats.track(run.run_id, {
|
|
1698
|
+
path: heartbeatPathFor(run),
|
|
1699
|
+
api: apiForRun(api, run),
|
|
1700
|
+
venue: isVenueRun(run),
|
|
1701
|
+
progress: () => entry.progress,
|
|
1702
|
+
// Deliberately not awaited here — see `background`.
|
|
1703
|
+
onCancelRequested: () => {
|
|
1704
|
+
background.push(cancelAccepted(entry).catch((err) => {
|
|
1705
|
+
log(`${timestamp()} ✗ closing cancelled run ${run.run_id} failed: ${String(err)}`);
|
|
1706
|
+
}));
|
|
1560
1707
|
},
|
|
1561
|
-
|
|
1562
|
-
const isolation = { lane, parallel: concurrency > 1, worktrees };
|
|
1563
|
-
// Resolved here rather than inside `handleRun` so the account this
|
|
1564
|
-
// run is on can be handed back however the run ends.
|
|
1565
|
-
let mcp;
|
|
1566
|
-
try {
|
|
1567
|
-
mcp = runtime.prepareMcp();
|
|
1568
|
-
}
|
|
1569
|
-
catch (err) {
|
|
1570
|
-
// Fails the run closed, and — unlike a throw from here — still
|
|
1571
|
-
// reports it, so the server is not left waiting on it.
|
|
1572
|
-
mcp = { mcpWired: false, error: `could not prepare the run (${String(err)})` };
|
|
1573
|
-
}
|
|
1574
|
-
const outcome = await handleRun(run, options, runtime, api, log, sleep, control, isolation, mcp).catch((err) => {
|
|
1575
|
-
log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
|
|
1576
|
-
return 'failed';
|
|
1708
|
+
onResolved: () => resolveAccepted(entry),
|
|
1577
1709
|
});
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
//
|
|
1581
|
-
//
|
|
1582
|
-
//
|
|
1583
|
-
|
|
1584
|
-
//
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1710
|
+
// A venue run's grant ends at its expiry whether or not the server
|
|
1711
|
+
// can be reached to say so: stop it just before then, while the
|
|
1712
|
+
// margin still lets its token close the run (heartbeats stop it
|
|
1713
|
+
// sooner on any refusal). Re-checked at least every
|
|
1714
|
+
// VENUE_GRANT_CHECK_MS rather than timed once: a timer counts on the
|
|
1715
|
+
// monotonic clock, which can stand still while the machine sleeps,
|
|
1716
|
+
// and the grant does not.
|
|
1717
|
+
if (run.venue_deadline_ms !== undefined) {
|
|
1718
|
+
const checkGrant = () => {
|
|
1719
|
+
if (!accepted.has(run.run_id) || entry.disposition)
|
|
1720
|
+
return;
|
|
1721
|
+
const left = venueGrantRemainingMs(run) - VENUE_EXPIRY_MARGIN_MS;
|
|
1722
|
+
if (left > 0) {
|
|
1723
|
+
setTimeout(checkGrant, Math.min(VENUE_GRANT_CHECK_MS, left)).unref?.();
|
|
1724
|
+
return;
|
|
1725
|
+
}
|
|
1726
|
+
log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)}'s connected-team grant is ending — stopping it`);
|
|
1727
|
+
background.push(expireAccepted(entry).catch((err) => {
|
|
1728
|
+
log(`${timestamp()} ✗ closing expired run ${run.run_id} failed: ${String(err)}`);
|
|
1729
|
+
}));
|
|
1730
|
+
};
|
|
1731
|
+
setTimeout(checkGrant, 0).unref?.();
|
|
1597
1732
|
}
|
|
1598
|
-
//
|
|
1599
|
-
//
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1733
|
+
// Queue execution but never block the socket reader: the ack and the
|
|
1734
|
+
// heartbeat have to keep flowing while the agent thinks. The lane is
|
|
1735
|
+
// what keeps two turns of one conversation from overlapping.
|
|
1736
|
+
const lane = laneFor(run);
|
|
1737
|
+
scheduler.submit(lane, async () => {
|
|
1738
|
+
// Settled while it waited — cancelled, or closed server-side. Both
|
|
1739
|
+
// were dealt with where they were decided; there is nothing to run
|
|
1740
|
+
// and nothing left to report.
|
|
1741
|
+
if (entry.disposition)
|
|
1742
|
+
return;
|
|
1743
|
+
// Shutdown was requested while this sat in the queue. Don't start a
|
|
1744
|
+
// fresh command the operator is waiting to be rid of — but do close
|
|
1745
|
+
// the run out, so the server isn't left waiting on it either.
|
|
1746
|
+
if (options.signal?.aborted) {
|
|
1747
|
+
release(run.run_id);
|
|
1748
|
+
log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
|
|
1749
|
+
const closed = await completeWithRetry(apiForRun(api, run), run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
|
|
1750
|
+
if (closed.ok)
|
|
1751
|
+
runs.markCompleted(run.run_id);
|
|
1752
|
+
metrics.runsFailed += 1;
|
|
1753
|
+
return;
|
|
1754
|
+
}
|
|
1755
|
+
// Set before the first `await`, so a cancellation arriving from the
|
|
1756
|
+
// heartbeat either sees a queued run (and closes it out itself) or an
|
|
1757
|
+
// executing one (and aborts it) — never both, and never neither.
|
|
1758
|
+
entry.executing = true;
|
|
1759
|
+
const control = {
|
|
1760
|
+
signal: entry.controller.signal,
|
|
1761
|
+
disposition: () => entry.disposition,
|
|
1762
|
+
report: (activity) => {
|
|
1763
|
+
entry.progress = activity;
|
|
1764
|
+
},
|
|
1765
|
+
};
|
|
1766
|
+
const isolation = { lane, parallel: concurrency > 1, worktrees };
|
|
1767
|
+
// Resolved here rather than inside `handleRun` so the account this
|
|
1768
|
+
// run is on can be handed back however the run ends.
|
|
1769
|
+
let mcp;
|
|
1770
|
+
try {
|
|
1771
|
+
// A venue session gets no MCP at all — not the ProhostAI server, not
|
|
1772
|
+
// the operator's extras — only the account that runs the brain.
|
|
1773
|
+
mcp = isVenueRun(run) ? prepareVenueSession(runtime) : runtime.prepareMcp();
|
|
1774
|
+
}
|
|
1775
|
+
catch (err) {
|
|
1776
|
+
// Fails the run closed, and — unlike a throw from here — still
|
|
1777
|
+
// reports it, so the server is not left waiting on it.
|
|
1778
|
+
mcp = { mcpWired: false, error: `could not prepare the run (${String(err)})` };
|
|
1779
|
+
}
|
|
1780
|
+
const outcome = await handleRun(run, options, runtime, apiForRun(api, run), log, sleep, control, isolation, mcp).catch((err) => {
|
|
1781
|
+
log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
|
|
1782
|
+
return 'failed';
|
|
1783
|
+
});
|
|
1784
|
+
if (mcp.accountLabel !== undefined)
|
|
1785
|
+
runtime.account?.finishSpawn(mcp.accountLabel);
|
|
1786
|
+
// Held until here on purpose: heartbeats keep the row alive for the
|
|
1787
|
+
// whole of `handleRun`, including the ten minutes of completion
|
|
1788
|
+
// retries at the end of it.
|
|
1789
|
+
release(run.run_id);
|
|
1790
|
+
// An account picked mid-run was held back from the status frame
|
|
1791
|
+
// until this run (on the old account) was done; report it now. Asked
|
|
1792
|
+
// about *this* run's account: a newer run may already be on the pick.
|
|
1793
|
+
if (runtime.account?.changedSinceSpawn(mcp.accountLabel))
|
|
1794
|
+
status.accountsChanged();
|
|
1795
|
+
// Both terminal outcomes reached the completion endpoint; only then
|
|
1796
|
+
// is the run settled and safe to skip on redelivery.
|
|
1797
|
+
if (outcome === 'succeeded')
|
|
1798
|
+
metrics.runsSucceeded += 1;
|
|
1799
|
+
// A cancellation is reported to the server as `failed`, and counted
|
|
1800
|
+
// the same way here — the run did not produce the answer it was for.
|
|
1801
|
+
if (outcome === 'failed' || outcome === 'unreported' || outcome === 'cancelled') {
|
|
1802
|
+
metrics.runsFailed += 1;
|
|
1803
|
+
}
|
|
1804
|
+
// Only a run the server acknowledged as terminal is settled; an
|
|
1805
|
+
// unreported one stays open so a redelivery can report it.
|
|
1806
|
+
if (outcome === 'succeeded' ||
|
|
1807
|
+
outcome === 'failed' ||
|
|
1808
|
+
outcome === 'cancelled' ||
|
|
1809
|
+
outcome === 'resolved') {
|
|
1810
|
+
runs.markCompleted(run.run_id);
|
|
1811
|
+
}
|
|
1812
|
+
});
|
|
1813
|
+
};
|
|
1814
|
+
if (!isVenueRun(run)) {
|
|
1815
|
+
ack(ACK_ACCEPTED);
|
|
1816
|
+
accept(run);
|
|
1817
|
+
return;
|
|
1818
|
+
}
|
|
1819
|
+
// A connected team's thread: the event carries none of it. Open the
|
|
1820
|
+
// run's venue session over this socket for the thread and the run's
|
|
1821
|
+
// own token before anything is tracked, so every request the run makes
|
|
1822
|
+
// — heartbeats included — goes out with the token and never with this
|
|
1823
|
+
// machine's key. The delivery is acknowledged only once the server has
|
|
1824
|
+
// answered: with no answer it stays retryable, so a dropped socket or
|
|
1825
|
+
// a slow server does not strand the run.
|
|
1826
|
+
const runId = run.run_id;
|
|
1827
|
+
let settleOpening = () => { };
|
|
1828
|
+
venueOpenings.set(runId, new Promise((resolve) => {
|
|
1829
|
+
settleOpening = resolve;
|
|
1830
|
+
}));
|
|
1831
|
+
const openedAt = { openedAtMono: performance.now(), openedAtWall: Date.now() };
|
|
1832
|
+
void askVenueSession(runId).then((answer) => {
|
|
1833
|
+
const opened = answer === null ? null : venueRunFromSession(run, eventData(frame.payload), answer, openedAt);
|
|
1834
|
+
venueOpenings.delete(runId);
|
|
1835
|
+
settleOpening(opened !== null && opened.ok);
|
|
1836
|
+
if (opened === null || !opened.ok) {
|
|
1837
|
+
// Not taken: the delivery stays retryable, and a redelivery comes
|
|
1838
|
+
// back through the server's socket check — which either hands the
|
|
1839
|
+
// run to this socket after all or ends it there. Acknowledging a
|
|
1840
|
+
// run this harness never took would leave it open with nobody on it.
|
|
1841
|
+
handledThisSession.delete(runId);
|
|
1842
|
+
ack(ACK_RETRY);
|
|
1843
|
+
log(`${timestamp()} … run ${runId.slice(0, 12)} is in a connected team's thread, but its venue ` +
|
|
1844
|
+
`session could not be opened (${opened === null ? 'no answer' : opened.error}) — not running it; ` +
|
|
1845
|
+
'the server will retry or end it');
|
|
1846
|
+
return;
|
|
1605
1847
|
}
|
|
1848
|
+
ack(ACK_ACCEPTED);
|
|
1849
|
+
accept(opened.run);
|
|
1606
1850
|
});
|
|
1607
1851
|
});
|
|
1608
1852
|
ws.on('close', () => {
|
|
1853
|
+
for (const waiter of venueWaiters.values())
|
|
1854
|
+
waiter(null);
|
|
1855
|
+
venueWaiters.clear();
|
|
1609
1856
|
liveness.stop();
|
|
1610
1857
|
status.detach();
|
|
1611
1858
|
if (options.signal)
|