@evident-ai/cli 3.0.1-dev.fffc02d → 3.1.1-dev.019c4e9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -24
- package/dist/index.js +2626 -309
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
+
import { dirname } from "path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -47,8 +47,35 @@ var credentials = new Conf({
|
|
|
47
47
|
projectName: "evident",
|
|
48
48
|
projectSuffix: "",
|
|
49
49
|
configName: "credentials",
|
|
50
|
-
defaults: {}
|
|
50
|
+
defaults: {},
|
|
51
|
+
configFileMode: 384
|
|
51
52
|
});
|
|
53
|
+
var CREDENTIALS_FILE_MODE = 384;
|
|
54
|
+
var CREDENTIALS_DIR_MODE = 448;
|
|
55
|
+
var permissionWarningEmitted = false;
|
|
56
|
+
function hardenCredentialsPermissions() {
|
|
57
|
+
if (process.platform === "win32") {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const file = credentials.path;
|
|
61
|
+
for (const [path, mode] of [
|
|
62
|
+
[file, CREDENTIALS_FILE_MODE],
|
|
63
|
+
[dirname(file), CREDENTIALS_DIR_MODE]
|
|
64
|
+
]) {
|
|
65
|
+
try {
|
|
66
|
+
if (existsSync(path) && (statSync(path).mode & 511) !== mode) {
|
|
67
|
+
chmodSync(path, mode);
|
|
68
|
+
}
|
|
69
|
+
} catch (err) {
|
|
70
|
+
if (!permissionWarningEmitted) {
|
|
71
|
+
permissionWarningEmitted = true;
|
|
72
|
+
console.error(
|
|
73
|
+
`[config] could not restrict permissions on ${path}; the credentials file may be readable by other users on this machine: ${err instanceof Error ? err.message : String(err)}`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
52
79
|
function getApiUrlConfig() {
|
|
53
80
|
return getApiUrl();
|
|
54
81
|
}
|
|
@@ -59,6 +86,7 @@ function credentialsKey() {
|
|
|
59
86
|
return getApiUrl();
|
|
60
87
|
}
|
|
61
88
|
function getCredentials() {
|
|
89
|
+
hardenCredentialsPermissions();
|
|
62
90
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
63
91
|
return byEndpoint[credentialsKey()] ?? {};
|
|
64
92
|
}
|
|
@@ -70,14 +98,17 @@ function setCredentials(creds) {
|
|
|
70
98
|
expiresAt: creds.expiresAt
|
|
71
99
|
};
|
|
72
100
|
credentials.set("byEndpoint", byEndpoint);
|
|
101
|
+
hardenCredentialsPermissions();
|
|
73
102
|
}
|
|
74
103
|
function clearCredentials() {
|
|
75
104
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
76
105
|
delete byEndpoint[credentialsKey()];
|
|
77
106
|
credentials.set("byEndpoint", byEndpoint);
|
|
107
|
+
hardenCredentialsPermissions();
|
|
78
108
|
}
|
|
79
109
|
function clearAllCredentials() {
|
|
80
110
|
credentials.clear();
|
|
111
|
+
hardenCredentialsPermissions();
|
|
81
112
|
}
|
|
82
113
|
function getCliName() {
|
|
83
114
|
const argv1 = process.argv[1] || "";
|
|
@@ -285,14 +316,14 @@ function blank() {
|
|
|
285
316
|
console.log();
|
|
286
317
|
}
|
|
287
318
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
319
|
+
return new Promise((resolve3) => {
|
|
289
320
|
process.stdout.write(chalk.dim(prompt));
|
|
290
321
|
const handler = () => {
|
|
291
322
|
process.stdin.removeListener("data", handler);
|
|
292
323
|
process.stdin.setRawMode?.(false);
|
|
293
324
|
process.stdin.pause();
|
|
294
325
|
console.log();
|
|
295
|
-
|
|
326
|
+
resolve3();
|
|
296
327
|
};
|
|
297
328
|
if (process.stdin.isTTY) {
|
|
298
329
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +333,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
333
|
});
|
|
303
334
|
}
|
|
304
335
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
336
|
+
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
306
337
|
}
|
|
307
338
|
|
|
308
339
|
// src/commands/login.ts
|
|
@@ -373,22 +404,23 @@ async function deviceFlowLogin(options) {
|
|
|
373
404
|
}
|
|
374
405
|
async function tokenLogin() {
|
|
375
406
|
console.log("Token login mode.");
|
|
376
|
-
console.log("
|
|
407
|
+
console.log("Run `evident login` on a machine with a browser to get a token.");
|
|
408
|
+
console.log("Manage or revoke existing tokens under Settings \u2192 CLI tokens.");
|
|
377
409
|
blank();
|
|
378
410
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
411
|
+
const token = await new Promise((resolve3) => {
|
|
380
412
|
let data = "";
|
|
381
413
|
process.stdin.setEncoding("utf8");
|
|
382
414
|
process.stdin.on("data", (chunk) => {
|
|
383
415
|
data += chunk;
|
|
384
416
|
});
|
|
385
417
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
418
|
+
resolve3(data.trim());
|
|
387
419
|
});
|
|
388
420
|
if (process.stdin.isTTY) {
|
|
389
421
|
process.stdin.once("data", (chunk) => {
|
|
390
422
|
process.stdin.pause();
|
|
391
|
-
|
|
423
|
+
resolve3(chunk.toString().trim());
|
|
392
424
|
});
|
|
393
425
|
process.stdin.resume();
|
|
394
426
|
}
|
|
@@ -467,9 +499,9 @@ async function whoami() {
|
|
|
467
499
|
}
|
|
468
500
|
|
|
469
501
|
// src/commands/run.ts
|
|
502
|
+
import { homedir as homedir2 } from "os";
|
|
503
|
+
import { isAbsolute as isAbsolute2, join as join2, parse, resolve as resolvePath } from "path";
|
|
470
504
|
import chalk6 from "chalk";
|
|
471
|
-
import ora3 from "ora";
|
|
472
|
-
import { select as select3 } from "@inquirer/prompts";
|
|
473
505
|
|
|
474
506
|
// ../../packages/types/src/telemetry/index.ts
|
|
475
507
|
var TelemetryEventTypes = {
|
|
@@ -485,6 +517,10 @@ var TelemetryEventTypes = {
|
|
|
485
517
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
486
518
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
487
519
|
|
|
520
|
+
// ../../packages/types/src/runner-files.ts
|
|
521
|
+
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
522
|
+
var MAX_FILE_SYNC_DIRECTORIES = 16;
|
|
523
|
+
|
|
488
524
|
// ../../packages/types/src/logging/index.ts
|
|
489
525
|
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
490
526
|
function log(level, event, fields) {
|
|
@@ -499,6 +535,12 @@ function log(level, event, fields) {
|
|
|
499
535
|
);
|
|
500
536
|
}
|
|
501
537
|
}
|
|
538
|
+
function errorFields(err) {
|
|
539
|
+
if (err instanceof Error) {
|
|
540
|
+
return { error: err.message, error_name: err.name };
|
|
541
|
+
}
|
|
542
|
+
return { error: String(err) };
|
|
543
|
+
}
|
|
502
544
|
function stripQuery(url) {
|
|
503
545
|
try {
|
|
504
546
|
return new URL(url).pathname;
|
|
@@ -508,6 +550,10 @@ function stripQuery(url) {
|
|
|
508
550
|
}
|
|
509
551
|
}
|
|
510
552
|
|
|
553
|
+
// src/commands/run.ts
|
|
554
|
+
import ora3 from "ora";
|
|
555
|
+
import { select as select3 } from "@inquirer/prompts";
|
|
556
|
+
|
|
511
557
|
// src/lib/telemetry.ts
|
|
512
558
|
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
559
|
function getCliVersion() {
|
|
@@ -645,14 +691,27 @@ var EventTypes = {
|
|
|
645
691
|
// CLI lifecycle
|
|
646
692
|
CLI_STARTED: "cli.started",
|
|
647
693
|
CLI_COMMAND: "cli.command",
|
|
648
|
-
CLI_ERROR: "cli.error"
|
|
694
|
+
CLI_ERROR: "cli.error",
|
|
695
|
+
// Deprecation telemetry (#412) — usage of the old `--agent`/`EVIDENT_AGENT_KEY`
|
|
696
|
+
// names instead of the preferred `--runner`/`EVIDENT_RUNNER_KEY` (#409).
|
|
697
|
+
DEPRECATED_AGENT_FLAG_USED: "cli.deprecated_agent_flag_used",
|
|
698
|
+
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
649
699
|
};
|
|
650
700
|
|
|
651
701
|
// src/lib/auth.ts
|
|
652
702
|
async function getAuthCredentials() {
|
|
703
|
+
const runnerKey = process.env.EVIDENT_RUNNER_KEY;
|
|
653
704
|
const agentKey = process.env.EVIDENT_AGENT_KEY;
|
|
705
|
+
if (runnerKey) {
|
|
706
|
+
return {
|
|
707
|
+
token: runnerKey,
|
|
708
|
+
authType: "agent_key",
|
|
709
|
+
keySource: "runner_key",
|
|
710
|
+
notice: agentKey ? "Both EVIDENT_RUNNER_KEY and EVIDENT_AGENT_KEY are set; using EVIDENT_RUNNER_KEY." : void 0
|
|
711
|
+
};
|
|
712
|
+
}
|
|
654
713
|
if (agentKey) {
|
|
655
|
-
return { token: agentKey, authType: "agent_key" };
|
|
714
|
+
return { token: agentKey, authType: "agent_key", keySource: "agent_key" };
|
|
656
715
|
}
|
|
657
716
|
const userToken = process.env.EVIDENT_TOKEN;
|
|
658
717
|
if (userToken) {
|
|
@@ -706,7 +765,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
706
765
|
if (health.healthy) {
|
|
707
766
|
return health;
|
|
708
767
|
}
|
|
709
|
-
await new Promise((
|
|
768
|
+
await new Promise((resolve3) => setTimeout(resolve3, 1e3));
|
|
710
769
|
}
|
|
711
770
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
712
771
|
}
|
|
@@ -721,7 +780,7 @@ function buildOpenCodeVersionWarning(version2) {
|
|
|
721
780
|
if (isQueueValidatedVersion(version2)) return null;
|
|
722
781
|
const detected = version2 ? `v${version2}` : "unknown";
|
|
723
782
|
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
724
|
-
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack
|
|
783
|
+
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
725
784
|
}
|
|
726
785
|
|
|
727
786
|
// src/lib/opencode/process.ts
|
|
@@ -1013,6 +1072,12 @@ async function promptOpenCodeInstall(interactive) {
|
|
|
1013
1072
|
return action;
|
|
1014
1073
|
}
|
|
1015
1074
|
|
|
1075
|
+
// src/lib/opencode/provider-check.ts
|
|
1076
|
+
function buildNoProviderWarning(hasProvider) {
|
|
1077
|
+
if (hasProvider !== false) return null;
|
|
1078
|
+
return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1016
1081
|
// src/lib/opencode/session.ts
|
|
1017
1082
|
function opencodeBase(port) {
|
|
1018
1083
|
return `http://127.0.0.1:${port}`;
|
|
@@ -1078,6 +1143,84 @@ async function getSessionMessages(port, sessionId) {
|
|
|
1078
1143
|
return null;
|
|
1079
1144
|
}
|
|
1080
1145
|
}
|
|
1146
|
+
function isSessionActivelyGenerating(messages) {
|
|
1147
|
+
if (!messages || messages.length === 0) return false;
|
|
1148
|
+
const last = messages[messages.length - 1];
|
|
1149
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1150
|
+
return completedOf(last) == null;
|
|
1151
|
+
}
|
|
1152
|
+
function sessionLastActivityMs(session) {
|
|
1153
|
+
const candidates = [
|
|
1154
|
+
session.time?.updated,
|
|
1155
|
+
session.time?.created,
|
|
1156
|
+
session.time_updated,
|
|
1157
|
+
session.time_created,
|
|
1158
|
+
session.updated,
|
|
1159
|
+
session.created
|
|
1160
|
+
];
|
|
1161
|
+
for (const c of candidates) {
|
|
1162
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1163
|
+
}
|
|
1164
|
+
return null;
|
|
1165
|
+
}
|
|
1166
|
+
async function listSessions(port) {
|
|
1167
|
+
try {
|
|
1168
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1169
|
+
if (!res.ok) return null;
|
|
1170
|
+
const body = await res.json();
|
|
1171
|
+
return Array.isArray(body) ? body : null;
|
|
1172
|
+
} catch {
|
|
1173
|
+
return null;
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
async function deleteSession(port, id) {
|
|
1177
|
+
try {
|
|
1178
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1179
|
+
return res.status >= 200 && res.status < 300;
|
|
1180
|
+
} catch {
|
|
1181
|
+
return false;
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
async function sessionExists(port, id) {
|
|
1185
|
+
try {
|
|
1186
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1187
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1188
|
+
if (res.status === 404) return false;
|
|
1189
|
+
return null;
|
|
1190
|
+
} catch {
|
|
1191
|
+
return null;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
async function getSessionStatuses(port) {
|
|
1195
|
+
try {
|
|
1196
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1197
|
+
if (!res.ok) {
|
|
1198
|
+
console.error(
|
|
1199
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1200
|
+
);
|
|
1201
|
+
return null;
|
|
1202
|
+
}
|
|
1203
|
+
const body = await res.json();
|
|
1204
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1205
|
+
console.error(
|
|
1206
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1207
|
+
);
|
|
1208
|
+
return null;
|
|
1209
|
+
}
|
|
1210
|
+
return body;
|
|
1211
|
+
} catch (err) {
|
|
1212
|
+
console.error(
|
|
1213
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1214
|
+
);
|
|
1215
|
+
return null;
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
async function isSessionOngoing(port, id) {
|
|
1219
|
+
const map = await getSessionStatuses(port);
|
|
1220
|
+
if (map == null) return null;
|
|
1221
|
+
const entry = map[id];
|
|
1222
|
+
return entry != null && entry.type !== "idle";
|
|
1223
|
+
}
|
|
1081
1224
|
async function createOpenCodeSession(port, directory) {
|
|
1082
1225
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1083
1226
|
if (directory && directory.trim()) {
|
|
@@ -1095,17 +1238,128 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1095
1238
|
const data = await response.json();
|
|
1096
1239
|
return data.id;
|
|
1097
1240
|
}
|
|
1241
|
+
async function getModelAttachmentCapability(port, model) {
|
|
1242
|
+
try {
|
|
1243
|
+
const res = await fetch(`${opencodeBase(port)}/config/providers`);
|
|
1244
|
+
if (!res.ok) {
|
|
1245
|
+
console.error(
|
|
1246
|
+
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
1247
|
+
);
|
|
1248
|
+
return null;
|
|
1249
|
+
}
|
|
1250
|
+
const body = await res.json();
|
|
1251
|
+
const providers = Array.isArray(body?.providers) ? body.providers : null;
|
|
1252
|
+
if (!providers) {
|
|
1253
|
+
console.error(
|
|
1254
|
+
`[getModelAttachmentCapability] GET /config/providers body had no providers array (port ${port})`
|
|
1255
|
+
);
|
|
1256
|
+
return null;
|
|
1257
|
+
}
|
|
1258
|
+
const slash = model ? model.indexOf("/") : -1;
|
|
1259
|
+
const providerId = slash > 0 ? model.slice(0, slash) : void 0;
|
|
1260
|
+
let modelId = slash > 0 ? model.slice(slash + 1) : void 0;
|
|
1261
|
+
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
1262
|
+
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
1263
|
+
if (!provider && !providerId) {
|
|
1264
|
+
const defaultProviderIds = defaults2 ? Object.keys(defaults2) : [];
|
|
1265
|
+
if (defaultProviderIds.length === 1) {
|
|
1266
|
+
provider = providers.find((p) => p?.id === defaultProviderIds[0]);
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
if (!provider || !provider.models) return null;
|
|
1270
|
+
if (!modelId && defaults2 && typeof provider.id === "string") {
|
|
1271
|
+
const def = defaults2[provider.id];
|
|
1272
|
+
if (typeof def === "string") modelId = def;
|
|
1273
|
+
}
|
|
1274
|
+
if (!modelId) {
|
|
1275
|
+
if (providerId) {
|
|
1276
|
+
const keys = Object.keys(provider.models);
|
|
1277
|
+
if (keys.length === 1) modelId = keys[0];
|
|
1278
|
+
}
|
|
1279
|
+
if (!modelId) return null;
|
|
1280
|
+
}
|
|
1281
|
+
const entry = provider.models[modelId];
|
|
1282
|
+
if (!entry || typeof entry !== "object") return null;
|
|
1283
|
+
if (entry.capabilities && typeof entry.capabilities === "object") {
|
|
1284
|
+
if (typeof entry.capabilities.attachment === "boolean") {
|
|
1285
|
+
return entry.capabilities.attachment;
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
return typeof entry.attachment === "boolean" ? entry.attachment : null;
|
|
1289
|
+
} catch (err) {
|
|
1290
|
+
console.error(
|
|
1291
|
+
`[getModelAttachmentCapability] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1292
|
+
);
|
|
1293
|
+
return null;
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
async function buildFileParts(attachments, capable) {
|
|
1297
|
+
const outcomes = [];
|
|
1298
|
+
const parts = [];
|
|
1299
|
+
const capabilityUnknown = capable === null;
|
|
1300
|
+
if (capable !== true) {
|
|
1301
|
+
for (const a of attachments.inputs) {
|
|
1302
|
+
outcomes.push({ index: a.index, mime: a.mime, filename: a.filename, status: "skipped" });
|
|
1303
|
+
}
|
|
1304
|
+
return { parts, outcomes, capabilityUnknown };
|
|
1305
|
+
}
|
|
1306
|
+
for (const a of attachments.inputs) {
|
|
1307
|
+
let dataUrl = null;
|
|
1308
|
+
try {
|
|
1309
|
+
dataUrl = await attachments.fetchDataUrl(a.index);
|
|
1310
|
+
} catch (err) {
|
|
1311
|
+
console.error(
|
|
1312
|
+
`[buildFileParts] attachment ${a.index} (${a.mime}) fetch threw \u2014 omitting: ${err instanceof Error ? err.message : String(err)}`
|
|
1313
|
+
);
|
|
1314
|
+
dataUrl = null;
|
|
1315
|
+
}
|
|
1316
|
+
if (dataUrl !== null && typeof dataUrl === "object") {
|
|
1317
|
+
outcomes.push({
|
|
1318
|
+
index: a.index,
|
|
1319
|
+
mime: a.mime,
|
|
1320
|
+
filename: a.filename,
|
|
1321
|
+
status: "failed",
|
|
1322
|
+
reason: "needs_reauth"
|
|
1323
|
+
});
|
|
1324
|
+
continue;
|
|
1325
|
+
}
|
|
1326
|
+
if (dataUrl == null) {
|
|
1327
|
+
outcomes.push({ index: a.index, mime: a.mime, filename: a.filename, status: "failed" });
|
|
1328
|
+
continue;
|
|
1329
|
+
}
|
|
1330
|
+
parts.push({
|
|
1331
|
+
type: "file",
|
|
1332
|
+
mime: a.mime,
|
|
1333
|
+
url: dataUrl,
|
|
1334
|
+
...a.filename ? { filename: a.filename } : {}
|
|
1335
|
+
});
|
|
1336
|
+
outcomes.push({ index: a.index, mime: a.mime, filename: a.filename, status: "sent" });
|
|
1337
|
+
}
|
|
1338
|
+
return { parts, outcomes, capabilityUnknown };
|
|
1339
|
+
}
|
|
1098
1340
|
function messageText(m) {
|
|
1099
1341
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
1100
1342
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1101
1343
|
}
|
|
1102
|
-
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1344
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
1103
1345
|
const before = await getSessionMessages(port, sessionId);
|
|
1104
1346
|
const knownUserIds = new Set(
|
|
1105
1347
|
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1106
1348
|
);
|
|
1349
|
+
const parts = [{ type: "text", text: content }];
|
|
1350
|
+
let pendingOutcomes = null;
|
|
1351
|
+
if (attachments && attachments.inputs.length > 0) {
|
|
1352
|
+
const capable = await getModelAttachmentCapability(port, options?.model);
|
|
1353
|
+
const {
|
|
1354
|
+
parts: fileParts,
|
|
1355
|
+
outcomes,
|
|
1356
|
+
capabilityUnknown
|
|
1357
|
+
} = await buildFileParts(attachments, capable);
|
|
1358
|
+
parts.push(...fileParts);
|
|
1359
|
+
if (attachments.onOutcomes) pendingOutcomes = { outcomes, capabilityUnknown };
|
|
1360
|
+
}
|
|
1107
1361
|
const body = {
|
|
1108
|
-
parts
|
|
1362
|
+
parts
|
|
1109
1363
|
};
|
|
1110
1364
|
if (options?.agent) {
|
|
1111
1365
|
body.agent = options.agent;
|
|
@@ -1144,10 +1398,13 @@ async function sendPromptAsync(port, sessionId, content, options) {
|
|
|
1144
1398
|
best = { id, created };
|
|
1145
1399
|
}
|
|
1146
1400
|
}
|
|
1147
|
-
if (best)
|
|
1401
|
+
if (best) {
|
|
1402
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
1403
|
+
return best.id;
|
|
1404
|
+
}
|
|
1148
1405
|
}
|
|
1149
1406
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1150
|
-
await new Promise((
|
|
1407
|
+
await new Promise((resolve3) => setTimeout(resolve3, READ_BACK_DELAY_MS));
|
|
1151
1408
|
}
|
|
1152
1409
|
}
|
|
1153
1410
|
return null;
|
|
@@ -1193,6 +1450,72 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
1193
1450
|
}
|
|
1194
1451
|
return lastOk ?? last;
|
|
1195
1452
|
}
|
|
1453
|
+
function messageUsage(messages, userMessageId) {
|
|
1454
|
+
if (!messages || messages.length === 0) return null;
|
|
1455
|
+
const byParentAll = messages.filter(
|
|
1456
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1457
|
+
);
|
|
1458
|
+
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
1459
|
+
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
1460
|
+
let correlated;
|
|
1461
|
+
if (byParent.length > 0) {
|
|
1462
|
+
correlated = byParent;
|
|
1463
|
+
} else {
|
|
1464
|
+
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
1465
|
+
correlated = reply ? [reply] : [];
|
|
1466
|
+
}
|
|
1467
|
+
if (correlated.length === 0) return null;
|
|
1468
|
+
let sawAnyUsage = false;
|
|
1469
|
+
let inputSum = 0;
|
|
1470
|
+
let outputSum = 0;
|
|
1471
|
+
let reasoningSum = 0;
|
|
1472
|
+
let cacheReadSum = 0;
|
|
1473
|
+
let cacheWriteSum = 0;
|
|
1474
|
+
let costSum = 0;
|
|
1475
|
+
let sawCost = false;
|
|
1476
|
+
let modelId = null;
|
|
1477
|
+
let providerId = null;
|
|
1478
|
+
for (const m of correlated) {
|
|
1479
|
+
const info = m.info;
|
|
1480
|
+
if (!info) continue;
|
|
1481
|
+
const tokens = info.tokens;
|
|
1482
|
+
if (tokens) {
|
|
1483
|
+
sawAnyUsage = true;
|
|
1484
|
+
inputSum += tokens.input ?? 0;
|
|
1485
|
+
outputSum += tokens.output ?? 0;
|
|
1486
|
+
reasoningSum += tokens.reasoning ?? 0;
|
|
1487
|
+
cacheReadSum += tokens.cache?.read ?? 0;
|
|
1488
|
+
cacheWriteSum += tokens.cache?.write ?? 0;
|
|
1489
|
+
}
|
|
1490
|
+
if (typeof info.cost === "number") {
|
|
1491
|
+
sawAnyUsage = true;
|
|
1492
|
+
sawCost = true;
|
|
1493
|
+
costSum += info.cost;
|
|
1494
|
+
}
|
|
1495
|
+
if (typeof info.modelID === "string") {
|
|
1496
|
+
sawAnyUsage = true;
|
|
1497
|
+
modelId = info.modelID;
|
|
1498
|
+
}
|
|
1499
|
+
if (typeof info.providerID === "string") {
|
|
1500
|
+
sawAnyUsage = true;
|
|
1501
|
+
providerId = info.providerID;
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
if (!sawAnyUsage) return null;
|
|
1505
|
+
return {
|
|
1506
|
+
usage_provider_id: providerId,
|
|
1507
|
+
usage_model_id: modelId,
|
|
1508
|
+
usage_tokens_input: inputSum,
|
|
1509
|
+
usage_tokens_output: outputSum,
|
|
1510
|
+
usage_tokens_reasoning: reasoningSum,
|
|
1511
|
+
usage_tokens_cache_read: cacheReadSum,
|
|
1512
|
+
usage_tokens_cache_write: cacheWriteSum,
|
|
1513
|
+
// NULL means "OpenCode never reported a cost" (never inferred from
|
|
1514
|
+
// tokens) — distinct from a genuine 0-cost turn, which would set
|
|
1515
|
+
// `sawCost` true with `costSum === 0`.
|
|
1516
|
+
usage_cost_usd: sawCost ? costSum : null
|
|
1517
|
+
};
|
|
1518
|
+
}
|
|
1196
1519
|
function messageRunState(messages, userMessageId) {
|
|
1197
1520
|
if (!messages || messages.length === 0) return "unknown";
|
|
1198
1521
|
const hasUser = messages.some((m) => idOf(m) === userMessageId);
|
|
@@ -1204,6 +1527,14 @@ function messageRunState(messages, userMessageId) {
|
|
|
1204
1527
|
if (isAssistantInFlight(reply)) return "running";
|
|
1205
1528
|
return errorOf(reply) != null ? "failed" : "done";
|
|
1206
1529
|
}
|
|
1530
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1531
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1532
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1533
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1534
|
+
}
|
|
1535
|
+
function isB2AbandonmentConfirmed(params) {
|
|
1536
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
1537
|
+
}
|
|
1207
1538
|
function messageError(messages, userMessageId) {
|
|
1208
1539
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1209
1540
|
const error2 = errorOf(reply);
|
|
@@ -1223,6 +1554,141 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
1223
1554
|
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1224
1555
|
);
|
|
1225
1556
|
}
|
|
1557
|
+
async function hasAnyConfiguredProvider(port) {
|
|
1558
|
+
try {
|
|
1559
|
+
const res = await fetch(`${opencodeBase(port)}/config/providers`);
|
|
1560
|
+
if (!res.ok) {
|
|
1561
|
+
console.error(
|
|
1562
|
+
`[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
1563
|
+
);
|
|
1564
|
+
return null;
|
|
1565
|
+
}
|
|
1566
|
+
const body = await res.json();
|
|
1567
|
+
if (!body || typeof body !== "object" || Array.isArray(body)) {
|
|
1568
|
+
console.error(
|
|
1569
|
+
`[hasAnyConfiguredProvider] GET /config/providers body was not a plain object (port ${port})`
|
|
1570
|
+
);
|
|
1571
|
+
return null;
|
|
1572
|
+
}
|
|
1573
|
+
const defaults2 = body.default;
|
|
1574
|
+
if (!defaults2 || typeof defaults2 !== "object" || Array.isArray(defaults2)) {
|
|
1575
|
+
console.error(
|
|
1576
|
+
`[hasAnyConfiguredProvider] GET /config/providers body had no \`default\` object (port ${port})`
|
|
1577
|
+
);
|
|
1578
|
+
return null;
|
|
1579
|
+
}
|
|
1580
|
+
return Object.keys(defaults2).length > 0;
|
|
1581
|
+
} catch (err) {
|
|
1582
|
+
console.error(
|
|
1583
|
+
`[hasAnyConfiguredProvider] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1584
|
+
);
|
|
1585
|
+
return null;
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1590
|
+
var DURATION_UNIT_MS = {
|
|
1591
|
+
s: 1e3,
|
|
1592
|
+
m: 60 * 1e3,
|
|
1593
|
+
h: 60 * 60 * 1e3,
|
|
1594
|
+
d: 24 * 60 * 60 * 1e3
|
|
1595
|
+
};
|
|
1596
|
+
function parseDurationMs(input) {
|
|
1597
|
+
const trimmed = input.trim();
|
|
1598
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1599
|
+
if (!match) {
|
|
1600
|
+
throw new Error(
|
|
1601
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1602
|
+
);
|
|
1603
|
+
}
|
|
1604
|
+
const value = Number(match[1]);
|
|
1605
|
+
if (value <= 0) {
|
|
1606
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1607
|
+
}
|
|
1608
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1609
|
+
}
|
|
1610
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1611
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1612
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1613
|
+
const ageEligible = (s) => {
|
|
1614
|
+
if (maxAgeMs === void 0) return false;
|
|
1615
|
+
if (s.lastActivityMs === null) return true;
|
|
1616
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1617
|
+
};
|
|
1618
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1619
|
+
if (maxCount !== void 0) {
|
|
1620
|
+
const byActivityDesc = [...sessions].sort(
|
|
1621
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1622
|
+
);
|
|
1623
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1624
|
+
countEligibleIds.add(s.id);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
const toDelete = [];
|
|
1628
|
+
for (const s of sessions) {
|
|
1629
|
+
if (protectedIds.has(s.id)) continue;
|
|
1630
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1631
|
+
toDelete.push(s.id);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
return toDelete;
|
|
1635
|
+
}
|
|
1636
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1637
|
+
function resolve(flag, envValue, fallback) {
|
|
1638
|
+
return flag ?? envValue ?? fallback;
|
|
1639
|
+
}
|
|
1640
|
+
function parseMaxCount(input) {
|
|
1641
|
+
const trimmed = input.trim();
|
|
1642
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1643
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1644
|
+
}
|
|
1645
|
+
const value = Number(trimmed);
|
|
1646
|
+
if (value <= 0) {
|
|
1647
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1648
|
+
}
|
|
1649
|
+
return value;
|
|
1650
|
+
}
|
|
1651
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1652
|
+
const warnings = [];
|
|
1653
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1654
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1655
|
+
const intervalRaw = resolve(
|
|
1656
|
+
flags.interval,
|
|
1657
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1658
|
+
DEFAULT_INTERVAL
|
|
1659
|
+
);
|
|
1660
|
+
let maxAgeMs;
|
|
1661
|
+
if (maxAgeRaw !== void 0) {
|
|
1662
|
+
try {
|
|
1663
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1664
|
+
} catch (err) {
|
|
1665
|
+
warnings.push(
|
|
1666
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1667
|
+
);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
let maxCount;
|
|
1671
|
+
if (maxCountRaw !== void 0) {
|
|
1672
|
+
try {
|
|
1673
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1674
|
+
} catch (err) {
|
|
1675
|
+
warnings.push(
|
|
1676
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1677
|
+
);
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
let intervalMs;
|
|
1681
|
+
try {
|
|
1682
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1683
|
+
} catch (err) {
|
|
1684
|
+
warnings.push(
|
|
1685
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1686
|
+
);
|
|
1687
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1688
|
+
}
|
|
1689
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1690
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1691
|
+
}
|
|
1226
1692
|
|
|
1227
1693
|
// src/lib/tunnel/connection.ts
|
|
1228
1694
|
import WebSocket2 from "ws";
|
|
@@ -1277,10 +1743,11 @@ var StreamForwarder = class {
|
|
|
1277
1743
|
* Abort every in-flight stream (e.g. on WebSocket close).
|
|
1278
1744
|
*/
|
|
1279
1745
|
abortAll() {
|
|
1280
|
-
for (const stream of this.inflight.
|
|
1746
|
+
for (const [sid, stream] of this.inflight.entries()) {
|
|
1281
1747
|
try {
|
|
1282
1748
|
stream.abort();
|
|
1283
|
-
} catch {
|
|
1749
|
+
} catch (err) {
|
|
1750
|
+
log("error", "forwarder_abort_failed", { sid, ...errorFields(err) });
|
|
1284
1751
|
}
|
|
1285
1752
|
}
|
|
1286
1753
|
this.inflight.clear();
|
|
@@ -1314,12 +1781,12 @@ var StreamForwarder = class {
|
|
|
1314
1781
|
let endBody;
|
|
1315
1782
|
if (has_body) {
|
|
1316
1783
|
const chunks = [];
|
|
1317
|
-
bodyPromise = new Promise((
|
|
1784
|
+
bodyPromise = new Promise((resolve3) => {
|
|
1318
1785
|
pushBody = (buf) => {
|
|
1319
1786
|
chunks.push(buf);
|
|
1320
1787
|
};
|
|
1321
1788
|
endBody = () => {
|
|
1322
|
-
|
|
1789
|
+
resolve3(Buffer.concat(chunks));
|
|
1323
1790
|
};
|
|
1324
1791
|
});
|
|
1325
1792
|
}
|
|
@@ -1430,31 +1897,20 @@ function connectTunnel(options) {
|
|
|
1430
1897
|
onConnected,
|
|
1431
1898
|
onDisconnected,
|
|
1432
1899
|
onError,
|
|
1433
|
-
onRequest,
|
|
1434
1900
|
onResponse,
|
|
1435
1901
|
onInfo,
|
|
1436
1902
|
onDrainPing
|
|
1437
1903
|
} = options;
|
|
1438
1904
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1439
1905
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1440
|
-
return new Promise((
|
|
1906
|
+
return new Promise((resolve3, reject) => {
|
|
1441
1907
|
const ws = new WebSocket2(url, {
|
|
1442
1908
|
headers: {
|
|
1443
1909
|
Authorization: authHeader
|
|
1444
1910
|
}
|
|
1445
1911
|
});
|
|
1446
|
-
const streamStartTimes = /* @__PURE__ */ new Map();
|
|
1447
1912
|
const forwarder = new StreamForwarder(ws, port, {
|
|
1448
|
-
|
|
1449
|
-
if (path === TUNNEL_DRAIN_PING_PATH) return;
|
|
1450
|
-
streamStartTimes.set(sid, Date.now());
|
|
1451
|
-
onRequest?.(method, path, sid);
|
|
1452
|
-
},
|
|
1453
|
-
onHead: (sid, status) => {
|
|
1454
|
-
const startedAt = streamStartTimes.get(sid);
|
|
1455
|
-
streamStartTimes.delete(sid);
|
|
1456
|
-
onResponse?.(status, startedAt ? Date.now() - startedAt : 0, sid);
|
|
1457
|
-
},
|
|
1913
|
+
onHead: () => onResponse?.(),
|
|
1458
1914
|
onDrainPing: () => onDrainPing?.()
|
|
1459
1915
|
});
|
|
1460
1916
|
const connectionTimeout = setTimeout(() => {
|
|
@@ -1502,7 +1958,7 @@ function connectTunnel(options) {
|
|
|
1502
1958
|
clearTimeout(connectionTimeout);
|
|
1503
1959
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1504
1960
|
onConnected?.(connectedAgentId);
|
|
1505
|
-
|
|
1961
|
+
resolve3({
|
|
1506
1962
|
ws,
|
|
1507
1963
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1508
1964
|
});
|
|
@@ -1530,7 +1986,6 @@ function connectTunnel(options) {
|
|
|
1530
1986
|
ws.on("close", (code, reason) => {
|
|
1531
1987
|
const reasonStr = reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
1532
1988
|
forwarder.abortAll();
|
|
1533
|
-
streamStartTimes.clear();
|
|
1534
1989
|
onDisconnected?.(code, reasonStr);
|
|
1535
1990
|
});
|
|
1536
1991
|
});
|
|
@@ -1565,7 +2020,11 @@ var RunnerConnection = class {
|
|
|
1565
2020
|
if (this.connection) {
|
|
1566
2021
|
try {
|
|
1567
2022
|
this.connection.close();
|
|
1568
|
-
} catch {
|
|
2023
|
+
} catch (err) {
|
|
2024
|
+
log("error", "runner_connection_close_failed", {
|
|
2025
|
+
agent_id: this.resolvedAgentId,
|
|
2026
|
+
...errorFields(err)
|
|
2027
|
+
});
|
|
1569
2028
|
}
|
|
1570
2029
|
this.connection = null;
|
|
1571
2030
|
}
|
|
@@ -1617,66 +2076,523 @@ var RunnerConnection = class {
|
|
|
1617
2076
|
}
|
|
1618
2077
|
};
|
|
1619
2078
|
|
|
2079
|
+
// src/lib/tunnel/ready-marker.ts
|
|
2080
|
+
import { writeFileSync } from "fs";
|
|
2081
|
+
function writeTunnelReadyMarker(path, agentId) {
|
|
2082
|
+
try {
|
|
2083
|
+
writeFileSync(path, `${agentId}
|
|
2084
|
+
`);
|
|
2085
|
+
return { ok: true };
|
|
2086
|
+
} catch (error2) {
|
|
2087
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
2090
|
+
|
|
1620
2091
|
// src/lib/channels/driver.ts
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
2092
|
+
import { homedir } from "os";
|
|
2093
|
+
|
|
2094
|
+
// src/lib/file-push.ts
|
|
2095
|
+
import { randomUUID } from "crypto";
|
|
2096
|
+
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2097
|
+
import { basename, dirname as dirname2, isAbsolute, join, relative, resolve as resolve2, sep } from "path";
|
|
2098
|
+
var FILE_MODE = 384;
|
|
2099
|
+
var DIRECTORY_MODE = 448;
|
|
2100
|
+
async function writePushedFile(request) {
|
|
2101
|
+
const { requestedPath, content, allowedDirectories, homeDir } = request;
|
|
2102
|
+
const bytes = content.byteLength;
|
|
2103
|
+
if (allowedDirectories.length === 0) {
|
|
2104
|
+
return refuse("file_sync_disabled", "File sync is not enabled on this runner.", {
|
|
2105
|
+
path: requestedPath,
|
|
2106
|
+
bytes
|
|
2107
|
+
});
|
|
2108
|
+
}
|
|
2109
|
+
if (bytes > MAX_FILE_PUSH_BYTES) {
|
|
2110
|
+
return refuse(
|
|
2111
|
+
"file_too_large",
|
|
2112
|
+
`File is ${bytes} bytes; the limit is ${MAX_FILE_PUSH_BYTES}.`,
|
|
2113
|
+
{
|
|
2114
|
+
path: requestedPath,
|
|
2115
|
+
bytes
|
|
2116
|
+
}
|
|
2117
|
+
);
|
|
2118
|
+
}
|
|
2119
|
+
const candidate = expandAndValidate(requestedPath, homeDir);
|
|
2120
|
+
if (candidate === null) {
|
|
2121
|
+
return refuse("invalid_path", "The requested path is not a valid absolute file path.", {
|
|
2122
|
+
path: requestedPath,
|
|
2123
|
+
bytes
|
|
2124
|
+
});
|
|
2125
|
+
}
|
|
2126
|
+
try {
|
|
2127
|
+
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
2128
|
+
dirname2(candidate)
|
|
2129
|
+
);
|
|
2130
|
+
const realTarget = join(existingAncestor, ...missingSegments, basename(candidate));
|
|
2131
|
+
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
2132
|
+
if (allowedDirectory === null) {
|
|
2133
|
+
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2134
|
+
path: realTarget,
|
|
2135
|
+
bytes
|
|
2136
|
+
});
|
|
2137
|
+
}
|
|
2138
|
+
if (missingSegments.length > 0) {
|
|
2139
|
+
await createMissingDirectories(existingAncestor, missingSegments);
|
|
2140
|
+
const realParent = await realpath(dirname2(realTarget));
|
|
2141
|
+
if (realParent !== dirname2(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
2142
|
+
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2143
|
+
path: realTarget,
|
|
2144
|
+
bytes,
|
|
2145
|
+
reason: "parent_changed_after_create"
|
|
2146
|
+
});
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
await writeAtomically(realTarget, content);
|
|
2150
|
+
log("info", "file_push_written", { path: realTarget, bytes });
|
|
2151
|
+
return { ok: true, path: realTarget };
|
|
2152
|
+
} catch (err) {
|
|
2153
|
+
const errno = err.code ?? "UNKNOWN";
|
|
2154
|
+
return refuse("write_failed", `The runner could not write the file (${errno}).`, {
|
|
2155
|
+
path: candidate,
|
|
2156
|
+
bytes,
|
|
2157
|
+
errno,
|
|
2158
|
+
...errorFields(err)
|
|
2159
|
+
});
|
|
2160
|
+
}
|
|
1626
2161
|
}
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
maxDelayMs: 3e4
|
|
1631
|
-
};
|
|
1632
|
-
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1633
|
-
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1634
|
-
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1635
|
-
var ChannelAuthError = class extends Error {
|
|
1636
|
-
constructor(message) {
|
|
1637
|
-
super(message);
|
|
1638
|
-
this.name = "ChannelAuthError";
|
|
2162
|
+
function expandAndValidate(requestedPath, homeDir) {
|
|
2163
|
+
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
2164
|
+
return null;
|
|
1639
2165
|
}
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
constructor(message, status) {
|
|
1644
|
-
super(message);
|
|
1645
|
-
this.name = "ChannelTerminalError";
|
|
1646
|
-
this.status = status;
|
|
2166
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
2167
|
+
if (expanded.split(/[/\\]/).includes("..")) {
|
|
2168
|
+
return null;
|
|
1647
2169
|
}
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
const
|
|
1652
|
-
|
|
2170
|
+
if (!isAbsolute(expanded)) {
|
|
2171
|
+
return null;
|
|
2172
|
+
}
|
|
2173
|
+
const candidate = resolve2(expanded);
|
|
2174
|
+
const name = basename(candidate);
|
|
2175
|
+
return name === "" || name === "." || name === ".." ? null : candidate;
|
|
1653
2176
|
}
|
|
1654
|
-
function
|
|
1655
|
-
|
|
2177
|
+
async function resolveNearestExistingAncestor(directory) {
|
|
2178
|
+
const missingSegments = [];
|
|
2179
|
+
let current = directory;
|
|
2180
|
+
for (; ; ) {
|
|
2181
|
+
try {
|
|
2182
|
+
return { existingAncestor: await realpath(current), missingSegments };
|
|
2183
|
+
} catch (err) {
|
|
2184
|
+
const parent = dirname2(current);
|
|
2185
|
+
if (err.code !== "ENOENT" || parent === current) {
|
|
2186
|
+
throw err;
|
|
2187
|
+
}
|
|
2188
|
+
missingSegments.unshift(basename(current));
|
|
2189
|
+
current = parent;
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
1656
2192
|
}
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
2193
|
+
async function findContainingAllowedDirectory(allowedDirectories, realTarget) {
|
|
2194
|
+
for (const directory of allowedDirectories) {
|
|
2195
|
+
if (!isAbsolute(directory)) {
|
|
2196
|
+
log("warn", "file_push_allowed_directory_skipped", { directory, reason: "not_absolute" });
|
|
2197
|
+
continue;
|
|
2198
|
+
}
|
|
2199
|
+
const realDirectory = await realpathCreatingIfMissing(directory);
|
|
2200
|
+
if (realDirectory !== null && contains(realDirectory, realTarget)) {
|
|
2201
|
+
return realDirectory;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
return null;
|
|
2205
|
+
}
|
|
2206
|
+
async function realpathCreatingIfMissing(directory) {
|
|
2207
|
+
try {
|
|
2208
|
+
return await realpath(directory);
|
|
2209
|
+
} catch (err) {
|
|
2210
|
+
if (err.code !== "ENOENT") {
|
|
2211
|
+
log("warn", "file_push_allowed_directory_skipped", {
|
|
2212
|
+
directory,
|
|
2213
|
+
reason: "unresolvable",
|
|
2214
|
+
...errorFields(err)
|
|
2215
|
+
});
|
|
2216
|
+
return null;
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
try {
|
|
2220
|
+
await mkdir(directory, { recursive: true, mode: DIRECTORY_MODE });
|
|
2221
|
+
await chmod(directory, DIRECTORY_MODE);
|
|
2222
|
+
return await realpath(directory);
|
|
2223
|
+
} catch (err) {
|
|
2224
|
+
log("warn", "file_push_allowed_directory_skipped", {
|
|
2225
|
+
directory,
|
|
2226
|
+
reason: "create_failed",
|
|
2227
|
+
...errorFields(err)
|
|
2228
|
+
});
|
|
2229
|
+
return null;
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
function contains(realDirectory, realTarget) {
|
|
2233
|
+
const rel = relative(realDirectory, realTarget);
|
|
2234
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
|
|
2235
|
+
}
|
|
2236
|
+
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
2237
|
+
let current = existingAncestor;
|
|
2238
|
+
for (const segment of missingSegments) {
|
|
2239
|
+
current = join(current, segment);
|
|
2240
|
+
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
2241
|
+
await chmod(current, DIRECTORY_MODE);
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
async function writeAtomically(realTarget, content) {
|
|
2245
|
+
const temporaryPath = join(dirname2(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
2246
|
+
let handle;
|
|
2247
|
+
try {
|
|
2248
|
+
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
2249
|
+
await handle.writeFile(content);
|
|
2250
|
+
await handle.chmod(FILE_MODE);
|
|
2251
|
+
await handle.close();
|
|
2252
|
+
handle = void 0;
|
|
2253
|
+
await rename(temporaryPath, realTarget);
|
|
2254
|
+
} catch (err) {
|
|
2255
|
+
await discardTemporaryFile(temporaryPath, handle);
|
|
2256
|
+
throw err;
|
|
2257
|
+
}
|
|
2258
|
+
}
|
|
2259
|
+
async function discardTemporaryFile(temporaryPath, handle) {
|
|
2260
|
+
try {
|
|
2261
|
+
await handle?.close();
|
|
2262
|
+
} catch (err) {
|
|
2263
|
+
log("warn", "file_push_temp_close_failed", { path: temporaryPath, ...errorFields(err) });
|
|
2264
|
+
}
|
|
2265
|
+
try {
|
|
2266
|
+
await unlink(temporaryPath);
|
|
2267
|
+
} catch (err) {
|
|
2268
|
+
const errno = err.code;
|
|
2269
|
+
if (errno !== "ENOENT" && errno !== "ENOTDIR") {
|
|
2270
|
+
log("warn", "file_push_temp_cleanup_failed", { path: temporaryPath, ...errorFields(err) });
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
function refuse(code, message, fields) {
|
|
2275
|
+
log(code === "write_failed" ? "error" : "warn", "file_push_refused", { code, ...fields });
|
|
2276
|
+
return { ok: false, code, message };
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
// src/lib/runner-file-sync.ts
|
|
2280
|
+
var MAX_ACK_ATTEMPTS = 5;
|
|
2281
|
+
async function syncPendingRunnerFiles(options) {
|
|
2282
|
+
const pending = await listPendingFiles(options);
|
|
2283
|
+
const pendingIds = new Set(pending.map((file) => file.id));
|
|
2284
|
+
for (const id of options.ackFailures.keys()) {
|
|
2285
|
+
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
2286
|
+
}
|
|
2287
|
+
if (pending.length === 0) return 0;
|
|
2288
|
+
options.log({
|
|
2289
|
+
level: "info",
|
|
2290
|
+
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
2291
|
+
});
|
|
2292
|
+
let applied = 0;
|
|
2293
|
+
for (const file of pending) {
|
|
2294
|
+
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
2295
|
+
if (await applyOne(options, file)) applied += 1;
|
|
2296
|
+
}
|
|
2297
|
+
return applied;
|
|
2298
|
+
}
|
|
2299
|
+
async function listPendingFiles(options) {
|
|
2300
|
+
let res;
|
|
2301
|
+
try {
|
|
2302
|
+
res = await options.fetchImpl(`${options.apiUrl}/runners/${options.agentId}/files/pending`, {
|
|
2303
|
+
headers: { Authorization: options.getAuthHeader() }
|
|
2304
|
+
});
|
|
2305
|
+
} catch (err) {
|
|
2306
|
+
options.log({
|
|
2307
|
+
level: "warn",
|
|
2308
|
+
message: `Could not list pending runner files \u2014 retrying on the next drain: ${describe(err)}`
|
|
2309
|
+
});
|
|
2310
|
+
return [];
|
|
2311
|
+
}
|
|
2312
|
+
if (!res.ok) {
|
|
2313
|
+
options.log({
|
|
2314
|
+
level: res.status === 404 ? "debug" : "warn",
|
|
2315
|
+
message: `Listing pending runner files returned HTTP ${res.status} \u2014 retrying on the next drain`
|
|
2316
|
+
});
|
|
2317
|
+
return [];
|
|
2318
|
+
}
|
|
2319
|
+
let body;
|
|
2320
|
+
try {
|
|
2321
|
+
body = await res.json();
|
|
2322
|
+
} catch (err) {
|
|
2323
|
+
options.log({
|
|
2324
|
+
level: "warn",
|
|
2325
|
+
message: `Pending runner file list was not readable JSON \u2014 retrying on the next drain: ${describe(err)}`
|
|
2326
|
+
});
|
|
2327
|
+
return [];
|
|
2328
|
+
}
|
|
2329
|
+
if (!Array.isArray(body)) {
|
|
2330
|
+
options.log({
|
|
2331
|
+
level: "warn",
|
|
2332
|
+
message: "Pending runner file list was not an array \u2014 ignoring it for this drain"
|
|
2333
|
+
});
|
|
2334
|
+
return [];
|
|
2335
|
+
}
|
|
2336
|
+
const files = [];
|
|
2337
|
+
for (const entry of body) {
|
|
2338
|
+
const file = asPendingFile(entry);
|
|
2339
|
+
if (file === null) {
|
|
2340
|
+
options.log({
|
|
2341
|
+
level: "warn",
|
|
2342
|
+
message: "Ignoring a malformed pending runner file entry (expected id, path and size)"
|
|
2343
|
+
});
|
|
2344
|
+
continue;
|
|
2345
|
+
}
|
|
2346
|
+
files.push(file);
|
|
2347
|
+
}
|
|
2348
|
+
return files;
|
|
2349
|
+
}
|
|
2350
|
+
function asPendingFile(entry) {
|
|
2351
|
+
if (entry === null || typeof entry !== "object") return null;
|
|
2352
|
+
const { id, path, size } = entry;
|
|
2353
|
+
if (typeof id !== "string" || id === "") return null;
|
|
2354
|
+
if (typeof path !== "string" || path === "") return null;
|
|
2355
|
+
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
2356
|
+
return { id, path, size };
|
|
2357
|
+
}
|
|
2358
|
+
async function applyOne(options, file) {
|
|
2359
|
+
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
2360
|
+
if (options.allowedDirectories.length === 0) {
|
|
2361
|
+
options.log({
|
|
2362
|
+
level: "warn",
|
|
2363
|
+
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
2364
|
+
});
|
|
2365
|
+
await ack(options, file, "rejected", "file_sync_disabled");
|
|
2366
|
+
return false;
|
|
2367
|
+
}
|
|
2368
|
+
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
2369
|
+
options.log({
|
|
2370
|
+
level: "warn",
|
|
2371
|
+
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
2372
|
+
});
|
|
2373
|
+
await ack(options, file, "rejected", "file_too_large");
|
|
2374
|
+
return false;
|
|
2375
|
+
}
|
|
2376
|
+
const download = await downloadContent(options, file, label);
|
|
2377
|
+
if (!download.ok) {
|
|
2378
|
+
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
2379
|
+
return false;
|
|
2380
|
+
}
|
|
2381
|
+
let outcome;
|
|
2382
|
+
try {
|
|
2383
|
+
outcome = await writePushedFile({
|
|
2384
|
+
requestedPath: file.path,
|
|
2385
|
+
content: download.content,
|
|
2386
|
+
allowedDirectories: options.allowedDirectories,
|
|
2387
|
+
homeDir: options.homeDir
|
|
2388
|
+
});
|
|
2389
|
+
} catch (err) {
|
|
2390
|
+
options.log({
|
|
2391
|
+
level: "error",
|
|
2392
|
+
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
2393
|
+
});
|
|
2394
|
+
await ack(options, file, "rejected", "write_failed");
|
|
2395
|
+
return false;
|
|
2396
|
+
}
|
|
2397
|
+
if (!outcome.ok) {
|
|
2398
|
+
options.log({
|
|
2399
|
+
level: "warn",
|
|
2400
|
+
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
2401
|
+
});
|
|
2402
|
+
await ack(options, file, "rejected", outcome.code);
|
|
2403
|
+
return false;
|
|
2404
|
+
}
|
|
2405
|
+
options.log({
|
|
2406
|
+
level: "info",
|
|
2407
|
+
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
2408
|
+
});
|
|
2409
|
+
await ack(options, file, "applied");
|
|
2410
|
+
return true;
|
|
2411
|
+
}
|
|
2412
|
+
function durableDownloadCode(status) {
|
|
2413
|
+
return status === 413 ? "file_too_large" : "write_failed";
|
|
2414
|
+
}
|
|
2415
|
+
async function downloadContent(options, file, label) {
|
|
2416
|
+
try {
|
|
2417
|
+
const res = await options.fetchImpl(
|
|
2418
|
+
`${options.apiUrl}/runners/${options.agentId}/files/${file.id}/content`,
|
|
2419
|
+
{ headers: { Authorization: options.getAuthHeader() } }
|
|
2420
|
+
);
|
|
2421
|
+
if (!res.ok) {
|
|
2422
|
+
const terminal = res.status >= 400 && res.status < 500 && res.status !== 401 && res.status !== 403 && res.status !== 408 && res.status !== 429;
|
|
2423
|
+
if (!terminal) {
|
|
2424
|
+
options.log({
|
|
2425
|
+
level: "warn",
|
|
2426
|
+
message: `Downloading runner file ${label} returned HTTP ${res.status} \u2014 retrying on the next drain`
|
|
2427
|
+
});
|
|
2428
|
+
return { ok: false, terminal: false };
|
|
2429
|
+
}
|
|
2430
|
+
const code = durableDownloadCode(res.status);
|
|
2431
|
+
options.log({
|
|
2432
|
+
level: "error",
|
|
2433
|
+
message: `Downloading runner file ${label} returned HTTP ${res.status} \u2014 rejecting it as ${code} (the bytes never reached the writer)`
|
|
2434
|
+
});
|
|
2435
|
+
return { ok: false, terminal: true, code };
|
|
2436
|
+
}
|
|
2437
|
+
return { ok: true, content: Buffer.from(await res.arrayBuffer()) };
|
|
2438
|
+
} catch (err) {
|
|
2439
|
+
options.log({
|
|
2440
|
+
level: "warn",
|
|
2441
|
+
message: `Downloading runner file ${label} failed \u2014 retrying on the next drain: ${describe(err)}`
|
|
2442
|
+
});
|
|
2443
|
+
return { ok: false, terminal: false };
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
async function ack(options, file, status, reason) {
|
|
2447
|
+
const outcome = `${status}${reason ? ` (${reason})` : ""}`;
|
|
2448
|
+
try {
|
|
2449
|
+
const res = await options.fetchImpl(
|
|
2450
|
+
`${options.apiUrl}/runners/${options.agentId}/files/${file.id}/ack`,
|
|
2451
|
+
{
|
|
2452
|
+
method: "POST",
|
|
2453
|
+
headers: {
|
|
2454
|
+
Authorization: options.getAuthHeader(),
|
|
2455
|
+
"Content-Type": "application/json"
|
|
2456
|
+
},
|
|
2457
|
+
body: JSON.stringify(reason ? { status, reason } : { status })
|
|
2458
|
+
}
|
|
2459
|
+
);
|
|
2460
|
+
if (!res.ok) {
|
|
2461
|
+
recordAckFailure(
|
|
2462
|
+
options,
|
|
2463
|
+
file,
|
|
2464
|
+
`Acking runner file ${file.id.slice(0, 8)} as ${outcome} returned HTTP ${res.status}`
|
|
2465
|
+
);
|
|
2466
|
+
return;
|
|
2467
|
+
}
|
|
2468
|
+
options.ackFailures.delete(file.id);
|
|
2469
|
+
} catch (err) {
|
|
2470
|
+
recordAckFailure(
|
|
2471
|
+
options,
|
|
2472
|
+
file,
|
|
2473
|
+
`Acking runner file ${file.id.slice(0, 8)} as ${outcome} failed: ${describe(err)}`
|
|
2474
|
+
);
|
|
2475
|
+
}
|
|
2476
|
+
}
|
|
2477
|
+
function recordAckFailure(options, file, what) {
|
|
2478
|
+
const attempts = (options.ackFailures.get(file.id) ?? 0) + 1;
|
|
2479
|
+
options.ackFailures.set(file.id, attempts);
|
|
2480
|
+
options.log({
|
|
2481
|
+
level: "error",
|
|
2482
|
+
message: attempts >= MAX_ACK_ATTEMPTS ? `${what} \u2014 giving up after ${attempts} attempts. It stays pending until the server expires it; restart the runner to retry.` : `${what} \u2014 it stays pending until a later drain re-acks it (attempt ${attempts} of ${MAX_ACK_ATTEMPTS})`
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
function describe(err) {
|
|
2486
|
+
return err instanceof Error ? err.message : String(err);
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
// src/lib/channels/driver.ts
|
|
2490
|
+
function messageIdOf(m) {
|
|
2491
|
+
if (!m || typeof m !== "object") return void 0;
|
|
2492
|
+
if (typeof m.id === "string") return m.id;
|
|
2493
|
+
const infoId = m.info?.id;
|
|
2494
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
2495
|
+
}
|
|
2496
|
+
function cleanImageMime(contentType) {
|
|
2497
|
+
if (!contentType) return null;
|
|
2498
|
+
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
2499
|
+
return /^image\/[a-z0-9.+-]+$/.test(media) ? media : null;
|
|
2500
|
+
}
|
|
2501
|
+
var LOG_LEVELS = {
|
|
2502
|
+
debug: 0,
|
|
2503
|
+
info: 1,
|
|
2504
|
+
warn: 2,
|
|
2505
|
+
error: 3
|
|
2506
|
+
};
|
|
2507
|
+
var DEFAULT_RETRY_POLICY = {
|
|
2508
|
+
maxAttempts: 6,
|
|
2509
|
+
baseDelayMs: 500,
|
|
2510
|
+
maxDelayMs: 3e4
|
|
2511
|
+
};
|
|
2512
|
+
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
2513
|
+
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
2514
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
2515
|
+
var HEARTBEAT_MS = 6e4;
|
|
2516
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
2517
|
+
var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
2518
|
+
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
2519
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
2520
|
+
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
2521
|
+
var ChannelAuthError = class extends Error {
|
|
2522
|
+
constructor(message) {
|
|
2523
|
+
super(message);
|
|
2524
|
+
this.name = "ChannelAuthError";
|
|
2525
|
+
}
|
|
2526
|
+
};
|
|
2527
|
+
var ChannelTerminalError = class extends Error {
|
|
2528
|
+
status;
|
|
2529
|
+
constructor(message, status) {
|
|
2530
|
+
super(message);
|
|
2531
|
+
this.name = "ChannelTerminalError";
|
|
2532
|
+
this.status = status;
|
|
2533
|
+
}
|
|
2534
|
+
};
|
|
2535
|
+
function backoffDelay(attempt, policy) {
|
|
2536
|
+
const exp = policy.baseDelayMs * Math.pow(2, attempt);
|
|
2537
|
+
const capped = Math.min(policy.maxDelayMs, exp);
|
|
2538
|
+
return Math.floor(Math.random() * capped);
|
|
2539
|
+
}
|
|
2540
|
+
function isRetryableStatus(status) {
|
|
2541
|
+
return status === 429 || status >= 500 && status <= 599;
|
|
2542
|
+
}
|
|
2543
|
+
var ChannelDriver = class _ChannelDriver {
|
|
2544
|
+
agentId;
|
|
2545
|
+
port;
|
|
2546
|
+
apiUrl;
|
|
2547
|
+
getAuthHeader;
|
|
2548
|
+
conversationFilter;
|
|
2549
|
+
retry;
|
|
2550
|
+
log;
|
|
2551
|
+
fetchImpl;
|
|
2552
|
+
sleep;
|
|
2553
|
+
pausedPollIntervalMs;
|
|
2554
|
+
pausedMaxWaitMs;
|
|
2555
|
+
stuckQueuedMs;
|
|
2556
|
+
now;
|
|
2557
|
+
fileSyncDirectories;
|
|
2558
|
+
homeDir;
|
|
2559
|
+
/** Cache of conversationId → opencode sessionId. */
|
|
2560
|
+
sessions = /* @__PURE__ */ new Map();
|
|
2561
|
+
/**
|
|
2562
|
+
* conversationId → the opencode session this runner has ABANDONED as that
|
|
2563
|
+
* conversation's binding (#553), after a genuine (`sessionExists === true`)
|
|
2564
|
+
* dispatch failure: the session still exists but is wedged, so #485's self-heal
|
|
2565
|
+
* must bind a fresh one.
|
|
2566
|
+
*
|
|
2567
|
+
* Dropping the local binding + clearing the server row is not enough on its own:
|
|
2568
|
+
* a SIBLING message dispatched earlier in the same drain is still in-flight under
|
|
2569
|
+
* the same session, and its watcher's routine status writes carry
|
|
2570
|
+
* `opencode_session_id`, RESURRECTING the wedged id server-side after the clear —
|
|
2571
|
+
* and `ensureSession`'s persisted-id fallback then reuses it, defeating the
|
|
2572
|
+
* self-heal. This map makes the runner authoritative instead of racing those
|
|
2573
|
+
* writes: *`ensureSession` never reuses an abandoned id for that conversation,
|
|
2574
|
+
* whatever the server row says* — which holds even when the resurrecting write
|
|
2575
|
+
* is one we deliberately keep (see `markDone`).
|
|
2576
|
+
*
|
|
2577
|
+
* Bounded by construction, on both axes: keyed by CONVERSATION, so N failures on
|
|
2578
|
+
* one conversation hold ONE entry (the newest abandonment replaces the older), and
|
|
2579
|
+
* hard-capped at `MAX_SUPERSEDED_CONVERSATIONS` with FIFO eviction. Only the
|
|
2580
|
+
* NEWEST abandoned id per conversation is guarded: after a second abandonment a
|
|
2581
|
+
* late sibling of the FIRST session can write that id back and `ensureSession`
|
|
2582
|
+
* will reuse it — costing ONE repeat failure, which re-supersedes it. Deliberately
|
|
2583
|
+
* NOT dropped when the session's watcher tears down: `markDone` still writes the
|
|
2584
|
+
* abandoned id back (it must, or the reply is lost), so the guard has to outlive
|
|
2585
|
+
* the turn that resurrects it. In-memory only — a restart forgets it, at the same
|
|
2586
|
+
* bounded cost.
|
|
2587
|
+
*/
|
|
2588
|
+
supersededSessions = /* @__PURE__ */ new Map();
|
|
2589
|
+
/**
|
|
2590
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
2591
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
2592
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
2593
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
2594
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
2595
|
+
*/
|
|
1680
2596
|
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1681
2597
|
/**
|
|
1682
2598
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
@@ -1728,6 +2644,15 @@ var ChannelDriver = class {
|
|
|
1728
2644
|
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1729
2645
|
*/
|
|
1730
2646
|
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
2647
|
+
/**
|
|
2648
|
+
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
2649
|
+
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
2650
|
+
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
2651
|
+
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
2652
|
+
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
2653
|
+
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
2654
|
+
*/
|
|
2655
|
+
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
1731
2656
|
/**
|
|
1732
2657
|
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1733
2658
|
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
@@ -1742,6 +2667,15 @@ var ChannelDriver = class {
|
|
|
1742
2667
|
* so the NEXT tick may retry exactly once more).
|
|
1743
2668
|
*/
|
|
1744
2669
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
2670
|
+
/**
|
|
2671
|
+
* "Already signalled `attachments_skipped` for this Evident message id" (#376).
|
|
2672
|
+
* The in-thread skip note is an OUTCOME, so it must fire AT MOST ONCE per message
|
|
2673
|
+
* — never re-post on a re-dispatch of the same row (`forceReadoptRun` or the
|
|
2674
|
+
* next-tick null-id retry both re-run `sendPromptAsync`, which re-fires
|
|
2675
|
+
* `onOutcomes`). Mirrors `readoptPollUnresolvedSignalled`: a local dedup on the
|
|
2676
|
+
* outcome, not the dispatch. Not cleared (a message is signalled once for life).
|
|
2677
|
+
*/
|
|
2678
|
+
attachmentsSkippedSignalled = /* @__PURE__ */ new Set();
|
|
1745
2679
|
/**
|
|
1746
2680
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1747
2681
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1759,8 +2693,39 @@ var ChannelDriver = class {
|
|
|
1759
2693
|
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1760
2694
|
*/
|
|
1761
2695
|
sessionParents = /* @__PURE__ */ new Map();
|
|
2696
|
+
/**
|
|
2697
|
+
* Per-session OpenCode title cache (#310), keyed by sessionId. Only a resolved
|
|
2698
|
+
* NON-EMPTY, non-placeholder name is stored (terminal — a real session name
|
|
2699
|
+
* won't later un-name), so we do NOT re-GET `/session/:id` every tick. "Non-empty"
|
|
2700
|
+
* excludes OpenCode's synchronous default title (see
|
|
2701
|
+
* `OPENCODE_DEFAULT_TITLE_PREFIX`, #549) — that placeholder is treated the same
|
|
2702
|
+
* as an empty title so it never latches. A missing entry = not yet resolved OR
|
|
2703
|
+
* resolved-but-still-empty/placeholder → re-fetch on next need, since OpenCode
|
|
2704
|
+
* names sessions asynchronously mid-turn. Driver-level (not per-watcher) so both
|
|
2705
|
+
* the watcher completion path AND the restart-recovery re-adopt path (which has
|
|
2706
|
+
* no watcher) can resolve the title.
|
|
2707
|
+
*/
|
|
2708
|
+
sessionTitles = /* @__PURE__ */ new Map();
|
|
1762
2709
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1763
2710
|
draining = false;
|
|
2711
|
+
/**
|
|
2712
|
+
* Serialises runner-file syncs (#559) so the ~2s poll tick and a concurrent
|
|
2713
|
+
* drain ping don't download, write and ack the same file twice.
|
|
2714
|
+
*/
|
|
2715
|
+
syncingFiles = false;
|
|
2716
|
+
/**
|
|
2717
|
+
* Consecutive failed acks per pending file (#559). Lives on the driver so it
|
|
2718
|
+
* survives across drains — without it, a file whose ack keeps failing is
|
|
2719
|
+
* re-downloaded and re-written every ~2s until the server expires it.
|
|
2720
|
+
*/
|
|
2721
|
+
fileAckFailures = /* @__PURE__ */ new Map();
|
|
2722
|
+
/**
|
|
2723
|
+
* Monotonic count of files this runner has pulled and written (#559). Only
|
|
2724
|
+
* ever increases, so `run.ts` detects work by comparing it against the value
|
|
2725
|
+
* it saw on the previous cycle — including work that landed mid-sleep, the
|
|
2726
|
+
* same trick `lastProxiedActivityAt` uses.
|
|
2727
|
+
*/
|
|
2728
|
+
appliedFileCount = 0;
|
|
1764
2729
|
/**
|
|
1765
2730
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1766
2731
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -1791,14 +2756,13 @@ var ChannelDriver = class {
|
|
|
1791
2756
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1792
2757
|
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1793
2758
|
this.now = config2.now ?? (() => Date.now());
|
|
2759
|
+
this.fileSyncDirectories = config2.fileSyncDirectories ?? [];
|
|
2760
|
+
this.homeDir = config2.homeDir ?? homedir();
|
|
1794
2761
|
}
|
|
1795
2762
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
1796
2763
|
get opencodeBase() {
|
|
1797
2764
|
return `http://127.0.0.1:${this.port}`;
|
|
1798
2765
|
}
|
|
1799
|
-
// -------------------------------------------------------------------------
|
|
1800
|
-
// Public API
|
|
1801
|
-
// -------------------------------------------------------------------------
|
|
1802
2766
|
/**
|
|
1803
2767
|
* Drain all pending channel conversations once: poll → dispatch → register.
|
|
1804
2768
|
* Called on tunnel `connected` (WI-CHAN-4) and on each poll tick by `run.ts`.
|
|
@@ -1821,6 +2785,47 @@ var ChannelDriver = class {
|
|
|
1821
2785
|
);
|
|
1822
2786
|
return run2;
|
|
1823
2787
|
}
|
|
2788
|
+
/**
|
|
2789
|
+
* Pull-and-apply any files Evident has queued for this runner (#559), riding
|
|
2790
|
+
* the EXISTING drain cycle — `run.ts` calls it from the same ~2s channel poll
|
|
2791
|
+
* and drain ping that call `drainPending()`. There is deliberately no channel,
|
|
2792
|
+
* control frame or poll loop of its own: worst-case latency is one poll tick.
|
|
2793
|
+
*
|
|
2794
|
+
* NEVER throws and never surfaces a `ChannelAuthError`: a file failure must not
|
|
2795
|
+
* cost a conversation turn. Failures are logged and either acked as a terminal
|
|
2796
|
+
* outcome or left pending for the next drain (see `runner-file-sync.ts`).
|
|
2797
|
+
*
|
|
2798
|
+
* Re-entrant calls are skipped (the poll tick and a drain ping can overlap).
|
|
2799
|
+
*
|
|
2800
|
+
* @returns the number of files written to disk.
|
|
2801
|
+
*/
|
|
2802
|
+
async syncPendingFiles() {
|
|
2803
|
+
if (this.stopped) return 0;
|
|
2804
|
+
if (this.syncingFiles) return 0;
|
|
2805
|
+
this.syncingFiles = true;
|
|
2806
|
+
try {
|
|
2807
|
+
const applied = await syncPendingRunnerFiles({
|
|
2808
|
+
agentId: this.agentId,
|
|
2809
|
+
apiUrl: this.apiUrl,
|
|
2810
|
+
getAuthHeader: this.getAuthHeader,
|
|
2811
|
+
fetchImpl: this.fetchImpl,
|
|
2812
|
+
allowedDirectories: this.fileSyncDirectories,
|
|
2813
|
+
homeDir: this.homeDir,
|
|
2814
|
+
ackFailures: this.fileAckFailures,
|
|
2815
|
+
log: this.log
|
|
2816
|
+
});
|
|
2817
|
+
this.appliedFileCount += applied;
|
|
2818
|
+
return applied;
|
|
2819
|
+
} catch (err) {
|
|
2820
|
+
this.log({
|
|
2821
|
+
level: "error",
|
|
2822
|
+
message: `Runner file sync failed unexpectedly (message processing is unaffected): ${err instanceof Error ? err.message : String(err)}`
|
|
2823
|
+
});
|
|
2824
|
+
return 0;
|
|
2825
|
+
} finally {
|
|
2826
|
+
this.syncingFiles = false;
|
|
2827
|
+
}
|
|
2828
|
+
}
|
|
1824
2829
|
async runDrain() {
|
|
1825
2830
|
let dispatched = 0;
|
|
1826
2831
|
try {
|
|
@@ -1854,6 +2859,50 @@ var ChannelDriver = class {
|
|
|
1854
2859
|
}
|
|
1855
2860
|
return false;
|
|
1856
2861
|
}
|
|
2862
|
+
/**
|
|
2863
|
+
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
2864
|
+
*
|
|
2865
|
+
* Pulling a file is real work that `drainPending()` knows nothing about, so
|
|
2866
|
+
* without this a near-idle runner counts a credential pull as an empty tick
|
|
2867
|
+
* and `--idle-timeout` can `process.exit` mid-pull — leaving a
|
|
2868
|
+
* `.evident-push-*.tmp` behind — or immediately after the write, before the
|
|
2869
|
+
* browser has run the authorize/callback that activates it (the user then sees
|
|
2870
|
+
* `saved_not_activated` for a runner that was fine).
|
|
2871
|
+
*
|
|
2872
|
+
* Two signals because one cannot cover both cases: `inFlight` is the pull
|
|
2873
|
+
* happening RIGHT NOW (it may outlive the tick that started it), and
|
|
2874
|
+
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
2875
|
+
* idle checks still shows up as an advance.
|
|
2876
|
+
*
|
|
2877
|
+
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
2878
|
+
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
2879
|
+
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
2880
|
+
*/
|
|
2881
|
+
fileSyncActivity() {
|
|
2882
|
+
return { appliedFiles: this.appliedFileCount, inFlight: this.syncingFiles };
|
|
2883
|
+
}
|
|
2884
|
+
/**
|
|
2885
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2886
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2887
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2888
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2889
|
+
*
|
|
2890
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2891
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2892
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2893
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2894
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2895
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2896
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2897
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2898
|
+
*/
|
|
2899
|
+
protectedSessionIds() {
|
|
2900
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2901
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2902
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2903
|
+
}
|
|
2904
|
+
return ids;
|
|
2905
|
+
}
|
|
1857
2906
|
/**
|
|
1858
2907
|
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
1859
2908
|
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
@@ -1893,7 +2942,7 @@ var ChannelDriver = class {
|
|
|
1893
2942
|
await this.sleep(step);
|
|
1894
2943
|
}
|
|
1895
2944
|
}
|
|
1896
|
-
while (this.hasInFlightWatchers()) {
|
|
2945
|
+
while (this.hasInFlightWatchers() || this.syncingFiles) {
|
|
1897
2946
|
if (this.now() >= deadline) return false;
|
|
1898
2947
|
await this.sleep(step);
|
|
1899
2948
|
}
|
|
@@ -1918,9 +2967,7 @@ var ChannelDriver = class {
|
|
|
1918
2967
|
if (!stillLive) return;
|
|
1919
2968
|
}
|
|
1920
2969
|
}
|
|
1921
|
-
// -------------------------------------------------------------------------
|
|
1922
2970
|
// Conversation processing (WI-3 — async dispatch)
|
|
1923
|
-
// -------------------------------------------------------------------------
|
|
1924
2971
|
/**
|
|
1925
2972
|
* Dispatch each pending message for a conversation to opencode's native queue
|
|
1926
2973
|
* via `prompt_async` (Task 3.2) and register it with the conversation's
|
|
@@ -1930,10 +2977,15 @@ var ChannelDriver = class {
|
|
|
1930
2977
|
* @returns the count of messages NEWLY dispatched (not already in-flight).
|
|
1931
2978
|
*/
|
|
1932
2979
|
async processConversation(conv) {
|
|
1933
|
-
const sessionId = await this.ensureSession(conv);
|
|
2980
|
+
const { sessionId, refusedSessionId } = await this.ensureSession(conv);
|
|
1934
2981
|
const messages = await this.getPendingMessages(conv.id);
|
|
1935
2982
|
let dispatched = 0;
|
|
1936
2983
|
let skippedAlreadyDispatched = 0;
|
|
2984
|
+
if (refusedSessionId && messages.length > 0) {
|
|
2985
|
+
void this.postSignal(conv.id, messages[0].id, "session_superseded", {
|
|
2986
|
+
superseded_session_id: refusedSessionId
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
1937
2989
|
for (const message of messages) {
|
|
1938
2990
|
if (this.stopped) break;
|
|
1939
2991
|
if (this.dispatched.has(message.id)) {
|
|
@@ -1952,26 +3004,62 @@ var ChannelDriver = class {
|
|
|
1952
3004
|
conversation_id: conv.id,
|
|
1953
3005
|
message_id: message.id
|
|
1954
3006
|
});
|
|
3007
|
+
const sendAttachments = this.buildSendAttachments(conv, message);
|
|
1955
3008
|
opencodeMessageId = await this.dispatchLocked(
|
|
1956
3009
|
sessionId,
|
|
1957
|
-
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
3010
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options, sendAttachments)
|
|
1958
3011
|
);
|
|
1959
3012
|
} catch (err) {
|
|
1960
3013
|
if (err instanceof ChannelAuthError) throw err;
|
|
1961
3014
|
this.dispatched.delete(message.id);
|
|
1962
|
-
await this.
|
|
3015
|
+
const exists = await sessionExists(this.port, sessionId);
|
|
3016
|
+
if (exists === false) {
|
|
3017
|
+
this.sessions.delete(conv.id);
|
|
3018
|
+
this.log({
|
|
3019
|
+
level: "warn",
|
|
3020
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
3021
|
+
conversation_id: conv.id,
|
|
3022
|
+
message_id: message.id
|
|
3023
|
+
});
|
|
3024
|
+
break;
|
|
3025
|
+
}
|
|
3026
|
+
if (exists === null) {
|
|
3027
|
+
this.log({
|
|
3028
|
+
level: "warn",
|
|
3029
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed and session (${sessionId.slice(0, 8)}) existence could not be confirmed (opencode momentarily unreachable) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick rather than treating it as a genuine failure.`,
|
|
3030
|
+
conversation_id: conv.id,
|
|
3031
|
+
message_id: message.id
|
|
3032
|
+
});
|
|
3033
|
+
break;
|
|
3034
|
+
}
|
|
3035
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
3036
|
+
this.sessions.delete(conv.id);
|
|
3037
|
+
this.supersede(conv.id, sessionId);
|
|
3038
|
+
this.log({
|
|
3039
|
+
level: "warn",
|
|
3040
|
+
message: `Abandoning OpenCode session ${sessionId.slice(0, 8)} as the binding for conversation ${conv.id.slice(0, 8)} (it exists but failed to run a turn) \u2014 a fresh session is created on the next tick, whatever the persisted binding says by then.`,
|
|
3041
|
+
conversation_id: conv.id,
|
|
3042
|
+
message_id: message.id
|
|
3043
|
+
});
|
|
3044
|
+
await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
|
|
3045
|
+
this.log({
|
|
3046
|
+
level: "warn",
|
|
3047
|
+
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
3048
|
+
conversation_id: conv.id,
|
|
3049
|
+
message_id: message.id
|
|
3050
|
+
});
|
|
1963
3051
|
});
|
|
1964
3052
|
this.log({
|
|
1965
3053
|
level: "error",
|
|
1966
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
3054
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage}`,
|
|
1967
3055
|
conversation_id: conv.id,
|
|
1968
3056
|
message_id: message.id
|
|
1969
3057
|
});
|
|
1970
|
-
|
|
3058
|
+
break;
|
|
1971
3059
|
}
|
|
1972
3060
|
if (opencodeMessageId === null) {
|
|
1973
3061
|
this.log({
|
|
1974
|
-
level: "
|
|
3062
|
+
level: "warn",
|
|
1975
3063
|
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
1976
3064
|
conversation_id: conv.id,
|
|
1977
3065
|
message_id: message.id
|
|
@@ -1985,7 +3073,7 @@ var ChannelDriver = class {
|
|
|
1985
3073
|
}
|
|
1986
3074
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1987
3075
|
this.log({
|
|
1988
|
-
level: "
|
|
3076
|
+
level: "warn",
|
|
1989
3077
|
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
1990
3078
|
conversation_id: conv.id
|
|
1991
3079
|
});
|
|
@@ -1993,17 +3081,68 @@ var ChannelDriver = class {
|
|
|
1993
3081
|
this.ensureWatcherRunning(sessionId);
|
|
1994
3082
|
return dispatched;
|
|
1995
3083
|
}
|
|
3084
|
+
/**
|
|
3085
|
+
* Record that `sessionId` is no longer a valid binding for `conversationId`
|
|
3086
|
+
* (#553). Keyed by conversation and hard-capped, so it cannot grow with the
|
|
3087
|
+
* number of failures — see the `supersededSessions` field doc.
|
|
3088
|
+
*/
|
|
3089
|
+
supersede(conversationId, sessionId) {
|
|
3090
|
+
this.supersededSessions.delete(conversationId);
|
|
3091
|
+
this.supersededSessions.set(conversationId, sessionId);
|
|
3092
|
+
while (this.supersededSessions.size > MAX_SUPERSEDED_CONVERSATIONS) {
|
|
3093
|
+
const oldest = this.supersededSessions.keys().next().value;
|
|
3094
|
+
if (oldest === void 0) return;
|
|
3095
|
+
this.supersededSessions.delete(oldest);
|
|
3096
|
+
}
|
|
3097
|
+
}
|
|
3098
|
+
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
3099
|
+
isSuperseded(conversationId, sessionId) {
|
|
3100
|
+
return this.supersededSessions.get(conversationId) === sessionId;
|
|
3101
|
+
}
|
|
3102
|
+
/**
|
|
3103
|
+
* Resolve the opencode session to run this conversation's turns in.
|
|
3104
|
+
*
|
|
3105
|
+
* `refusedSessionId` is set when the #553 guard fired — i.e. the persisted
|
|
3106
|
+
* binding was an id this runner had abandoned, so a resurrection genuinely
|
|
3107
|
+
* happened and a fresh session was bound instead. The caller reports it.
|
|
3108
|
+
*/
|
|
1996
3109
|
async ensureSession(conv) {
|
|
1997
|
-
const
|
|
1998
|
-
if (
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
3110
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
3111
|
+
if (bound && this.isSuperseded(conv.id, bound)) {
|
|
3112
|
+
this.log({
|
|
3113
|
+
level: "warn",
|
|
3114
|
+
message: `OpenCode session ${bound.slice(0, 8)} was abandoned for conversation ${conv.id.slice(0, 8)} after a failed dispatch but is still bound to it (the persisted id was written back by a turn already in flight) \u2014 ignoring it and binding a fresh session.`,
|
|
3115
|
+
conversation_id: conv.id
|
|
3116
|
+
});
|
|
3117
|
+
this.sessions.delete(conv.id);
|
|
3118
|
+
return { sessionId: await this.createAndBindSession(conv.id), refusedSessionId: bound };
|
|
2002
3119
|
}
|
|
3120
|
+
if (bound) {
|
|
3121
|
+
const exists = await sessionExists(this.port, bound);
|
|
3122
|
+
if (exists === false) {
|
|
3123
|
+
this.log({
|
|
3124
|
+
level: "debug",
|
|
3125
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
3126
|
+
conversation_id: conv.id
|
|
3127
|
+
});
|
|
3128
|
+
this.sessions.delete(conv.id);
|
|
3129
|
+
return { sessionId: await this.createAndBindSession(conv.id) };
|
|
3130
|
+
}
|
|
3131
|
+
this.sessions.set(conv.id, bound);
|
|
3132
|
+
return { sessionId: bound };
|
|
3133
|
+
}
|
|
3134
|
+
return { sessionId: await this.createAndBindSession(conv.id) };
|
|
3135
|
+
}
|
|
3136
|
+
/**
|
|
3137
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
3138
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
3139
|
+
* self-heal recreate path in `ensureSession`.
|
|
3140
|
+
*/
|
|
3141
|
+
async createAndBindSession(conversationId) {
|
|
2003
3142
|
const directory = await this.resolveOpenCodeDirectory();
|
|
2004
3143
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
2005
|
-
this.sessions.set(
|
|
2006
|
-
await this.persistSession(
|
|
3144
|
+
this.sessions.set(conversationId, sessionId);
|
|
3145
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
2007
3146
|
});
|
|
2008
3147
|
return sessionId;
|
|
2009
3148
|
}
|
|
@@ -2017,15 +3156,13 @@ var ChannelDriver = class {
|
|
|
2017
3156
|
this.opencodeDirectory = await getOpenCodeDirectory(this.port);
|
|
2018
3157
|
if (!this.opencodeDirectory) {
|
|
2019
3158
|
this.log({
|
|
2020
|
-
level: "
|
|
3159
|
+
level: "warn",
|
|
2021
3160
|
message: "Could not determine opencode directory (GET /path) \u2014 new sessions may not appear in opencode web"
|
|
2022
3161
|
});
|
|
2023
3162
|
}
|
|
2024
3163
|
return this.opencodeDirectory;
|
|
2025
3164
|
}
|
|
2026
|
-
// -------------------------------------------------------------------------
|
|
2027
3165
|
// Per-session watcher (WI-3)
|
|
2028
|
-
// -------------------------------------------------------------------------
|
|
2029
3166
|
/**
|
|
2030
3167
|
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2031
3168
|
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
@@ -2045,6 +3182,130 @@ var ChannelDriver = class {
|
|
|
2045
3182
|
);
|
|
2046
3183
|
return run2;
|
|
2047
3184
|
}
|
|
3185
|
+
// Inbound image attachments (#255, WI-8)
|
|
3186
|
+
/**
|
|
3187
|
+
* Build the `SendAttachmentsInput` for a message's inbound images, or
|
|
3188
|
+
* `undefined` when the message has none (so a text-only turn is unchanged).
|
|
3189
|
+
*
|
|
3190
|
+
* The driver OWNS the two channel-facing concerns the session module cannot:
|
|
3191
|
+
* - the AUTHENTICATED byte fetch through Evident's WI-6 endpoint
|
|
3192
|
+
* (`fetchAttachmentDataUrl`), using the SAME `getAuthHeader()` as every
|
|
3193
|
+
* other combinedAuth callback — the CLI NEVER talks to Slack directly;
|
|
3194
|
+
* - the in-thread SKIP NOTE (`signalAttachmentsSkipped`) posted over the
|
|
3195
|
+
* existing callback surface when any image was skipped/failed.
|
|
3196
|
+
* `sendPromptAsync` applies the capability gate + appends the `file` parts and
|
|
3197
|
+
* reports outcomes back via `onOutcomes`.
|
|
3198
|
+
*/
|
|
3199
|
+
buildSendAttachments(conv, message) {
|
|
3200
|
+
const refs = message.attachments;
|
|
3201
|
+
if (!refs || refs.length === 0) return void 0;
|
|
3202
|
+
return {
|
|
3203
|
+
inputs: refs.map((a, index) => ({
|
|
3204
|
+
index,
|
|
3205
|
+
mime: a.mime,
|
|
3206
|
+
...a.filename ? { filename: a.filename } : {}
|
|
3207
|
+
})),
|
|
3208
|
+
fetchDataUrl: (index) => this.fetchAttachmentDataUrl(message.id, index, refs[index].mime),
|
|
3209
|
+
onOutcomes: ({ outcomes, capabilityUnknown }) => this.signalAttachmentsSkipped(conv.id, message.id, outcomes, capabilityUnknown)
|
|
3210
|
+
};
|
|
3211
|
+
}
|
|
3212
|
+
/**
|
|
3213
|
+
* Fetch ONE inbound image's bytes through Evident's WI-6 endpoint
|
|
3214
|
+
* (`GET {apiUrl}/runners/{agentId}/attachments/{messageId}/{index}`) using the
|
|
3215
|
+
* existing authenticated fetch, and base64-encode into a
|
|
3216
|
+
* `data:<mime>;base64,<…>` URL for the opencode `file` part's `url`.
|
|
3217
|
+
*
|
|
3218
|
+
* The endpoint streams the source bytes verbatim (200), or returns 404
|
|
3219
|
+
* (not-owned / out-of-range / deleted-at-source / workspace gone) / 413
|
|
3220
|
+
* (over-cap). On ANY non-2xx or thrown failure we return `null` so the caller
|
|
3221
|
+
* OMITS that one image and the text turn still sends — NEVER throws the turn.
|
|
3222
|
+
* A 404 body carrying `{ reason: 'needs_reauth' }` (#547 — the server CONFIRMED
|
|
3223
|
+
* a Slack `files:read` scope problem via `files.info`) instead resolves the
|
|
3224
|
+
* `AttachmentFetchNeedsReauth` sentinel, so the in-thread note can steer the
|
|
3225
|
+
* user to reconnect Slack instead of a generic "unavailable". Failures are
|
|
3226
|
+
* logged with context (no silent swallow).
|
|
3227
|
+
*/
|
|
3228
|
+
async fetchAttachmentDataUrl(messageId, index, mime) {
|
|
3229
|
+
try {
|
|
3230
|
+
const res = await this.fetchImpl(
|
|
3231
|
+
`${this.apiUrl}/runners/${this.agentId}/attachments/${messageId}/${index}`,
|
|
3232
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
3233
|
+
);
|
|
3234
|
+
if (!res.ok) {
|
|
3235
|
+
let reason;
|
|
3236
|
+
try {
|
|
3237
|
+
const body = await res.json();
|
|
3238
|
+
if (body && typeof body.reason === "string") reason = body.reason;
|
|
3239
|
+
} catch (parseErr) {
|
|
3240
|
+
this.log({
|
|
3241
|
+
level: "debug",
|
|
3242
|
+
message: `Attachment fetch for message ${messageId.slice(0, 8)} index ${index}: error body was not JSON (${parseErr instanceof Error ? parseErr.message : String(parseErr)}) \u2014 treating as a plain failure`,
|
|
3243
|
+
message_id: messageId
|
|
3244
|
+
});
|
|
3245
|
+
}
|
|
3246
|
+
if (reason === "needs_reauth") {
|
|
3247
|
+
this.log({
|
|
3248
|
+
level: "error",
|
|
3249
|
+
message: `Attachment fetch for message ${messageId.slice(0, 8)} index ${index} returned HTTP ${res.status} \u2014 server confirmed a Slack reauth/scope problem \u2014 omitting this image (text turn proceeds)`,
|
|
3250
|
+
message_id: messageId
|
|
3251
|
+
});
|
|
3252
|
+
return { needsReauth: true };
|
|
3253
|
+
}
|
|
3254
|
+
this.log({
|
|
3255
|
+
level: "error",
|
|
3256
|
+
message: `Attachment fetch for message ${messageId.slice(0, 8)} index ${index} returned HTTP ${res.status} \u2014 omitting this image (text turn proceeds)`,
|
|
3257
|
+
message_id: messageId
|
|
3258
|
+
});
|
|
3259
|
+
return null;
|
|
3260
|
+
}
|
|
3261
|
+
const buf = await res.arrayBuffer();
|
|
3262
|
+
const base64 = Buffer.from(buf).toString("base64");
|
|
3263
|
+
const dataMime = cleanImageMime(res.headers.get("content-type")) || mime;
|
|
3264
|
+
return `data:${dataMime};base64,${base64}`;
|
|
3265
|
+
} catch (err) {
|
|
3266
|
+
this.log({
|
|
3267
|
+
level: "error",
|
|
3268
|
+
message: `Attachment fetch for message ${messageId.slice(0, 8)} index ${index} failed \u2014 omitting this image (text turn proceeds): ${err instanceof Error ? err.message : String(err)}`,
|
|
3269
|
+
message_id: messageId
|
|
3270
|
+
});
|
|
3271
|
+
return null;
|
|
3272
|
+
}
|
|
3273
|
+
}
|
|
3274
|
+
/**
|
|
3275
|
+
* On any skipped/failed image, post an in-thread note to Evident over the
|
|
3276
|
+
* EXISTING combinedAuth callback surface — the CLI NEVER posts to Slack directly.
|
|
3277
|
+
* Evident routes the note to source via `conversation.deliver`.
|
|
3278
|
+
*
|
|
3279
|
+
* The `POST .../messages/:id/signal` route accepts `attachments_skipped` (in
|
|
3280
|
+
* `messageSignalSchema`) and turns it into an in-thread note delivered through
|
|
3281
|
+
* `conversation.deliver` (e.g. "N image(s) couldn't be forwarded"), so the note
|
|
3282
|
+
* reaches the channel.
|
|
3283
|
+
*
|
|
3284
|
+
* Fire-and-forget: never throws into the send/tick (logs its own failure).
|
|
3285
|
+
*/
|
|
3286
|
+
signalAttachmentsSkipped(conversationId, messageId, outcomes, capabilityUnknown) {
|
|
3287
|
+
const skipped = outcomes.filter((o) => o.status === "skipped").length;
|
|
3288
|
+
const failed = outcomes.filter((o) => o.status === "failed").length;
|
|
3289
|
+
if (skipped === 0 && failed === 0) return;
|
|
3290
|
+
if (this.attachmentsSkippedSignalled.has(messageId)) return;
|
|
3291
|
+
this.attachmentsSkippedSignalled.add(messageId);
|
|
3292
|
+
const skippedReason = capabilityUnknown ? "unknown" : "unsupported";
|
|
3293
|
+
const failedReason = outcomes.some(
|
|
3294
|
+
(o) => o.status === "failed" && o.reason === "needs_reauth"
|
|
3295
|
+
) ? "needs_reauth" : void 0;
|
|
3296
|
+
this.log({
|
|
3297
|
+
level: "info",
|
|
3298
|
+
message: `Message ${messageId.slice(0, 8)}: ${skipped} image(s) skipped (${capabilityUnknown ? "capability was unreadable \u2014 failed open to text-only" : "model not attachment-capable"}), ${failed} image(s) unavailable (deleted-at-source or fetch failure) \u2014 noting to Evident`,
|
|
3299
|
+
conversation_id: conversationId,
|
|
3300
|
+
message_id: messageId
|
|
3301
|
+
});
|
|
3302
|
+
void this.postSignal(conversationId, messageId, "attachments_skipped", {
|
|
3303
|
+
skipped,
|
|
3304
|
+
failed,
|
|
3305
|
+
...skipped > 0 ? { skipped_reason: skippedReason } : {},
|
|
3306
|
+
...failedReason ? { failed_reason: failedReason } : {}
|
|
3307
|
+
});
|
|
3308
|
+
}
|
|
2048
3309
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
2049
3310
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
2050
3311
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -2054,7 +3315,9 @@ var ChannelDriver = class {
|
|
|
2054
3315
|
inFlight: /* @__PURE__ */ new Map(),
|
|
2055
3316
|
loop: null,
|
|
2056
3317
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2057
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
3318
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
3319
|
+
lastGoodPollAt: this.now(),
|
|
3320
|
+
hadUsablePoll: false
|
|
2058
3321
|
};
|
|
2059
3322
|
this.watchers.set(sessionId, watcher);
|
|
2060
3323
|
}
|
|
@@ -2064,20 +3327,42 @@ var ChannelDriver = class {
|
|
|
2064
3327
|
opencodeMessageId,
|
|
2065
3328
|
message,
|
|
2066
3329
|
dispatchedAt: now,
|
|
3330
|
+
processingAnchorMs: now,
|
|
2067
3331
|
deadline: now + this.pausedMaxWaitMs,
|
|
2068
3332
|
started: false,
|
|
2069
3333
|
done: false,
|
|
2070
|
-
stuckReported: false
|
|
3334
|
+
stuckReported: false,
|
|
3335
|
+
lastAliveAt: 0,
|
|
3336
|
+
aliveInFlight: false,
|
|
3337
|
+
titleSynced: false,
|
|
3338
|
+
titleSyncInFlight: false,
|
|
3339
|
+
awaitingHumanLatched: false,
|
|
3340
|
+
pausedOnQuestion: false,
|
|
3341
|
+
pausedOnPermission: false,
|
|
3342
|
+
pausedClearConfirmed: false,
|
|
3343
|
+
pausedInFlight: false,
|
|
3344
|
+
deliveryDeadlineAnchored: false,
|
|
3345
|
+
b2PinnedSinceMs: 0,
|
|
3346
|
+
b2LastDescendantCheckMs: 0,
|
|
3347
|
+
b2AbandonedSignalled: false
|
|
2071
3348
|
});
|
|
2072
3349
|
}
|
|
2073
3350
|
/**
|
|
2074
3351
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
2075
3352
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
2076
3353
|
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
2077
|
-
* `now
|
|
2078
|
-
* (10 min after `processed_at
|
|
2079
|
-
*
|
|
2080
|
-
*
|
|
3354
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
3355
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
3356
|
+
*
|
|
3357
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
3358
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
3359
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
3360
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
3361
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
3362
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
3363
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
3364
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
3365
|
+
* (only the appear-guard uses it).
|
|
2081
3366
|
*
|
|
2082
3367
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
2083
3368
|
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
@@ -2095,7 +3380,9 @@ var ChannelDriver = class {
|
|
|
2095
3380
|
inFlight: /* @__PURE__ */ new Map(),
|
|
2096
3381
|
loop: null,
|
|
2097
3382
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2098
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
3383
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
3384
|
+
lastGoodPollAt: this.now(),
|
|
3385
|
+
hadUsablePoll: false
|
|
2099
3386
|
};
|
|
2100
3387
|
this.watchers.set(sessionId, watcher);
|
|
2101
3388
|
}
|
|
@@ -2104,6 +3391,10 @@ var ChannelDriver = class {
|
|
|
2104
3391
|
opencodeMessageId,
|
|
2105
3392
|
message,
|
|
2106
3393
|
dispatchedAt: this.now(),
|
|
3394
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
3395
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
3396
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
3397
|
+
processingAnchorMs: processedAtMs,
|
|
2107
3398
|
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2108
3399
|
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2109
3400
|
started: true,
|
|
@@ -2113,7 +3404,25 @@ var ChannelDriver = class {
|
|
|
2113
3404
|
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2114
3405
|
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2115
3406
|
// (#210/#220 observability).
|
|
2116
|
-
stuckReported: false
|
|
3407
|
+
stuckReported: false,
|
|
3408
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
3409
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
3410
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
3411
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
3412
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
3413
|
+
lastAliveAt: 0,
|
|
3414
|
+
aliveInFlight: false,
|
|
3415
|
+
titleSynced: false,
|
|
3416
|
+
titleSyncInFlight: false,
|
|
3417
|
+
awaitingHumanLatched: false,
|
|
3418
|
+
pausedOnQuestion: false,
|
|
3419
|
+
pausedOnPermission: false,
|
|
3420
|
+
pausedClearConfirmed: false,
|
|
3421
|
+
pausedInFlight: false,
|
|
3422
|
+
deliveryDeadlineAnchored: false,
|
|
3423
|
+
b2PinnedSinceMs: 0,
|
|
3424
|
+
b2LastDescendantCheckMs: 0,
|
|
3425
|
+
b2AbandonedSignalled: false
|
|
2117
3426
|
});
|
|
2118
3427
|
}
|
|
2119
3428
|
/**
|
|
@@ -2164,12 +3473,30 @@ var ChannelDriver = class {
|
|
|
2164
3473
|
messages = Array.isArray(body) ? body : null;
|
|
2165
3474
|
}
|
|
2166
3475
|
} catch {
|
|
2167
|
-
continue;
|
|
2168
3476
|
}
|
|
3477
|
+
if (messages != null && messages.length > 0) {
|
|
3478
|
+
watcher.lastGoodPollAt = this.now();
|
|
3479
|
+
watcher.hadUsablePoll = true;
|
|
3480
|
+
} else {
|
|
3481
|
+
const emptyButReachable = messages != null;
|
|
3482
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
3483
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
3484
|
+
continue;
|
|
3485
|
+
}
|
|
3486
|
+
}
|
|
3487
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
2169
3488
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2170
|
-
await this.serviceInFlightMessage(
|
|
3489
|
+
await this.serviceInFlightMessage(
|
|
3490
|
+
sessionId,
|
|
3491
|
+
watcher,
|
|
3492
|
+
inFlight,
|
|
3493
|
+
messages,
|
|
3494
|
+
openQuestions,
|
|
3495
|
+
openPermissions,
|
|
3496
|
+
questionsPolledOk,
|
|
3497
|
+
permissionsPolledOk
|
|
3498
|
+
);
|
|
2171
3499
|
}
|
|
2172
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
2173
3500
|
}
|
|
2174
3501
|
} catch (err) {
|
|
2175
3502
|
if (err instanceof ChannelAuthError) {
|
|
@@ -2191,28 +3518,55 @@ var ChannelDriver = class {
|
|
|
2191
3518
|
});
|
|
2192
3519
|
}
|
|
2193
3520
|
}
|
|
3521
|
+
/**
|
|
3522
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
3523
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
3524
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
3525
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
3526
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
3527
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
3528
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
3529
|
+
* or past now, so a still-ample window is left untouched.
|
|
3530
|
+
*/
|
|
3531
|
+
anchorDeliveryDeadline(inFlight) {
|
|
3532
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
3533
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
3534
|
+
if (this.now() >= inFlight.deadline) {
|
|
3535
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
3536
|
+
}
|
|
3537
|
+
}
|
|
2194
3538
|
/**
|
|
2195
3539
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2196
3540
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2197
3541
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2198
3542
|
* in-flight set on completion or timeout.
|
|
2199
3543
|
*/
|
|
2200
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
3544
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
2201
3545
|
const conv = watcher.conv;
|
|
2202
3546
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
3547
|
+
const id = inFlight.evidentMessageId;
|
|
3548
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
3549
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
3550
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
3551
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
3552
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
3553
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
3554
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2203
3555
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
3556
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
2204
3557
|
let claimed;
|
|
2205
3558
|
try {
|
|
2206
3559
|
claimed = await this.markProcessing(
|
|
2207
3560
|
conv.id,
|
|
2208
3561
|
inFlight.evidentMessageId,
|
|
2209
3562
|
sessionId,
|
|
2210
|
-
inFlight.opencodeMessageId
|
|
3563
|
+
inFlight.opencodeMessageId,
|
|
3564
|
+
title
|
|
2211
3565
|
);
|
|
2212
3566
|
} catch (err) {
|
|
2213
3567
|
if (err instanceof ChannelAuthError) throw err;
|
|
2214
3568
|
this.log({
|
|
2215
|
-
level: "
|
|
3569
|
+
level: "warn",
|
|
2216
3570
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2217
3571
|
conversation_id: conv.id,
|
|
2218
3572
|
message_id: inFlight.evidentMessageId
|
|
@@ -2222,7 +3576,7 @@ var ChannelDriver = class {
|
|
|
2222
3576
|
inFlight.started = true;
|
|
2223
3577
|
if (!claimed) {
|
|
2224
3578
|
this.log({
|
|
2225
|
-
level: "
|
|
3579
|
+
level: "debug",
|
|
2226
3580
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} already marked processing \u2014 continuing`,
|
|
2227
3581
|
conversation_id: conv.id,
|
|
2228
3582
|
message_id: inFlight.evidentMessageId
|
|
@@ -2230,56 +3584,11 @@ var ChannelDriver = class {
|
|
|
2230
3584
|
}
|
|
2231
3585
|
}
|
|
2232
3586
|
if (state === "done") {
|
|
2233
|
-
|
|
2234
|
-
this.log({
|
|
2235
|
-
level: "info",
|
|
2236
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
2237
|
-
conversation_id: conv.id,
|
|
2238
|
-
message_id: inFlight.evidentMessageId
|
|
2239
|
-
});
|
|
2240
|
-
try {
|
|
2241
|
-
await this.markDone(
|
|
2242
|
-
conv.id,
|
|
2243
|
-
inFlight.evidentMessageId,
|
|
2244
|
-
sessionId,
|
|
2245
|
-
inFlight.opencodeMessageId
|
|
2246
|
-
);
|
|
2247
|
-
} catch (err) {
|
|
2248
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2249
|
-
if (err instanceof ChannelTerminalError) {
|
|
2250
|
-
this.log({
|
|
2251
|
-
level: "error",
|
|
2252
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2253
|
-
conversation_id: conv.id,
|
|
2254
|
-
message_id: inFlight.evidentMessageId
|
|
2255
|
-
});
|
|
2256
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2257
|
-
return;
|
|
2258
|
-
}
|
|
2259
|
-
if (this.now() >= inFlight.deadline) {
|
|
2260
|
-
this.log({
|
|
2261
|
-
level: "error",
|
|
2262
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2263
|
-
conversation_id: conv.id,
|
|
2264
|
-
message_id: inFlight.evidentMessageId
|
|
2265
|
-
});
|
|
2266
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2267
|
-
return;
|
|
2268
|
-
}
|
|
2269
|
-
this.log({
|
|
2270
|
-
level: "error",
|
|
2271
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2272
|
-
conversation_id: conv.id,
|
|
2273
|
-
message_id: inFlight.evidentMessageId
|
|
2274
|
-
});
|
|
2275
|
-
return;
|
|
2276
|
-
}
|
|
2277
|
-
inFlight.done = true;
|
|
2278
|
-
}
|
|
2279
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3587
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
2280
3588
|
return;
|
|
2281
3589
|
}
|
|
2282
3590
|
if (state === "failed") {
|
|
3591
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2283
3592
|
if (!inFlight.done) {
|
|
2284
3593
|
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2285
3594
|
this.log({
|
|
@@ -2288,13 +3597,14 @@ var ChannelDriver = class {
|
|
|
2288
3597
|
conversation_id: conv.id,
|
|
2289
3598
|
message_id: inFlight.evidentMessageId
|
|
2290
3599
|
});
|
|
3600
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
2291
3601
|
try {
|
|
2292
|
-
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
3602
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2, usage);
|
|
2293
3603
|
} catch (err) {
|
|
2294
3604
|
if (err instanceof ChannelAuthError) throw err;
|
|
2295
3605
|
if (err instanceof ChannelTerminalError) {
|
|
2296
3606
|
this.log({
|
|
2297
|
-
level: "
|
|
3607
|
+
level: "warn",
|
|
2298
3608
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2299
3609
|
conversation_id: conv.id,
|
|
2300
3610
|
message_id: inFlight.evidentMessageId
|
|
@@ -2304,7 +3614,7 @@ var ChannelDriver = class {
|
|
|
2304
3614
|
}
|
|
2305
3615
|
if (this.now() >= inFlight.deadline) {
|
|
2306
3616
|
this.log({
|
|
2307
|
-
level: "
|
|
3617
|
+
level: "warn",
|
|
2308
3618
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2309
3619
|
conversation_id: conv.id,
|
|
2310
3620
|
message_id: inFlight.evidentMessageId
|
|
@@ -2313,7 +3623,7 @@ var ChannelDriver = class {
|
|
|
2313
3623
|
return;
|
|
2314
3624
|
}
|
|
2315
3625
|
this.log({
|
|
2316
|
-
level: "
|
|
3626
|
+
level: "warn",
|
|
2317
3627
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2318
3628
|
conversation_id: conv.id,
|
|
2319
3629
|
message_id: inFlight.evidentMessageId
|
|
@@ -2333,9 +3643,103 @@ var ChannelDriver = class {
|
|
|
2333
3643
|
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2334
3644
|
});
|
|
2335
3645
|
}
|
|
2336
|
-
|
|
3646
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
3647
|
+
const pinnedNow = activelyRunning && isPreamblePinnedRunning(messages, inFlight.opencodeMessageId);
|
|
3648
|
+
const snapshotReadable = messages != null && messages.length > 0;
|
|
3649
|
+
if (!pinnedNow) {
|
|
3650
|
+
if (snapshotReadable) {
|
|
3651
|
+
inFlight.b2PinnedSinceMs = 0;
|
|
3652
|
+
inFlight.b2LastDescendantCheckMs = 0;
|
|
3653
|
+
inFlight.b2AbandonedSignalled = false;
|
|
3654
|
+
}
|
|
3655
|
+
} else {
|
|
3656
|
+
if (inFlight.b2AbandonedSignalled) {
|
|
3657
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3658
|
+
return;
|
|
3659
|
+
}
|
|
3660
|
+
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
3661
|
+
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
3662
|
+
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
3663
|
+
inFlight.b2LastDescendantCheckMs = this.now();
|
|
3664
|
+
const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
|
|
3665
|
+
if (isB2AbandonmentConfirmed({
|
|
3666
|
+
pinnedForMs,
|
|
3667
|
+
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
3668
|
+
descendantOngoing
|
|
3669
|
+
})) {
|
|
3670
|
+
inFlight.b2AbandonedSignalled = true;
|
|
3671
|
+
this.log({
|
|
3672
|
+
level: "warn",
|
|
3673
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
3674
|
+
conversation_id: conv.id,
|
|
3675
|
+
message_id: id
|
|
3676
|
+
});
|
|
3677
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
3678
|
+
watched_for_ms: pinnedForMs
|
|
3679
|
+
});
|
|
3680
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3681
|
+
return;
|
|
3682
|
+
}
|
|
3683
|
+
}
|
|
3684
|
+
}
|
|
3685
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2337
3686
|
this.log({
|
|
2338
|
-
level: "
|
|
3687
|
+
level: "warn",
|
|
3688
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
3689
|
+
conversation_id: conv.id,
|
|
3690
|
+
message_id: inFlight.evidentMessageId
|
|
3691
|
+
});
|
|
3692
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
3693
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
3694
|
+
});
|
|
3695
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3696
|
+
return;
|
|
3697
|
+
}
|
|
3698
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
3699
|
+
inFlight.aliveInFlight = true;
|
|
3700
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
3701
|
+
inFlight.aliveInFlight = false;
|
|
3702
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
3703
|
+
});
|
|
3704
|
+
if (!inFlight.titleSynced && !inFlight.titleSyncInFlight) {
|
|
3705
|
+
inFlight.titleSyncInFlight = true;
|
|
3706
|
+
void this.resolveSessionTitle(sessionId, conv.id).then(async (title) => {
|
|
3707
|
+
if (!title) {
|
|
3708
|
+
inFlight.titleSyncInFlight = false;
|
|
3709
|
+
return;
|
|
3710
|
+
}
|
|
3711
|
+
const ok = await this.patchConversationTitle(conv.id, title);
|
|
3712
|
+
inFlight.titleSyncInFlight = false;
|
|
3713
|
+
if (ok) inFlight.titleSynced = true;
|
|
3714
|
+
});
|
|
3715
|
+
}
|
|
3716
|
+
}
|
|
3717
|
+
if (awaitingHuman) {
|
|
3718
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
3719
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
3720
|
+
inFlight.awaitingHumanLatched = true;
|
|
3721
|
+
}
|
|
3722
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
3723
|
+
inFlight.pausedInFlight = true;
|
|
3724
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
3725
|
+
inFlight.pausedInFlight = false;
|
|
3726
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
3727
|
+
});
|
|
3728
|
+
}
|
|
3729
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
3730
|
+
inFlight.awaitingHumanLatched = false;
|
|
3731
|
+
inFlight.pausedOnQuestion = false;
|
|
3732
|
+
inFlight.pausedOnPermission = false;
|
|
3733
|
+
inFlight.pausedClearConfirmed = false;
|
|
3734
|
+
}
|
|
3735
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
3736
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
3737
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
3738
|
+
);
|
|
3739
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
3740
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
3741
|
+
this.log({
|
|
3742
|
+
level: "debug",
|
|
2339
3743
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2340
3744
|
conversation_id: conv.id,
|
|
2341
3745
|
message_id: inFlight.evidentMessageId
|
|
@@ -2346,9 +3750,71 @@ var ChannelDriver = class {
|
|
|
2346
3750
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2347
3751
|
}
|
|
2348
3752
|
}
|
|
2349
|
-
|
|
3753
|
+
/**
|
|
3754
|
+
* Settle a message whose run-state has resolved `'done'` — extracted verbatim
|
|
3755
|
+
* (pure refactor, no behavior change) from `serviceInFlightMessage`'s former
|
|
3756
|
+
* inline `state === 'done'` branch body, so a SECOND caller (the #721
|
|
3757
|
+
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
3758
|
+
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
3759
|
+
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
3760
|
+
* and risking the two copies silently drifting apart.
|
|
3761
|
+
*/
|
|
3762
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
3763
|
+
const conv = watcher.conv;
|
|
3764
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
3765
|
+
if (!inFlight.done) {
|
|
3766
|
+
this.log({
|
|
3767
|
+
level: "info",
|
|
3768
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
3769
|
+
conversation_id: conv.id,
|
|
3770
|
+
message_id: inFlight.evidentMessageId
|
|
3771
|
+
});
|
|
3772
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
3773
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3774
|
+
try {
|
|
3775
|
+
await this.markDone(
|
|
3776
|
+
conv.id,
|
|
3777
|
+
inFlight.evidentMessageId,
|
|
3778
|
+
sessionId,
|
|
3779
|
+
inFlight.opencodeMessageId,
|
|
3780
|
+
title,
|
|
3781
|
+
usage
|
|
3782
|
+
);
|
|
3783
|
+
} catch (err) {
|
|
3784
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3785
|
+
if (err instanceof ChannelTerminalError) {
|
|
3786
|
+
this.log({
|
|
3787
|
+
level: "warn",
|
|
3788
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
3789
|
+
conversation_id: conv.id,
|
|
3790
|
+
message_id: inFlight.evidentMessageId
|
|
3791
|
+
});
|
|
3792
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3793
|
+
return;
|
|
3794
|
+
}
|
|
3795
|
+
if (this.now() >= inFlight.deadline) {
|
|
3796
|
+
this.log({
|
|
3797
|
+
level: "warn",
|
|
3798
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
3799
|
+
conversation_id: conv.id,
|
|
3800
|
+
message_id: inFlight.evidentMessageId
|
|
3801
|
+
});
|
|
3802
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3803
|
+
return;
|
|
3804
|
+
}
|
|
3805
|
+
this.log({
|
|
3806
|
+
level: "warn",
|
|
3807
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
3808
|
+
conversation_id: conv.id,
|
|
3809
|
+
message_id: inFlight.evidentMessageId
|
|
3810
|
+
});
|
|
3811
|
+
return;
|
|
3812
|
+
}
|
|
3813
|
+
inFlight.done = true;
|
|
3814
|
+
}
|
|
3815
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3816
|
+
}
|
|
2350
3817
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2351
|
-
// -------------------------------------------------------------------------
|
|
2352
3818
|
/**
|
|
2353
3819
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2354
3820
|
*
|
|
@@ -2365,15 +3831,20 @@ var ChannelDriver = class {
|
|
|
2365
3831
|
*/
|
|
2366
3832
|
async readoptProcessing() {
|
|
2367
3833
|
const rows = await this.getProcessingMessages();
|
|
2368
|
-
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
3834
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
|
|
2369
3835
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2370
|
-
for (const id of [
|
|
3836
|
+
for (const id of [
|
|
3837
|
+
...this.dontRedispatch,
|
|
3838
|
+
...this.doneUndeliverable,
|
|
3839
|
+
...this.readoptPollUnresolvedSignalled
|
|
3840
|
+
]) {
|
|
2371
3841
|
if (!stillProcessing.has(id)) {
|
|
2372
3842
|
const cleared = this.dontRedispatch.delete(id);
|
|
2373
3843
|
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
3844
|
+
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2374
3845
|
if (cleared || clearedUndeliverable) {
|
|
2375
3846
|
this.log({
|
|
2376
|
-
level: "
|
|
3847
|
+
level: "debug",
|
|
2377
3848
|
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2378
3849
|
message_id: id
|
|
2379
3850
|
});
|
|
@@ -2386,7 +3857,7 @@ var ChannelDriver = class {
|
|
|
2386
3857
|
for (const row of rows) {
|
|
2387
3858
|
if (!row.opencode_session_id) {
|
|
2388
3859
|
this.log({
|
|
2389
|
-
level: "
|
|
3860
|
+
level: "warn",
|
|
2390
3861
|
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2391
3862
|
conversation_id: row.conversation_id,
|
|
2392
3863
|
message_id: row.id
|
|
@@ -2403,7 +3874,7 @@ var ChannelDriver = class {
|
|
|
2403
3874
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2404
3875
|
if (!res.ok) {
|
|
2405
3876
|
this.log({
|
|
2406
|
-
level: "
|
|
3877
|
+
level: "warn",
|
|
2407
3878
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2408
3879
|
});
|
|
2409
3880
|
continue;
|
|
@@ -2411,7 +3882,7 @@ var ChannelDriver = class {
|
|
|
2411
3882
|
const body = await res.json();
|
|
2412
3883
|
if (!Array.isArray(body)) {
|
|
2413
3884
|
this.log({
|
|
2414
|
-
level: "
|
|
3885
|
+
level: "warn",
|
|
2415
3886
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2416
3887
|
});
|
|
2417
3888
|
continue;
|
|
@@ -2419,13 +3890,15 @@ var ChannelDriver = class {
|
|
|
2419
3890
|
messages = body;
|
|
2420
3891
|
} catch (err) {
|
|
2421
3892
|
this.log({
|
|
2422
|
-
level: "
|
|
3893
|
+
level: "warn",
|
|
2423
3894
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2424
3895
|
});
|
|
2425
3896
|
continue;
|
|
2426
3897
|
}
|
|
3898
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
3899
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2427
3900
|
for (const row of sessionRows) {
|
|
2428
|
-
await this.readoptOne(sessionId, row, messages);
|
|
3901
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2429
3902
|
}
|
|
2430
3903
|
}
|
|
2431
3904
|
}
|
|
@@ -2447,10 +3920,10 @@ var ChannelDriver = class {
|
|
|
2447
3920
|
*
|
|
2448
3921
|
* Only `ChannelAuthError` propagates.
|
|
2449
3922
|
*/
|
|
2450
|
-
async readoptOne(sessionId, row, messages) {
|
|
3923
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2451
3924
|
if (this.isTracked(sessionId, row.id)) {
|
|
2452
3925
|
this.log({
|
|
2453
|
-
level: "
|
|
3926
|
+
level: "debug",
|
|
2454
3927
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2455
3928
|
conversation_id: row.conversation_id,
|
|
2456
3929
|
message_id: row.id
|
|
@@ -2462,7 +3935,7 @@ var ChannelDriver = class {
|
|
|
2462
3935
|
if (state === "done") {
|
|
2463
3936
|
if (this.doneUndeliverable.has(row.id)) {
|
|
2464
3937
|
this.log({
|
|
2465
|
-
level: "
|
|
3938
|
+
level: "debug",
|
|
2466
3939
|
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2467
3940
|
conversation_id: row.conversation_id,
|
|
2468
3941
|
message_id: row.id
|
|
@@ -2476,21 +3949,24 @@ var ChannelDriver = class {
|
|
|
2476
3949
|
message_id: row.id
|
|
2477
3950
|
});
|
|
2478
3951
|
try {
|
|
2479
|
-
await this.
|
|
3952
|
+
const title = await this.resolveSessionTitle(sessionId, row.conversation_id);
|
|
3953
|
+
const usage = messageUsage(messages, ocId ?? "");
|
|
3954
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId, title, usage);
|
|
2480
3955
|
} catch (err) {
|
|
2481
3956
|
if (err instanceof ChannelAuthError) throw err;
|
|
2482
3957
|
if (err instanceof ChannelTerminalError) {
|
|
2483
3958
|
this.doneUndeliverable.add(row.id);
|
|
2484
3959
|
this.log({
|
|
2485
|
-
level: "
|
|
3960
|
+
level: "warn",
|
|
2486
3961
|
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2487
3962
|
conversation_id: row.conversation_id,
|
|
2488
3963
|
message_id: row.id
|
|
2489
3964
|
});
|
|
3965
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2490
3966
|
return;
|
|
2491
3967
|
}
|
|
2492
3968
|
this.log({
|
|
2493
|
-
level: "
|
|
3969
|
+
level: "warn",
|
|
2494
3970
|
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2495
3971
|
conversation_id: row.conversation_id,
|
|
2496
3972
|
message_id: row.id
|
|
@@ -2498,10 +3974,12 @@ var ChannelDriver = class {
|
|
|
2498
3974
|
return;
|
|
2499
3975
|
}
|
|
2500
3976
|
this.dontRedispatch.delete(row.id);
|
|
3977
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
2501
3978
|
return;
|
|
2502
3979
|
}
|
|
2503
3980
|
if (state === "failed") {
|
|
2504
3981
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3982
|
+
const usage = messageUsage(messages, ocId ?? "");
|
|
2505
3983
|
this.log({
|
|
2506
3984
|
level: "error",
|
|
2507
3985
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -2509,38 +3987,105 @@ var ChannelDriver = class {
|
|
|
2509
3987
|
message_id: row.id
|
|
2510
3988
|
});
|
|
2511
3989
|
try {
|
|
2512
|
-
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
3990
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage);
|
|
2513
3991
|
} catch (err) {
|
|
2514
3992
|
if (err instanceof ChannelAuthError) throw err;
|
|
2515
3993
|
if (err instanceof ChannelTerminalError) {
|
|
2516
3994
|
this.doneUndeliverable.add(row.id);
|
|
2517
3995
|
this.log({
|
|
2518
|
-
level: "
|
|
3996
|
+
level: "warn",
|
|
2519
3997
|
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2520
3998
|
conversation_id: row.conversation_id,
|
|
2521
3999
|
message_id: row.id
|
|
2522
4000
|
});
|
|
4001
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2523
4002
|
return;
|
|
2524
4003
|
}
|
|
2525
4004
|
this.log({
|
|
2526
|
-
level: "
|
|
2527
|
-
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4005
|
+
level: "warn",
|
|
4006
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4007
|
+
conversation_id: row.conversation_id,
|
|
4008
|
+
message_id: row.id
|
|
4009
|
+
});
|
|
4010
|
+
return;
|
|
4011
|
+
}
|
|
4012
|
+
this.dontRedispatch.delete(row.id);
|
|
4013
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
4014
|
+
return;
|
|
4015
|
+
}
|
|
4016
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
4017
|
+
this.log({
|
|
4018
|
+
level: "debug",
|
|
4019
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
4020
|
+
conversation_id: row.conversation_id,
|
|
4021
|
+
message_id: row.id
|
|
4022
|
+
});
|
|
4023
|
+
return;
|
|
4024
|
+
}
|
|
4025
|
+
let statusReadableOngoing = null;
|
|
4026
|
+
if (state === "running" && ocId) {
|
|
4027
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
4028
|
+
const shape = this.replyCompletionShape(reply);
|
|
4029
|
+
const ongoing = sessionOngoing;
|
|
4030
|
+
statusReadableOngoing = ongoing;
|
|
4031
|
+
if (ongoing === false) {
|
|
4032
|
+
this.log({
|
|
4033
|
+
level: "info",
|
|
4034
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
4035
|
+
conversation_id: row.conversation_id,
|
|
4036
|
+
message_id: row.id
|
|
4037
|
+
});
|
|
4038
|
+
await this.forceReadoptRun(sessionId, row);
|
|
4039
|
+
return;
|
|
4040
|
+
}
|
|
4041
|
+
if (ongoing === true) {
|
|
4042
|
+
this.log({
|
|
4043
|
+
level: "debug",
|
|
4044
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
4045
|
+
conversation_id: row.conversation_id,
|
|
4046
|
+
message_id: row.id
|
|
4047
|
+
});
|
|
4048
|
+
} else {
|
|
4049
|
+
if (shape === "b1") {
|
|
4050
|
+
this.log({
|
|
4051
|
+
level: "debug",
|
|
4052
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
4053
|
+
conversation_id: row.conversation_id,
|
|
4054
|
+
message_id: row.id
|
|
4055
|
+
});
|
|
4056
|
+
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
4057
|
+
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
4058
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
4059
|
+
}
|
|
4060
|
+
return;
|
|
4061
|
+
}
|
|
4062
|
+
this.log({
|
|
4063
|
+
level: "debug",
|
|
4064
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
4065
|
+
conversation_id: row.conversation_id,
|
|
4066
|
+
message_id: row.id
|
|
4067
|
+
});
|
|
4068
|
+
}
|
|
4069
|
+
}
|
|
4070
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
4071
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
4072
|
+
if (descendantAlive === true) {
|
|
4073
|
+
this.log({
|
|
4074
|
+
level: "debug",
|
|
4075
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
4076
|
+
conversation_id: row.conversation_id,
|
|
4077
|
+
message_id: row.id
|
|
4078
|
+
});
|
|
4079
|
+
} else {
|
|
4080
|
+
this.log({
|
|
4081
|
+
level: "info",
|
|
4082
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2528
4083
|
conversation_id: row.conversation_id,
|
|
2529
4084
|
message_id: row.id
|
|
2530
4085
|
});
|
|
4086
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2531
4087
|
return;
|
|
2532
4088
|
}
|
|
2533
|
-
this.dontRedispatch.delete(row.id);
|
|
2534
|
-
return;
|
|
2535
|
-
}
|
|
2536
|
-
if (this.dontRedispatch.has(row.id)) {
|
|
2537
|
-
this.log({
|
|
2538
|
-
level: "info",
|
|
2539
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2540
|
-
conversation_id: row.conversation_id,
|
|
2541
|
-
message_id: row.id
|
|
2542
|
-
});
|
|
2543
|
-
return;
|
|
2544
4089
|
}
|
|
2545
4090
|
if ((state === "running" || state === "queued") && ocId) {
|
|
2546
4091
|
const conv = this.convForRow(sessionId, row);
|
|
@@ -2550,11 +4095,12 @@ var ChannelDriver = class {
|
|
|
2550
4095
|
this.readopted.add(row.id);
|
|
2551
4096
|
this.ensureWatcherRunning(sessionId);
|
|
2552
4097
|
this.log({
|
|
2553
|
-
level: "
|
|
4098
|
+
level: "debug",
|
|
2554
4099
|
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2555
4100
|
conversation_id: row.conversation_id,
|
|
2556
4101
|
message_id: row.id
|
|
2557
4102
|
});
|
|
4103
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
2558
4104
|
return;
|
|
2559
4105
|
}
|
|
2560
4106
|
await this.forceReadoptRun(sessionId, row);
|
|
@@ -2583,7 +4129,7 @@ var ChannelDriver = class {
|
|
|
2583
4129
|
async forceReadoptRun(sessionId, row) {
|
|
2584
4130
|
if (this.stopped) {
|
|
2585
4131
|
this.log({
|
|
2586
|
-
level: "
|
|
4132
|
+
level: "debug",
|
|
2587
4133
|
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
2588
4134
|
conversation_id: row.conversation_id,
|
|
2589
4135
|
message_id: row.id
|
|
@@ -2592,7 +4138,7 @@ var ChannelDriver = class {
|
|
|
2592
4138
|
}
|
|
2593
4139
|
if (this.awaitingReadopt.has(row.id)) {
|
|
2594
4140
|
this.log({
|
|
2595
|
-
level: "
|
|
4141
|
+
level: "debug",
|
|
2596
4142
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
2597
4143
|
conversation_id: row.conversation_id,
|
|
2598
4144
|
message_id: row.id
|
|
@@ -2602,11 +4148,12 @@ var ChannelDriver = class {
|
|
|
2602
4148
|
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2603
4149
|
this.dontRedispatch.add(row.id);
|
|
2604
4150
|
this.log({
|
|
2605
|
-
level: "
|
|
4151
|
+
level: "debug",
|
|
2606
4152
|
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2607
4153
|
conversation_id: row.conversation_id,
|
|
2608
4154
|
message_id: row.id
|
|
2609
4155
|
});
|
|
4156
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
2610
4157
|
return;
|
|
2611
4158
|
}
|
|
2612
4159
|
const options = {
|
|
@@ -2620,40 +4167,44 @@ var ChannelDriver = class {
|
|
|
2620
4167
|
message_id: row.id
|
|
2621
4168
|
});
|
|
2622
4169
|
this.awaitingReadopt.add(row.id);
|
|
4170
|
+
const readoptConv = this.convForRow(sessionId, row);
|
|
4171
|
+
const readoptMessage = this.queuedMessageForRow(row);
|
|
4172
|
+
const sendAttachments = this.buildSendAttachments(readoptConv, readoptMessage);
|
|
2623
4173
|
let ocId;
|
|
2624
4174
|
try {
|
|
2625
4175
|
ocId = await this.dispatchLocked(
|
|
2626
4176
|
sessionId,
|
|
2627
|
-
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
4177
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options, sendAttachments)
|
|
2628
4178
|
);
|
|
2629
4179
|
} catch (err) {
|
|
2630
4180
|
this.awaitingReadopt.delete(row.id);
|
|
2631
4181
|
if (err instanceof ChannelAuthError) throw err;
|
|
2632
4182
|
this.log({
|
|
2633
|
-
level: "
|
|
4183
|
+
level: "warn",
|
|
2634
4184
|
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2635
4185
|
conversation_id: row.conversation_id,
|
|
2636
4186
|
message_id: row.id
|
|
2637
4187
|
});
|
|
4188
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2638
4189
|
return;
|
|
2639
4190
|
}
|
|
2640
4191
|
if (ocId === null) {
|
|
2641
4192
|
this.awaitingReadopt.delete(row.id);
|
|
2642
4193
|
this.log({
|
|
2643
|
-
level: "
|
|
4194
|
+
level: "warn",
|
|
2644
4195
|
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
2645
4196
|
conversation_id: row.conversation_id,
|
|
2646
4197
|
message_id: row.id
|
|
2647
4198
|
});
|
|
4199
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2648
4200
|
return;
|
|
2649
4201
|
}
|
|
2650
|
-
|
|
2651
|
-
const message = this.queuedMessageForRow(row);
|
|
2652
|
-
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
4202
|
+
this.registerReadopted(readoptConv, sessionId, readoptMessage, ocId, this.processedAtMs(row));
|
|
2653
4203
|
this.dispatched.add(row.id);
|
|
2654
4204
|
this.readopted.add(row.id);
|
|
2655
4205
|
this.awaitingReadopt.delete(row.id);
|
|
2656
4206
|
this.ensureWatcherRunning(sessionId);
|
|
4207
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
2657
4208
|
}
|
|
2658
4209
|
/**
|
|
2659
4210
|
* True if `evidentMessageId` is already being driven — either in the
|
|
@@ -2702,7 +4253,8 @@ var ChannelDriver = class {
|
|
|
2702
4253
|
opencode_agent: row.opencode_agent,
|
|
2703
4254
|
opencode_model: row.opencode_model,
|
|
2704
4255
|
source_message_id: row.source_message_id,
|
|
2705
|
-
slack_user_id: row.slack_user_id
|
|
4256
|
+
slack_user_id: row.slack_user_id,
|
|
4257
|
+
attachments: row.attachments ?? null
|
|
2706
4258
|
};
|
|
2707
4259
|
}
|
|
2708
4260
|
/**
|
|
@@ -2723,7 +4275,7 @@ var ChannelDriver = class {
|
|
|
2723
4275
|
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2724
4276
|
this.dontRedispatch.add(evidentMessageId);
|
|
2725
4277
|
this.log({
|
|
2726
|
-
level: "
|
|
4278
|
+
level: "debug",
|
|
2727
4279
|
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2728
4280
|
conversation_id: watcher.conv.id,
|
|
2729
4281
|
message_id: evidentMessageId
|
|
@@ -2745,21 +4297,41 @@ var ChannelDriver = class {
|
|
|
2745
4297
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2746
4298
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2747
4299
|
* oldest running message.
|
|
4300
|
+
*
|
|
4301
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
4302
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
4303
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
4304
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
4305
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
4306
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
4307
|
+
* even after it was already surfaced to the channel.
|
|
2748
4308
|
*/
|
|
2749
4309
|
async pollInteractions(sessionId, watcher, messages) {
|
|
4310
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
4311
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
4312
|
+
let questionsPolledOk = true;
|
|
4313
|
+
let permissionsPolledOk = true;
|
|
2750
4314
|
let questions = [];
|
|
2751
4315
|
try {
|
|
2752
4316
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2753
4317
|
if (res.ok) {
|
|
2754
4318
|
const body = await res.json();
|
|
2755
|
-
|
|
4319
|
+
if (Array.isArray(body)) {
|
|
4320
|
+
questions = body;
|
|
4321
|
+
} else {
|
|
4322
|
+
questionsPolledOk = false;
|
|
4323
|
+
}
|
|
4324
|
+
} else {
|
|
4325
|
+
questionsPolledOk = false;
|
|
2756
4326
|
}
|
|
2757
4327
|
} catch {
|
|
4328
|
+
questionsPolledOk = false;
|
|
2758
4329
|
}
|
|
2759
4330
|
for (const q of questions) {
|
|
2760
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2761
4331
|
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2762
4332
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
4333
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
4334
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2763
4335
|
const reported = await this.reportInteraction(
|
|
2764
4336
|
watcher.conv.id,
|
|
2765
4337
|
"question",
|
|
@@ -2773,14 +4345,22 @@ var ChannelDriver = class {
|
|
|
2773
4345
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2774
4346
|
if (res.ok) {
|
|
2775
4347
|
const body = await res.json();
|
|
2776
|
-
|
|
4348
|
+
if (Array.isArray(body)) {
|
|
4349
|
+
permissions = body;
|
|
4350
|
+
} else {
|
|
4351
|
+
permissionsPolledOk = false;
|
|
4352
|
+
}
|
|
4353
|
+
} else {
|
|
4354
|
+
permissionsPolledOk = false;
|
|
2777
4355
|
}
|
|
2778
4356
|
} catch {
|
|
4357
|
+
permissionsPolledOk = false;
|
|
2779
4358
|
}
|
|
2780
4359
|
for (const p of permissions) {
|
|
2781
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2782
4360
|
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2783
4361
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
4362
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
4363
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2784
4364
|
const reported = await this.reportInteraction(
|
|
2785
4365
|
watcher.conv.id,
|
|
2786
4366
|
"permission",
|
|
@@ -2789,6 +4369,7 @@ var ChannelDriver = class {
|
|
|
2789
4369
|
);
|
|
2790
4370
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2791
4371
|
}
|
|
4372
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
2792
4373
|
}
|
|
2793
4374
|
/**
|
|
2794
4375
|
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
@@ -2812,6 +4393,47 @@ var ChannelDriver = class {
|
|
|
2812
4393
|
}
|
|
2813
4394
|
return false;
|
|
2814
4395
|
}
|
|
4396
|
+
/**
|
|
4397
|
+
* Tri-state variant of the upward parentID membership walk (#721), used ONLY
|
|
4398
|
+
* by `isAnyDescendantSessionOngoing`. Walks the SAME cached
|
|
4399
|
+
* `resolveSessionParent` chain `sessionBelongsTo` uses above, but — unlike
|
|
4400
|
+
* `sessionBelongsTo`, which deliberately collapses "confirmed not a
|
|
4401
|
+
* descendant" and "the walk's fetch failed" into the same `false` (safe for
|
|
4402
|
+
* its OTHER callers: interaction attribution and the recovery-path
|
|
4403
|
+
* `isAnyDescendantSessionAlive`, both of which just retry next tick with no
|
|
4404
|
+
* safety consequence either way) — this variant keeps those two outcomes
|
|
4405
|
+
* SEPARATE, because `isAnyDescendantSessionOngoing`'s caller
|
|
4406
|
+
* (`isB2AbandonmentConfirmed`) must never treat "couldn't tell" as "confirmed
|
|
4407
|
+
* not ongoing".
|
|
4408
|
+
*
|
|
4409
|
+
* Return contract:
|
|
4410
|
+
* - `true` → the walk reached `rootSessionId` — `sessionId` IS a descendant.
|
|
4411
|
+
* - `false` → the walk reached a definitive, parent-less root session
|
|
4412
|
+
* WITHOUT ever matching `rootSessionId` — `sessionId` is
|
|
4413
|
+
* CONFIRMED NOT a descendant of it.
|
|
4414
|
+
* - `null` → INDETERMINATE: a `GET /session/:id` fetch failed partway
|
|
4415
|
+
* through the walk (`resolveSessionParent` returned `undefined`),
|
|
4416
|
+
* or the depth cap (32) was hit without a definitive answer (a
|
|
4417
|
+
* pathological/cyclic chain proves nothing either way). NEVER
|
|
4418
|
+
* treat this the same as `false` — see `sessionBelongsTo`'s own
|
|
4419
|
+
* doc comment above for why that collapse is safe THERE but not
|
|
4420
|
+
* here.
|
|
4421
|
+
*
|
|
4422
|
+
* `sessionBelongsTo` itself is UNCHANGED — this is an additive helper scoped
|
|
4423
|
+
* to the live-path descendant check, not a modification of shared code used
|
|
4424
|
+
* by interaction attribution or the recovery path.
|
|
4425
|
+
*/
|
|
4426
|
+
async resolveSessionMembership(sessionId, rootSessionId) {
|
|
4427
|
+
let current = sessionId;
|
|
4428
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
4429
|
+
if (current === rootSessionId) return true;
|
|
4430
|
+
const parent = await this.resolveSessionParent(current);
|
|
4431
|
+
if (parent === void 0) return null;
|
|
4432
|
+
if (parent === null) return false;
|
|
4433
|
+
current = parent;
|
|
4434
|
+
}
|
|
4435
|
+
return null;
|
|
4436
|
+
}
|
|
2815
4437
|
/**
|
|
2816
4438
|
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
2817
4439
|
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
@@ -2834,6 +4456,271 @@ var ChannelDriver = class {
|
|
|
2834
4456
|
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
2835
4457
|
return parent;
|
|
2836
4458
|
}
|
|
4459
|
+
/**
|
|
4460
|
+
* OpenCode's synchronous default session title (e.g.
|
|
4461
|
+
* `"New session - 1737800000000"`), assigned immediately when a session is
|
|
4462
|
+
* created — before OpenCode's async LLM-based auto-titling later renames it
|
|
4463
|
+
* mid-turn (#549). Matched by this literal, case-sensitive prefix only; the
|
|
4464
|
+
* timestamp suffix's exact format is deliberately NOT matched, since the prefix
|
|
4465
|
+
* alone is the stable, cheap signal and over-anchoring on the timestamp
|
|
4466
|
+
* representation risks silently breaking if OpenCode ever changes it. Accepted
|
|
4467
|
+
* trade-off: a genuine LLM-assigned title that happens to literally start with
|
|
4468
|
+
* this prefix would also fail to latch (see `resolveSessionTitle`) —
|
|
4469
|
+
* vanishingly unlikely in practice, and deliberately not engineered around.
|
|
4470
|
+
*/
|
|
4471
|
+
static OPENCODE_DEFAULT_TITLE_PREFIX = /^New session - /;
|
|
4472
|
+
/**
|
|
4473
|
+
* Resolve (and cache in `sessionTitles`) the OpenCode session TITLE (#310) so the
|
|
4474
|
+
* status PATCH can carry it into the "Live sessions" list. Driver-level cache so
|
|
4475
|
+
* BOTH the watcher completion path and the restart-recovery re-adopt path (which
|
|
4476
|
+
* has no watcher) can use it. `conversationId` is passed only for log context.
|
|
4477
|
+
* Best-effort:
|
|
4478
|
+
* - a resolved NON-EMPTY title that does NOT match
|
|
4479
|
+
* `OPENCODE_DEFAULT_TITLE_PREFIX` is cached and terminal (a real session name
|
|
4480
|
+
* won't later un-name), so we do NOT re-GET `/session/:id` every tick;
|
|
4481
|
+
* - while the title is still absent, empty, or matches the OpenCode
|
|
4482
|
+
* placeholder prefix (#549) we do NOT latch it — OpenCode names sessions
|
|
4483
|
+
* asynchronously mid-turn, so an early call (e.g. at `processing`) must leave
|
|
4484
|
+
* the cache unresolved and re-fetch on the next need so a later call (e.g. at
|
|
4485
|
+
* `done`) picks up the name assigned in the meantime. Such a call returns
|
|
4486
|
+
* `null` (omit the title on THIS PATCH) without caching. If a session is
|
|
4487
|
+
* never renamed, the title is omitted forever rather than ever persisting
|
|
4488
|
+
* the placeholder as a last resort;
|
|
4489
|
+
* - a failed request likewise leaves the cache unresolved (retry next need)
|
|
4490
|
+
* and returns `null` — it must NEVER throw or block completion.
|
|
4491
|
+
* A failure is logged with agent/session context (no silent catch).
|
|
4492
|
+
*/
|
|
4493
|
+
async resolveSessionTitle(sessionId, conversationId) {
|
|
4494
|
+
const cached = this.sessionTitles.get(sessionId);
|
|
4495
|
+
if (cached != null) return cached;
|
|
4496
|
+
try {
|
|
4497
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
4498
|
+
if (res.ok) {
|
|
4499
|
+
const body = await res.json();
|
|
4500
|
+
const title = body && typeof body.title === "string" ? body.title.trim() : "";
|
|
4501
|
+
if (title.length > 0 && !_ChannelDriver.OPENCODE_DEFAULT_TITLE_PREFIX.test(title)) {
|
|
4502
|
+
this.sessionTitles.set(sessionId, title);
|
|
4503
|
+
return title;
|
|
4504
|
+
}
|
|
4505
|
+
return null;
|
|
4506
|
+
}
|
|
4507
|
+
this.log({
|
|
4508
|
+
level: "debug",
|
|
4509
|
+
message: `Session title fetch for session ${sessionId.slice(0, 8)} (agent ${this.agentId.slice(0, 8)}) returned HTTP ${res.status} \u2014 omitting title`,
|
|
4510
|
+
conversation_id: conversationId
|
|
4511
|
+
});
|
|
4512
|
+
} catch (err) {
|
|
4513
|
+
this.log({
|
|
4514
|
+
level: "debug",
|
|
4515
|
+
message: `Best-effort session title fetch failed for session ${sessionId.slice(0, 8)} (agent ${this.agentId.slice(0, 8)}) \u2014 omitting title: ${err instanceof Error ? err.message : String(err)}`,
|
|
4516
|
+
conversation_id: conversationId
|
|
4517
|
+
});
|
|
4518
|
+
}
|
|
4519
|
+
return null;
|
|
4520
|
+
}
|
|
4521
|
+
/**
|
|
4522
|
+
* Best-effort mid-turn title sync (#711 follow-up): PATCH a resolved OpenCode
|
|
4523
|
+
* session title onto the conversation via the PLAIN conversation-update
|
|
4524
|
+
* endpoint (`PATCH /runners/:agentId/conversations/:conversationId`) — NOT the
|
|
4525
|
+
* message-status endpoint `markProcessing`/`markDone` use. Deliberately a
|
|
4526
|
+
* separate, lighter call: it carries no `status`, so it cannot re-trigger the
|
|
4527
|
+
* `processing`/`done` transition side effects (Slack notices, activity-log
|
|
4528
|
+
* rows, delivery jobs) those PATCHes gate on `transitioned` — this call only
|
|
4529
|
+
* ever touches `conversations.title`. That route (`routes/conversations.ts`)
|
|
4530
|
+
* skips a title write matching the stored value, so a redundant call with the
|
|
4531
|
+
* same title is a real no-op — it does not bump `updated_at`, which the
|
|
4532
|
+
* conversation list sorts and paginates on. (Note this is a DIFFERENT guard
|
|
4533
|
+
* from `threads.ts`'s "non-empty AND changed" one, which only covers the
|
|
4534
|
+
* message-status PATCH; the non-empty half is enforced here instead, by
|
|
4535
|
+
* `resolveSessionTitle` never returning an empty/placeholder title.)
|
|
4536
|
+
*
|
|
4537
|
+
* Telemetry-only / never blocks the caller, mirroring `postSignal`: a failure
|
|
4538
|
+
* is logged and the title is simply retried on the next heartbeat tick (the
|
|
4539
|
+
* caller only latches `titleSynced` on `true`).
|
|
4540
|
+
*/
|
|
4541
|
+
async patchConversationTitle(conversationId, title) {
|
|
4542
|
+
try {
|
|
4543
|
+
const res = await this.fetchImpl(
|
|
4544
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/${conversationId}`,
|
|
4545
|
+
{
|
|
4546
|
+
method: "PATCH",
|
|
4547
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
4548
|
+
body: JSON.stringify({ title })
|
|
4549
|
+
}
|
|
4550
|
+
);
|
|
4551
|
+
if (!res.ok) {
|
|
4552
|
+
this.log({
|
|
4553
|
+
level: "debug",
|
|
4554
|
+
message: `Mid-turn title sync PATCH for conversation ${conversationId.slice(0, 8)} returned HTTP ${res.status} (best-effort, will retry next heartbeat)`,
|
|
4555
|
+
conversation_id: conversationId
|
|
4556
|
+
});
|
|
4557
|
+
return false;
|
|
4558
|
+
}
|
|
4559
|
+
return true;
|
|
4560
|
+
} catch (err) {
|
|
4561
|
+
this.log({
|
|
4562
|
+
level: "debug",
|
|
4563
|
+
message: `Best-effort mid-turn title sync PATCH failed for conversation ${conversationId.slice(0, 8)} (will retry next heartbeat): ${err instanceof Error ? err.message : String(err)}`,
|
|
4564
|
+
conversation_id: conversationId
|
|
4565
|
+
});
|
|
4566
|
+
return false;
|
|
4567
|
+
}
|
|
4568
|
+
}
|
|
4569
|
+
/**
|
|
4570
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
4571
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
4572
|
+
*
|
|
4573
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
4574
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
4575
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
4576
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
4577
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
4578
|
+
* provably in flight at the exact moment of recovery.
|
|
4579
|
+
*
|
|
4580
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
4581
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
4582
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
4583
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
4584
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
4585
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
4586
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
4587
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
4588
|
+
*
|
|
4589
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
4590
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
4591
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
4592
|
+
* case), OR no descendant is found at all.
|
|
4593
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
4594
|
+
*
|
|
4595
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
4596
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
4597
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
4598
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
4599
|
+
* true/false/null faithfully.
|
|
4600
|
+
*
|
|
4601
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
4602
|
+
* (already proven by the existing child-session interaction tests, via
|
|
4603
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
4604
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
4605
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
4606
|
+
* `SessionStatus` only.
|
|
4607
|
+
*/
|
|
4608
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
4609
|
+
const sessions = await listSessions(this.port);
|
|
4610
|
+
if (!sessions) {
|
|
4611
|
+
this.log({
|
|
4612
|
+
level: "warn",
|
|
4613
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
4614
|
+
});
|
|
4615
|
+
return null;
|
|
4616
|
+
}
|
|
4617
|
+
for (const candidate of sessions) {
|
|
4618
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
4619
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
4620
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
4621
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
4622
|
+
return true;
|
|
4623
|
+
}
|
|
4624
|
+
}
|
|
4625
|
+
return false;
|
|
4626
|
+
}
|
|
4627
|
+
/**
|
|
4628
|
+
* LIVE-PATH descendant-liveness check (#721): is any descendant (`task`
|
|
4629
|
+
* sub-agent) session under `rootSessionId` currently ONGOING per OpenCode's own
|
|
4630
|
+
* in-memory status map (`isSessionOngoing` — `busy`/`retry`)?
|
|
4631
|
+
*
|
|
4632
|
+
* Deliberately NOT `isAnyDescendantSessionAlive` (the RECOVERY-path
|
|
4633
|
+
* cross-check above): that method judges liveness from the child's OWN
|
|
4634
|
+
* TRANSCRIPT (`isSessionActivelyGenerating`), which is the right (only) option
|
|
4635
|
+
* on the recovery path because a restart WIPES `SessionStatus`. On the LIVE
|
|
4636
|
+
* path the local opencode server IS running, so its in-memory status map is
|
|
4637
|
+
* live and authoritative — and per ADR-0047 §4a ("the child has its own entry
|
|
4638
|
+
* [in the map]"), a `task` descendant's OWN busy/retry entry reflects its
|
|
4639
|
+
* ENTIRE turn (including any tool call it is itself executing), not a
|
|
4640
|
+
* per-message transcript snapshot. This sidesteps the "child's own tool is
|
|
4641
|
+
* executing, between its step's completion and the next generation step"
|
|
4642
|
+
* transcript gap that a transcript-based check would need a second,
|
|
4643
|
+
* sustained-window bound to guard against — it is simply not derived from
|
|
4644
|
+
* message timestamps at all.
|
|
4645
|
+
*
|
|
4646
|
+
* Why not just check `isSessionOngoing(port, rootSessionId)` (the ROOT's own
|
|
4647
|
+
* status, as the recovery path does per §4a)? Because on the LIVE path the
|
|
4648
|
+
* root session can be shared: a SECOND, unrelated user message can land on the
|
|
4649
|
+
* SAME session (issue #721's own root cause) and keep the root `busy` for a
|
|
4650
|
+
* reason that has nothing to do with THIS message's delegation. A `task`
|
|
4651
|
+
* descendant session is spawned for exactly one delegated turn and never
|
|
4652
|
+
* reused, so its OWN status-map entry is unambiguous evidence about that one
|
|
4653
|
+
* delegation — which the root's status is not.
|
|
4654
|
+
*
|
|
4655
|
+
* Why membership is checked via `resolveSessionMembership`, NOT
|
|
4656
|
+
* `sessionBelongsTo`: `sessionBelongsTo` collapses a transient
|
|
4657
|
+
* `GET /session/:id` fetch failure into "not a descendant", which would
|
|
4658
|
+
* silently drop a genuinely-live candidate from consideration on the one
|
|
4659
|
+
* unlucky tick its membership-walk fetch hiccups (#721).
|
|
4660
|
+
* `resolveSessionMembership` keeps that failure mode as a distinct `null`
|
|
4661
|
+
* (indeterminate) so it is folded into THIS method's own `indeterminate` flag
|
|
4662
|
+
* instead.
|
|
4663
|
+
*
|
|
4664
|
+
* Return contract (note the DIFFERENT judge vs. `isAnyDescendantSessionAlive`):
|
|
4665
|
+
* - `true` → some descendant session is `busy`/`retry` (genuinely ongoing).
|
|
4666
|
+
* - `false` → enumeration succeeded, EVERY candidate's MEMBERSHIP was
|
|
4667
|
+
* confirmed either way (`resolveSessionMembership` never
|
|
4668
|
+
* returned `null`), and every CONFIRMED descendant's status read
|
|
4669
|
+
* succeeded and is not ongoing (includes "no descendant session
|
|
4670
|
+
* exists at all" — e.g. a plain, non-`task` tool call).
|
|
4671
|
+
* - `null` → INDETERMINATE: `listSessions` failed, OR at least one
|
|
4672
|
+
* candidate's MEMBERSHIP could not be confirmed
|
|
4673
|
+
* (`resolveSessionMembership` returned `null` — a fetch failure
|
|
4674
|
+
* or pathological chain partway through the parent walk), OR at
|
|
4675
|
+
* least one CONFIRMED descendant's `isSessionOngoing` read
|
|
4676
|
+
* failed — and no OTHER candidate was already confirmed `true`.
|
|
4677
|
+
* The caller MUST NOT treat `null` the same as `false` here
|
|
4678
|
+
* (unlike the recovery cross-check's contract) — see
|
|
4679
|
+
* `isB2AbandonmentConfirmed`.
|
|
4680
|
+
*/
|
|
4681
|
+
async isAnyDescendantSessionOngoing(rootSessionId) {
|
|
4682
|
+
const sessions = await listSessions(this.port);
|
|
4683
|
+
if (!sessions) {
|
|
4684
|
+
this.log({
|
|
4685
|
+
level: "warn",
|
|
4686
|
+
message: `Could not enumerate sessions to check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating descendant liveness as indeterminate`
|
|
4687
|
+
});
|
|
4688
|
+
return null;
|
|
4689
|
+
}
|
|
4690
|
+
let indeterminate = false;
|
|
4691
|
+
for (const candidate of sessions) {
|
|
4692
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
4693
|
+
const membership = await this.resolveSessionMembership(candidate.id, rootSessionId);
|
|
4694
|
+
if (membership === null) {
|
|
4695
|
+
indeterminate = true;
|
|
4696
|
+
continue;
|
|
4697
|
+
}
|
|
4698
|
+
if (membership === false) continue;
|
|
4699
|
+
const ongoing = await isSessionOngoing(this.port, candidate.id);
|
|
4700
|
+
if (ongoing === true) return true;
|
|
4701
|
+
if (ongoing === null) indeterminate = true;
|
|
4702
|
+
}
|
|
4703
|
+
return indeterminate ? null : false;
|
|
4704
|
+
}
|
|
4705
|
+
/**
|
|
4706
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
4707
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
4708
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
4709
|
+
* the aborted-in-flight production bug after a restart.
|
|
4710
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
4711
|
+
* (the sub-agent preamble — #253's shape).
|
|
4712
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
4713
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
4714
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
4715
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
4716
|
+
*/
|
|
4717
|
+
replyCompletionShape(reply) {
|
|
4718
|
+
if (!reply) return "other";
|
|
4719
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
4720
|
+
if (completed == null) return "b1";
|
|
4721
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
4722
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
4723
|
+
}
|
|
2837
4724
|
/**
|
|
2838
4725
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
2839
4726
|
*
|
|
@@ -2884,12 +4771,10 @@ var ChannelDriver = class {
|
|
|
2884
4771
|
}
|
|
2885
4772
|
return inFlight.sort(byOldest)[0];
|
|
2886
4773
|
}
|
|
2887
|
-
// -------------------------------------------------------------------------
|
|
2888
4774
|
// Evident API calls (combinedAuth thread routes)
|
|
2889
|
-
// -------------------------------------------------------------------------
|
|
2890
4775
|
async getPendingConversations() {
|
|
2891
4776
|
const res = await this.fetchImpl(
|
|
2892
|
-
`${this.apiUrl}/
|
|
4777
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/pending`,
|
|
2893
4778
|
{
|
|
2894
4779
|
headers: { Authorization: this.getAuthHeader() }
|
|
2895
4780
|
}
|
|
@@ -2907,7 +4792,7 @@ var ChannelDriver = class {
|
|
|
2907
4792
|
}
|
|
2908
4793
|
async getPendingMessages(conversationId) {
|
|
2909
4794
|
const res = await this.fetchImpl(
|
|
2910
|
-
`${this.apiUrl}/
|
|
4795
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages?status=pending`,
|
|
2911
4796
|
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2912
4797
|
);
|
|
2913
4798
|
this.assertAuth(res, "fetching pending messages");
|
|
@@ -2931,7 +4816,7 @@ var ChannelDriver = class {
|
|
|
2931
4816
|
*/
|
|
2932
4817
|
async getProcessingMessages() {
|
|
2933
4818
|
const res = await this.fetchImpl(
|
|
2934
|
-
`${this.apiUrl}/
|
|
4819
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/processing`,
|
|
2935
4820
|
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2936
4821
|
);
|
|
2937
4822
|
this.assertAuth(res, "fetching processing messages");
|
|
@@ -2945,6 +4830,32 @@ var ChannelDriver = class {
|
|
|
2945
4830
|
}
|
|
2946
4831
|
return messages;
|
|
2947
4832
|
}
|
|
4833
|
+
/**
|
|
4834
|
+
* The `opencode_session_id` fragment of a status PATCH body — `{}` when this
|
|
4835
|
+
* conversation has ABANDONED that session (#553). The field is optional
|
|
4836
|
+
* server-side and an absent one leaves the persisted binding untouched, so
|
|
4837
|
+
* omitting it is how a routine status write stops resurrecting it.
|
|
4838
|
+
*
|
|
4839
|
+
* ONLY for writes whose sole cost is a lost deep link. The `processing` notice
|
|
4840
|
+
* degrades to no "View in Evident" link (the reaction swap still fires) and the
|
|
4841
|
+
* turn-failure notice is built from the PATCH's own `error` text with a link off
|
|
4842
|
+
* the persisted row — neither loses content the user came for. `markDone`
|
|
4843
|
+
* deliberately does NOT use this helper: the server fetches the reply text
|
|
4844
|
+
* THROUGH the session id it is given, so suppressing there would replace the
|
|
4845
|
+
* agent's answer with a bare "✅ Done!" (the #183/#187 failure). The
|
|
4846
|
+
* `ensureSession` guard, not this suppression, is what makes the self-heal
|
|
4847
|
+
* stick.
|
|
4848
|
+
*/
|
|
4849
|
+
sessionIdBody(sessionId, conversationId, messageId, status) {
|
|
4850
|
+
if (!this.isSuperseded(conversationId, sessionId)) return { opencode_session_id: sessionId };
|
|
4851
|
+
this.log({
|
|
4852
|
+
level: "debug",
|
|
4853
|
+
message: `Omitting the abandoned OpenCode session ${sessionId.slice(0, 8)} from the '${status}' update for message ${messageId.slice(0, 8)} so it is not re-bound to conversation ${conversationId.slice(0, 8)}`,
|
|
4854
|
+
conversation_id: conversationId,
|
|
4855
|
+
message_id: messageId
|
|
4856
|
+
});
|
|
4857
|
+
return {};
|
|
4858
|
+
}
|
|
2948
4859
|
/**
|
|
2949
4860
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2950
4861
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2966,16 +4877,17 @@ var ChannelDriver = class {
|
|
|
2966
4877
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2967
4878
|
* retry vehicle for the swap-to-running.
|
|
2968
4879
|
*/
|
|
2969
|
-
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
4880
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId, title) {
|
|
2970
4881
|
const res = await this.fetchImpl(
|
|
2971
|
-
`${this.apiUrl}/
|
|
4882
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2972
4883
|
{
|
|
2973
4884
|
method: "PATCH",
|
|
2974
4885
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2975
4886
|
body: JSON.stringify({
|
|
2976
4887
|
status: "processing",
|
|
2977
|
-
|
|
2978
|
-
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
4888
|
+
...this.sessionIdBody(sessionId, conversationId, messageId, "processing"),
|
|
4889
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
4890
|
+
...title ? { title } : {}
|
|
2979
4891
|
})
|
|
2980
4892
|
}
|
|
2981
4893
|
);
|
|
@@ -3014,16 +4926,23 @@ var ChannelDriver = class {
|
|
|
3014
4926
|
* watcher retries next tick within the
|
|
3015
4927
|
* deadline, Finding 4).
|
|
3016
4928
|
*/
|
|
3017
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
4929
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage) {
|
|
3018
4930
|
const res = await this.fetchImpl(
|
|
3019
|
-
`${this.apiUrl}/
|
|
4931
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
3020
4932
|
{
|
|
3021
4933
|
method: "PATCH",
|
|
3022
4934
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3023
4935
|
body: JSON.stringify({
|
|
3024
4936
|
status: "done",
|
|
4937
|
+
// ALWAYS sent, even for a session this conversation has abandoned
|
|
4938
|
+
// (#553): the server reads the reply text back out of THIS session id
|
|
4939
|
+
// to deliver it. Omitting it would leave the user with "✅ Done!"
|
|
4940
|
+
// instead of the answer — a worse regression than the resurrection it
|
|
4941
|
+
// would prevent, which `ensureSession`'s guard handles anyway.
|
|
3025
4942
|
opencode_session_id: sessionId,
|
|
3026
|
-
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
4943
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
4944
|
+
...title ? { title } : {},
|
|
4945
|
+
...usage ? usage : {}
|
|
3027
4946
|
})
|
|
3028
4947
|
}
|
|
3029
4948
|
);
|
|
@@ -3036,19 +4955,29 @@ var ChannelDriver = class {
|
|
|
3036
4955
|
}
|
|
3037
4956
|
/**
|
|
3038
4957
|
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3039
|
-
* when provided (issue #182)
|
|
3040
|
-
* `
|
|
3041
|
-
*
|
|
3042
|
-
*
|
|
4958
|
+
* when provided (issue #182). Three states for `sessionId`:
|
|
4959
|
+
* - omitted (`undefined`) → don't send the field, leave the persisted
|
|
4960
|
+
* session untouched (unused today; kept for API symmetry).
|
|
4961
|
+
* - a real id (`string`) → send it, update the persisted session (the
|
|
4962
|
+
* turn-failure call sites: an errored OpenCode turn).
|
|
4963
|
+
* - explicit `null` → send it, CLEAR the persisted session (issue
|
|
4964
|
+
* #485's dispatch-handoff-failure call site: the session id still
|
|
4965
|
+
* exists but is wedged, so the next attempt must get a fresh one
|
|
4966
|
+
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
3043
4967
|
*/
|
|
3044
|
-
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
4968
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage) {
|
|
3045
4969
|
const body = { status: "failed" };
|
|
3046
|
-
if (sessionId
|
|
4970
|
+
if (sessionId === null) {
|
|
4971
|
+
body.opencode_session_id = null;
|
|
4972
|
+
} else if (sessionId !== void 0) {
|
|
4973
|
+
Object.assign(body, this.sessionIdBody(sessionId, conversationId, messageId, "failed"));
|
|
4974
|
+
}
|
|
3047
4975
|
if (error2 !== void 0) body.error = error2;
|
|
4976
|
+
if (usage) Object.assign(body, usage);
|
|
3048
4977
|
await this.callWithRetry(
|
|
3049
4978
|
"marking message as failed",
|
|
3050
4979
|
() => this.fetchImpl(
|
|
3051
|
-
`${this.apiUrl}/
|
|
4980
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
3052
4981
|
{
|
|
3053
4982
|
method: "PATCH",
|
|
3054
4983
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
@@ -3065,11 +4994,17 @@ var ChannelDriver = class {
|
|
|
3065
4994
|
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3066
4995
|
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3067
4996
|
* context (no silent catch, per development-workflow).
|
|
4997
|
+
*
|
|
4998
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
4999
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
5000
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
5001
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
5002
|
+
* leaves liveness").
|
|
3068
5003
|
*/
|
|
3069
5004
|
async postSignal(conversationId, messageId, signal, extra) {
|
|
3070
5005
|
try {
|
|
3071
5006
|
const res = await this.fetchImpl(
|
|
3072
|
-
`${this.apiUrl}/
|
|
5007
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
3073
5008
|
{
|
|
3074
5009
|
method: "POST",
|
|
3075
5010
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
@@ -3078,24 +5013,27 @@ var ChannelDriver = class {
|
|
|
3078
5013
|
);
|
|
3079
5014
|
if (!res.ok) {
|
|
3080
5015
|
this.log({
|
|
3081
|
-
level: "
|
|
5016
|
+
level: "warn",
|
|
3082
5017
|
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
3083
5018
|
conversation_id: conversationId,
|
|
3084
5019
|
message_id: messageId
|
|
3085
5020
|
});
|
|
5021
|
+
return false;
|
|
3086
5022
|
}
|
|
5023
|
+
return true;
|
|
3087
5024
|
} catch (err) {
|
|
3088
5025
|
this.log({
|
|
3089
|
-
level: "
|
|
5026
|
+
level: "warn",
|
|
3090
5027
|
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
3091
5028
|
conversation_id: conversationId,
|
|
3092
5029
|
message_id: messageId
|
|
3093
5030
|
});
|
|
5031
|
+
return false;
|
|
3094
5032
|
}
|
|
3095
5033
|
}
|
|
3096
5034
|
async persistSession(conversationId, sessionId) {
|
|
3097
5035
|
const res = await this.fetchImpl(
|
|
3098
|
-
`${this.apiUrl}/
|
|
5036
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}`,
|
|
3099
5037
|
{
|
|
3100
5038
|
method: "PATCH",
|
|
3101
5039
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
@@ -3121,7 +5059,7 @@ var ChannelDriver = class {
|
|
|
3121
5059
|
await this.callWithRetry(
|
|
3122
5060
|
"reporting interactive event",
|
|
3123
5061
|
() => this.fetchImpl(
|
|
3124
|
-
`${this.apiUrl}/
|
|
5062
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/interactive-event`,
|
|
3125
5063
|
{
|
|
3126
5064
|
method: "POST",
|
|
3127
5065
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
@@ -3147,9 +5085,7 @@ var ChannelDriver = class {
|
|
|
3147
5085
|
return false;
|
|
3148
5086
|
}
|
|
3149
5087
|
}
|
|
3150
|
-
// -------------------------------------------------------------------------
|
|
3151
5088
|
// Retry wrapper
|
|
3152
|
-
// -------------------------------------------------------------------------
|
|
3153
5089
|
/**
|
|
3154
5090
|
* Invoke an Evident API call, retrying on transient failures (5xx / 429 /
|
|
3155
5091
|
* network errors) with exponential backoff + jitter (capped). Auth failures
|
|
@@ -3226,7 +5162,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
3226
5162
|
console.log(chalk5.yellow("Tip: Run with the correct port:"));
|
|
3227
5163
|
console.log(
|
|
3228
5164
|
chalk5.dim(
|
|
3229
|
-
` ${getCliName()} run --
|
|
5165
|
+
` ${getCliName()} run --runner ${ctx.agentId} --port ${runningInstances[0].port}`
|
|
3230
5166
|
)
|
|
3231
5167
|
);
|
|
3232
5168
|
}
|
|
@@ -3348,7 +5284,7 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
3348
5284
|
if (!response.ok) {
|
|
3349
5285
|
const serverMessage = await readErrorMessage(response);
|
|
3350
5286
|
return {
|
|
3351
|
-
error: `Failed to resolve
|
|
5287
|
+
error: `Failed to resolve runner from key (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3352
5288
|
};
|
|
3353
5289
|
}
|
|
3354
5290
|
const data = await response.json();
|
|
@@ -3356,19 +5292,21 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
3356
5292
|
return { agent_id: data.agent_id };
|
|
3357
5293
|
}
|
|
3358
5294
|
return {
|
|
3359
|
-
error: "Cannot resolve
|
|
5295
|
+
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --runner explicitly."
|
|
3360
5296
|
};
|
|
3361
5297
|
} catch (error2) {
|
|
3362
5298
|
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
3363
|
-
return { error: `Failed to resolve
|
|
5299
|
+
return { error: `Failed to resolve runner from key: ${message}` };
|
|
3364
5300
|
}
|
|
3365
5301
|
}
|
|
5302
|
+
var BEST_EFFORT_NOTIFY_TIMEOUT_MS = 2e3;
|
|
3366
5303
|
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3367
5304
|
const apiUrl = getApiUrlConfig();
|
|
3368
5305
|
try {
|
|
3369
|
-
const response = await fetch(`${apiUrl}/
|
|
5306
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
3370
5307
|
method: "POST",
|
|
3371
|
-
headers: { Authorization: authHeader }
|
|
5308
|
+
headers: { Authorization: authHeader },
|
|
5309
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
3372
5310
|
});
|
|
3373
5311
|
if (!response.ok) {
|
|
3374
5312
|
const serverMessage = await readErrorMessage(response);
|
|
@@ -3379,13 +5317,41 @@ async function notifyAgentDisconnected(agentId, authHeader) {
|
|
|
3379
5317
|
}
|
|
3380
5318
|
return { ok: true };
|
|
3381
5319
|
} catch (error2) {
|
|
3382
|
-
return { ok: false, error:
|
|
5320
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
5321
|
+
}
|
|
5322
|
+
}
|
|
5323
|
+
function describeBestEffortError(error2) {
|
|
5324
|
+
const name = error2?.name;
|
|
5325
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
5326
|
+
return `timed out after ${BEST_EFFORT_NOTIFY_TIMEOUT_MS}ms`;
|
|
5327
|
+
}
|
|
5328
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
5329
|
+
}
|
|
5330
|
+
async function reportMicrovmId(agentId, authHeader, microvmId) {
|
|
5331
|
+
try {
|
|
5332
|
+
const apiUrl = getApiUrlConfig();
|
|
5333
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/microvm`, {
|
|
5334
|
+
method: "POST",
|
|
5335
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
5336
|
+
body: JSON.stringify({ microvm_id: microvmId }),
|
|
5337
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
5338
|
+
});
|
|
5339
|
+
if (!response.ok) {
|
|
5340
|
+
const serverMessage = await readErrorMessage(response);
|
|
5341
|
+
return {
|
|
5342
|
+
ok: false,
|
|
5343
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
5344
|
+
};
|
|
5345
|
+
}
|
|
5346
|
+
return { ok: true };
|
|
5347
|
+
} catch (error2) {
|
|
5348
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
3383
5349
|
}
|
|
3384
5350
|
}
|
|
3385
5351
|
async function getAgentInfo(agentId, authHeader) {
|
|
3386
5352
|
const apiUrl = getApiUrlConfig();
|
|
3387
5353
|
try {
|
|
3388
|
-
const response = await fetch(`${apiUrl}/
|
|
5354
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}`, {
|
|
3389
5355
|
headers: { Authorization: authHeader }
|
|
3390
5356
|
});
|
|
3391
5357
|
if (response.status === 401) {
|
|
@@ -3396,12 +5362,12 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
3396
5362
|
const serverMessage = await readErrorMessage(response);
|
|
3397
5363
|
return {
|
|
3398
5364
|
valid: false,
|
|
3399
|
-
error: serverMessage ?? "You do not have access to this
|
|
5365
|
+
error: serverMessage ?? "You do not have access to this runner (it may belong to a different team or organization)."
|
|
3400
5366
|
};
|
|
3401
5367
|
}
|
|
3402
5368
|
if (response.status === 404) {
|
|
3403
5369
|
const serverMessage = await readErrorMessage(response);
|
|
3404
|
-
return { valid: false, error: serverMessage ?? `
|
|
5370
|
+
return { valid: false, error: serverMessage ?? `Runner ${agentId} not found` };
|
|
3405
5371
|
}
|
|
3406
5372
|
if (!response.ok) {
|
|
3407
5373
|
const serverMessage = await readErrorMessage(response);
|
|
@@ -3414,13 +5380,13 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
3414
5380
|
if (agent.agent_type !== "local") {
|
|
3415
5381
|
return {
|
|
3416
5382
|
valid: false,
|
|
3417
|
-
error: `
|
|
5383
|
+
error: `Runner is type '${agent.agent_type}', must be 'local' for CLI connection`
|
|
3418
5384
|
};
|
|
3419
5385
|
}
|
|
3420
5386
|
return { valid: true, agent };
|
|
3421
5387
|
} catch (error2) {
|
|
3422
5388
|
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
3423
|
-
return { valid: false, error: `Failed to validate
|
|
5389
|
+
return { valid: false, error: `Failed to validate runner: ${message}` };
|
|
3424
5390
|
}
|
|
3425
5391
|
}
|
|
3426
5392
|
|
|
@@ -3429,23 +5395,82 @@ var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
|
3429
5395
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
3430
5396
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3431
5397
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
3432
|
-
|
|
5398
|
+
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
5399
|
+
function resolveLogLevel(options) {
|
|
5400
|
+
const accepted = Object.keys(LOG_LEVELS);
|
|
5401
|
+
const validate = (value, source) => {
|
|
5402
|
+
const normalized = value.trim().toLowerCase();
|
|
5403
|
+
if (!accepted.includes(normalized)) {
|
|
5404
|
+
throw new Error(
|
|
5405
|
+
`Invalid log level "${value}"${source}; expected one of ${accepted.join(", ")}`
|
|
5406
|
+
);
|
|
5407
|
+
}
|
|
5408
|
+
return normalized;
|
|
5409
|
+
};
|
|
5410
|
+
if (options.logLevel !== void 0) {
|
|
5411
|
+
return validate(options.logLevel, " (--log-level)");
|
|
5412
|
+
}
|
|
5413
|
+
if (options.verbose) {
|
|
5414
|
+
return "debug";
|
|
5415
|
+
}
|
|
5416
|
+
const env = process.env.EVIDENT_LOG_LEVEL;
|
|
5417
|
+
if (env !== void 0 && env !== "") {
|
|
5418
|
+
return validate(env, " (EVIDENT_LOG_LEVEL)");
|
|
5419
|
+
}
|
|
5420
|
+
return "info";
|
|
5421
|
+
}
|
|
5422
|
+
function resolveFileSyncDirectories(raw, homeDir) {
|
|
5423
|
+
const directories = [];
|
|
5424
|
+
for (const entry of raw ?? []) {
|
|
5425
|
+
const trimmed = entry.trim();
|
|
5426
|
+
if (trimmed === "") {
|
|
5427
|
+
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
5428
|
+
}
|
|
5429
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join2(homeDir, trimmed.slice(2)) : trimmed;
|
|
5430
|
+
if (!isAbsolute2(expanded)) {
|
|
5431
|
+
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
5432
|
+
}
|
|
5433
|
+
const normalized = resolvePath(expanded);
|
|
5434
|
+
if (parse(normalized).root === normalized) {
|
|
5435
|
+
throw new Error(
|
|
5436
|
+
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
5437
|
+
);
|
|
5438
|
+
}
|
|
5439
|
+
if (!directories.includes(normalized)) {
|
|
5440
|
+
directories.push(normalized);
|
|
5441
|
+
}
|
|
5442
|
+
}
|
|
5443
|
+
if (directories.length > MAX_FILE_SYNC_DIRECTORIES) {
|
|
5444
|
+
throw new Error(
|
|
5445
|
+
`--enable-file-sync-to accepts at most ${MAX_FILE_SYNC_DIRECTORIES} directories; got ${directories.length}`
|
|
5446
|
+
);
|
|
5447
|
+
}
|
|
5448
|
+
return directories;
|
|
5449
|
+
}
|
|
5450
|
+
function meetsThreshold(state, level) {
|
|
5451
|
+
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
5452
|
+
}
|
|
5453
|
+
function log2(state, message, level = "info") {
|
|
5454
|
+
if (!meetsThreshold(state, level)) return;
|
|
3433
5455
|
if (state.json) {
|
|
3434
5456
|
console.log(
|
|
3435
5457
|
JSON.stringify({
|
|
3436
5458
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3437
|
-
level
|
|
5459
|
+
level,
|
|
3438
5460
|
message
|
|
3439
5461
|
})
|
|
3440
5462
|
);
|
|
3441
5463
|
} else if (!state.interactive) {
|
|
3442
|
-
const prefix =
|
|
5464
|
+
const prefix = level === "error" ? chalk6.red("\u2717") : level === "warn" ? chalk6.yellow("!") : level === "debug" ? chalk6.dim("\xB7") : chalk6.green("\u2022");
|
|
3443
5465
|
console.log(`${prefix} ${message}`);
|
|
3444
5466
|
}
|
|
3445
5467
|
}
|
|
3446
5468
|
function logActivity(state, entry) {
|
|
5469
|
+
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
5470
|
+
if (!meetsThreshold(state, level)) return;
|
|
3447
5471
|
const fullEntry = {
|
|
3448
5472
|
...entry,
|
|
5473
|
+
level,
|
|
3449
5474
|
timestamp: /* @__PURE__ */ new Date()
|
|
3450
5475
|
};
|
|
3451
5476
|
state.activityLog.push(fullEntry);
|
|
@@ -3454,9 +5479,9 @@ function logActivity(state, entry) {
|
|
|
3454
5479
|
}
|
|
3455
5480
|
if (!state.interactive) {
|
|
3456
5481
|
if (entry.type === "error") {
|
|
3457
|
-
log2(state, entry.error ?? "Unknown error",
|
|
3458
|
-
} else if (entry.
|
|
3459
|
-
log2(state, entry.message);
|
|
5482
|
+
log2(state, entry.error ?? "Unknown error", level);
|
|
5483
|
+
} else if (entry.message) {
|
|
5484
|
+
log2(state, entry.message, level);
|
|
3460
5485
|
}
|
|
3461
5486
|
}
|
|
3462
5487
|
}
|
|
@@ -3543,18 +5568,29 @@ async function handleAuthError(state, error2) {
|
|
|
3543
5568
|
async function driveChannels(state, driver) {
|
|
3544
5569
|
let idlePolls = 0;
|
|
3545
5570
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
5571
|
+
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
3546
5572
|
while (state.running) {
|
|
3547
5573
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
3548
5574
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
3549
5575
|
if (state.interactive) displayStatus(state);
|
|
3550
5576
|
await state.connection.reconnectPromise;
|
|
3551
5577
|
}
|
|
5578
|
+
const carriedOverFileSync = driver.fileSyncActivity().inFlight;
|
|
5579
|
+
void driver.syncPendingFiles().catch(
|
|
5580
|
+
(error2) => logActivity(state, {
|
|
5581
|
+
type: "error",
|
|
5582
|
+
error: `Runner file sync failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
5583
|
+
})
|
|
5584
|
+
);
|
|
3552
5585
|
try {
|
|
3553
5586
|
const processed = await driver.drainPending();
|
|
3554
5587
|
state.messageCount += processed;
|
|
3555
5588
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
3556
5589
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
3557
|
-
|
|
5590
|
+
const appliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
5591
|
+
const fileActivity = carriedOverFileSync || appliedFiles !== lastSeenAppliedFiles;
|
|
5592
|
+
lastSeenAppliedFiles = appliedFiles;
|
|
5593
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
3558
5594
|
idlePolls = 0;
|
|
3559
5595
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
3560
5596
|
} else if (state.idleTimeout !== null) {
|
|
@@ -3583,7 +5619,7 @@ async function driveChannels(state, driver) {
|
|
|
3583
5619
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
3584
5620
|
if (state.interactive) displayStatus(state);
|
|
3585
5621
|
}
|
|
3586
|
-
await new Promise((
|
|
5622
|
+
await new Promise((resolve3) => setTimeout(resolve3, CHANNEL_POLL_INTERVAL_MS));
|
|
3587
5623
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
3588
5624
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
3589
5625
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -3594,6 +5630,81 @@ async function driveChannels(state, driver) {
|
|
|
3594
5630
|
}
|
|
3595
5631
|
}
|
|
3596
5632
|
}
|
|
5633
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
5634
|
+
async function runSweep(state, driver, config2) {
|
|
5635
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
5636
|
+
try {
|
|
5637
|
+
const sessions = await listSessions(state.port);
|
|
5638
|
+
if (sessions === null) {
|
|
5639
|
+
logActivity(state, {
|
|
5640
|
+
type: "info",
|
|
5641
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
5642
|
+
});
|
|
5643
|
+
return;
|
|
5644
|
+
}
|
|
5645
|
+
const toDelete = selectSessionsToDelete(
|
|
5646
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
5647
|
+
{
|
|
5648
|
+
maxAgeMs: config2.maxAgeMs,
|
|
5649
|
+
maxCount: config2.maxCount,
|
|
5650
|
+
nowMs: Date.now(),
|
|
5651
|
+
protectedIds: driver.protectedSessionIds()
|
|
5652
|
+
}
|
|
5653
|
+
);
|
|
5654
|
+
const protectedNow = driver.protectedSessionIds();
|
|
5655
|
+
let deleted = 0;
|
|
5656
|
+
let failed = 0;
|
|
5657
|
+
let skippedNewlyActive = 0;
|
|
5658
|
+
for (const id of toDelete) {
|
|
5659
|
+
if (protectedNow.has(id)) {
|
|
5660
|
+
skippedNewlyActive++;
|
|
5661
|
+
logActivity(state, {
|
|
5662
|
+
type: "info",
|
|
5663
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
5664
|
+
});
|
|
5665
|
+
continue;
|
|
5666
|
+
}
|
|
5667
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
5668
|
+
else failed++;
|
|
5669
|
+
}
|
|
5670
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
5671
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
5672
|
+
logActivity(state, {
|
|
5673
|
+
type: "info",
|
|
5674
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
5675
|
+
});
|
|
5676
|
+
} catch (error2) {
|
|
5677
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
5678
|
+
logActivity(state, {
|
|
5679
|
+
type: "error",
|
|
5680
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
5681
|
+
});
|
|
5682
|
+
}
|
|
5683
|
+
}
|
|
5684
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
5685
|
+
const config2 = resolveSessionCleanupConfig(
|
|
5686
|
+
{
|
|
5687
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
5688
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
5689
|
+
interval: options.sessionCleanupInterval
|
|
5690
|
+
},
|
|
5691
|
+
process.env
|
|
5692
|
+
);
|
|
5693
|
+
for (const warning2 of config2.warnings) {
|
|
5694
|
+
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
5695
|
+
}
|
|
5696
|
+
if (!config2.enabled) return;
|
|
5697
|
+
logActivity(state, {
|
|
5698
|
+
type: "info",
|
|
5699
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
5700
|
+
});
|
|
5701
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
5702
|
+
const firstSweep = setTimeout(
|
|
5703
|
+
() => void runSweep(state, driver, config2),
|
|
5704
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
5705
|
+
);
|
|
5706
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
5707
|
+
}
|
|
3597
5708
|
async function notifyOffline(state) {
|
|
3598
5709
|
if (!state.agentId || !state.authHeader) return;
|
|
3599
5710
|
if (!state.connected) {
|
|
@@ -3602,7 +5713,7 @@ async function notifyOffline(state) {
|
|
|
3602
5713
|
}
|
|
3603
5714
|
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
3604
5715
|
if (result.ok) {
|
|
3605
|
-
log2(state, "Notified Evident the
|
|
5716
|
+
log2(state, "Notified Evident the runner is going offline");
|
|
3606
5717
|
} else {
|
|
3607
5718
|
logActivity(state, {
|
|
3608
5719
|
type: "error",
|
|
@@ -3611,8 +5722,24 @@ async function notifyOffline(state) {
|
|
|
3611
5722
|
if (state.interactive) displayStatus(state);
|
|
3612
5723
|
}
|
|
3613
5724
|
}
|
|
5725
|
+
async function timeShutdownPhase(state, durations, name, run2) {
|
|
5726
|
+
const startedAt = Date.now();
|
|
5727
|
+
try {
|
|
5728
|
+
return await run2();
|
|
5729
|
+
} finally {
|
|
5730
|
+
const elapsedMs = Date.now() - startedAt;
|
|
5731
|
+
durations[name] = elapsedMs;
|
|
5732
|
+
log2(state, `Shutdown phase ${name}: ${elapsedMs}ms`);
|
|
5733
|
+
}
|
|
5734
|
+
}
|
|
3614
5735
|
async function cleanup(state, opts = {}) {
|
|
5736
|
+
const durations = {};
|
|
3615
5737
|
state.running = false;
|
|
5738
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
5739
|
+
clearInterval(timer);
|
|
5740
|
+
clearTimeout(timer);
|
|
5741
|
+
}
|
|
5742
|
+
state.sessionCleanupTimers = [];
|
|
3616
5743
|
if (opts.graceful && state.channelDriver) {
|
|
3617
5744
|
state.channelDriver.stop();
|
|
3618
5745
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -3620,7 +5747,13 @@ async function cleanup(state, opts = {}) {
|
|
|
3620
5747
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
3621
5748
|
displayStatus(state);
|
|
3622
5749
|
}
|
|
3623
|
-
const
|
|
5750
|
+
const driver = state.channelDriver;
|
|
5751
|
+
const settled = await timeShutdownPhase(
|
|
5752
|
+
state,
|
|
5753
|
+
durations,
|
|
5754
|
+
"drain",
|
|
5755
|
+
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
5756
|
+
);
|
|
3624
5757
|
if (!settled) {
|
|
3625
5758
|
logActivity(state, {
|
|
3626
5759
|
type: "info",
|
|
@@ -3629,13 +5762,15 @@ async function cleanup(state, opts = {}) {
|
|
|
3629
5762
|
if (state.interactive) displayStatus(state);
|
|
3630
5763
|
}
|
|
3631
5764
|
}
|
|
3632
|
-
await notifyOffline(state);
|
|
5765
|
+
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
3633
5766
|
if (state.connection) {
|
|
3634
|
-
state.connection
|
|
5767
|
+
const connection = state.connection;
|
|
5768
|
+
await timeShutdownPhase(state, durations, "tunnel_close", () => connection.close());
|
|
3635
5769
|
state.connection = null;
|
|
3636
5770
|
}
|
|
3637
5771
|
if (state.opencodeProcess) {
|
|
3638
|
-
|
|
5772
|
+
const opencodeProcess = state.opencodeProcess;
|
|
5773
|
+
await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
|
|
3639
5774
|
if (state.interactive) {
|
|
3640
5775
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
3641
5776
|
displayStatus(state);
|
|
@@ -3644,17 +5779,35 @@ async function cleanup(state, opts = {}) {
|
|
|
3644
5779
|
}
|
|
3645
5780
|
state.opencodeProcess = null;
|
|
3646
5781
|
}
|
|
5782
|
+
return durations;
|
|
3647
5783
|
}
|
|
3648
5784
|
async function run(options) {
|
|
3649
5785
|
const interactive = isInteractive(options.json);
|
|
5786
|
+
let logLevel;
|
|
5787
|
+
let fileSyncDirectories;
|
|
5788
|
+
try {
|
|
5789
|
+
logLevel = resolveLogLevel(options);
|
|
5790
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir2());
|
|
5791
|
+
} catch (error2) {
|
|
5792
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
5793
|
+
if (options.json) {
|
|
5794
|
+
console.log(JSON.stringify({ status: "error", error: message }));
|
|
5795
|
+
} else {
|
|
5796
|
+
printError(message);
|
|
5797
|
+
}
|
|
5798
|
+
await shutdownTelemetry();
|
|
5799
|
+
process.exit(1);
|
|
5800
|
+
return;
|
|
5801
|
+
}
|
|
3650
5802
|
const state = {
|
|
3651
|
-
agentId: options.agent || "",
|
|
5803
|
+
agentId: options.runner || options.agent || "",
|
|
3652
5804
|
agentName: null,
|
|
3653
5805
|
port: options.port ?? 4096,
|
|
3654
5806
|
conversationFilter: options.conversation ?? null,
|
|
3655
5807
|
idleTimeout: options.idleTimeout ?? null,
|
|
3656
5808
|
json: options.json ?? false,
|
|
3657
5809
|
interactive,
|
|
5810
|
+
logLevel,
|
|
3658
5811
|
connected: false,
|
|
3659
5812
|
opencodeConnected: false,
|
|
3660
5813
|
opencodeVersion: null,
|
|
@@ -3666,26 +5819,69 @@ async function run(options) {
|
|
|
3666
5819
|
activityLog: [],
|
|
3667
5820
|
messageCount: 0,
|
|
3668
5821
|
lastProxiedActivityAt: null,
|
|
5822
|
+
sessionCleanupTimers: [],
|
|
3669
5823
|
authHeader: ""
|
|
3670
5824
|
};
|
|
5825
|
+
if (fileSyncDirectories.length > 0) {
|
|
5826
|
+
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
5827
|
+
} else {
|
|
5828
|
+
log2(state, "File sync is disabled (no --enable-file-sync-to given)", "debug");
|
|
5829
|
+
}
|
|
5830
|
+
if (!options.runner && options.agent) {
|
|
5831
|
+
telemetry.info(
|
|
5832
|
+
EventTypes.DEPRECATED_AGENT_FLAG_USED,
|
|
5833
|
+
"Deprecated --agent flag used instead of --runner",
|
|
5834
|
+
{ command: "run" },
|
|
5835
|
+
state.agentId
|
|
5836
|
+
);
|
|
5837
|
+
const agentFlagNotice = "--agent is deprecated, use --runner instead; will be removed in a future release.";
|
|
5838
|
+
log2(state, agentFlagNotice, "warn");
|
|
5839
|
+
if (state.interactive && !state.json) {
|
|
5840
|
+
logActivity(state, { type: "info", level: "warn", message: agentFlagNotice });
|
|
5841
|
+
}
|
|
5842
|
+
}
|
|
3671
5843
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
3672
5844
|
log2(
|
|
3673
5845
|
state,
|
|
3674
|
-
"
|
|
3675
|
-
|
|
5846
|
+
"No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
|
|
5847
|
+
"warn"
|
|
3676
5848
|
);
|
|
3677
5849
|
}
|
|
3678
5850
|
const handleSignal = async () => {
|
|
3679
5851
|
if (state.shuttingDown) return;
|
|
3680
5852
|
state.shuttingDown = true;
|
|
5853
|
+
const shutdownStartedAt = Date.now();
|
|
3681
5854
|
if (state.interactive) {
|
|
3682
5855
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
3683
5856
|
displayStatus(state);
|
|
3684
5857
|
} else {
|
|
3685
5858
|
log2(state, "Shutting down...");
|
|
3686
5859
|
}
|
|
3687
|
-
await cleanup(state, { graceful: true });
|
|
3688
|
-
|
|
5860
|
+
const durations = await cleanup(state, { graceful: true });
|
|
5861
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
5862
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
5863
|
+
let timer;
|
|
5864
|
+
const flushed = shutdownTelemetry().then(
|
|
5865
|
+
() => true,
|
|
5866
|
+
(error2) => {
|
|
5867
|
+
log2(
|
|
5868
|
+
state,
|
|
5869
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
5870
|
+
"warn"
|
|
5871
|
+
);
|
|
5872
|
+
return true;
|
|
5873
|
+
}
|
|
5874
|
+
);
|
|
5875
|
+
const timedOut = new Promise((resolve3) => {
|
|
5876
|
+
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
5877
|
+
});
|
|
5878
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
5879
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
5880
|
+
}
|
|
5881
|
+
clearTimeout(timer);
|
|
5882
|
+
});
|
|
5883
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
5884
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
3689
5885
|
process.exit(0);
|
|
3690
5886
|
};
|
|
3691
5887
|
process.on("SIGINT", handleSignal);
|
|
@@ -3696,7 +5892,9 @@ async function run(options) {
|
|
|
3696
5892
|
if (!interactive) {
|
|
3697
5893
|
printError("Authentication required");
|
|
3698
5894
|
blank();
|
|
3699
|
-
console.log(
|
|
5895
|
+
console.log(
|
|
5896
|
+
chalk6.dim("Set EVIDENT_RUNNER_KEY (or EVIDENT_AGENT_KEY) environment variable for CI")
|
|
5897
|
+
);
|
|
3700
5898
|
console.log(chalk6.dim("Or run `evident login` for interactive authentication"));
|
|
3701
5899
|
blank();
|
|
3702
5900
|
process.exit(1);
|
|
@@ -3710,26 +5908,51 @@ async function run(options) {
|
|
|
3710
5908
|
);
|
|
3711
5909
|
}
|
|
3712
5910
|
state.authHeader = getAuthHeader(credentials2);
|
|
5911
|
+
if (credentials2.notice) {
|
|
5912
|
+
log2(state, credentials2.notice, "warn");
|
|
5913
|
+
if (state.interactive && !state.json) {
|
|
5914
|
+
logActivity(state, { type: "info", level: "warn", message: credentials2.notice });
|
|
5915
|
+
}
|
|
5916
|
+
}
|
|
5917
|
+
if (credentials2.keySource === "agent_key") {
|
|
5918
|
+
telemetry.info(
|
|
5919
|
+
EventTypes.DEPRECATED_AGENT_KEY_ENV_USED,
|
|
5920
|
+
"Deprecated EVIDENT_AGENT_KEY env var used instead of EVIDENT_RUNNER_KEY",
|
|
5921
|
+
{ command: "run" },
|
|
5922
|
+
state.agentId
|
|
5923
|
+
);
|
|
5924
|
+
const agentKeyNotice = "EVIDENT_AGENT_KEY is deprecated, use EVIDENT_RUNNER_KEY instead; will be removed in a future release.";
|
|
5925
|
+
log2(state, agentKeyNotice, "warn");
|
|
5926
|
+
if (state.interactive && !state.json) {
|
|
5927
|
+
logActivity(state, { type: "info", level: "warn", message: agentKeyNotice });
|
|
5928
|
+
}
|
|
5929
|
+
}
|
|
3713
5930
|
if (!state.agentId) {
|
|
3714
5931
|
if (credentials2.authType === "agent_key") {
|
|
3715
5932
|
const resolved = await resolveAgentIdFromKey(state.authHeader);
|
|
3716
5933
|
if (resolved.agent_id) {
|
|
3717
5934
|
state.agentId = resolved.agent_id;
|
|
3718
|
-
log2(state, `Resolved
|
|
5935
|
+
log2(state, `Resolved runner ID from key: ${state.agentId}`);
|
|
3719
5936
|
if (state.interactive && !state.json) {
|
|
3720
5937
|
logActivity(state, {
|
|
3721
5938
|
type: "info",
|
|
3722
|
-
message: `
|
|
5939
|
+
message: `Runner ID resolved from key: ${state.agentId}`
|
|
3723
5940
|
});
|
|
3724
5941
|
}
|
|
3725
5942
|
} else {
|
|
3726
|
-
printError(resolved.error || "Failed to resolve
|
|
5943
|
+
printError(resolved.error || "Failed to resolve runner ID from key");
|
|
3727
5944
|
process.exit(1);
|
|
3728
5945
|
}
|
|
3729
5946
|
} else {
|
|
3730
|
-
printError(
|
|
5947
|
+
printError(
|
|
5948
|
+
"--runner (or --agent) is required when not using EVIDENT_RUNNER_KEY or EVIDENT_AGENT_KEY"
|
|
5949
|
+
);
|
|
3731
5950
|
blank();
|
|
3732
|
-
console.log(
|
|
5951
|
+
console.log(
|
|
5952
|
+
chalk6.dim(
|
|
5953
|
+
"Either provide --runner/--agent <id> or set EVIDENT_RUNNER_KEY/EVIDENT_AGENT_KEY"
|
|
5954
|
+
)
|
|
5955
|
+
);
|
|
3733
5956
|
blank();
|
|
3734
5957
|
process.exit(1);
|
|
3735
5958
|
}
|
|
@@ -3751,7 +5974,7 @@ async function run(options) {
|
|
|
3751
5974
|
console.log(chalk6.bold("Evident Run"));
|
|
3752
5975
|
console.log(chalk6.dim("-".repeat(40)));
|
|
3753
5976
|
}
|
|
3754
|
-
const spinner = interactive && !state.json ? ora3("Validating
|
|
5977
|
+
const spinner = interactive && !state.json ? ora3("Validating runner...").start() : null;
|
|
3755
5978
|
let validation = await getAgentInfo(state.agentId, state.authHeader);
|
|
3756
5979
|
if (!validation.valid && validation.authFailed && interactive) {
|
|
3757
5980
|
spinner?.fail("Authentication failed");
|
|
@@ -3763,15 +5986,30 @@ async function run(options) {
|
|
|
3763
5986
|
"Login successful! Retrying..."
|
|
3764
5987
|
);
|
|
3765
5988
|
state.authHeader = getAuthHeader(credentials2);
|
|
3766
|
-
spinner?.start("Validating
|
|
5989
|
+
spinner?.start("Validating runner...");
|
|
3767
5990
|
validation = await getAgentInfo(state.agentId, state.authHeader);
|
|
3768
5991
|
}
|
|
3769
5992
|
if (!validation.valid) {
|
|
3770
|
-
spinner?.fail(`
|
|
5993
|
+
spinner?.fail(`Runner validation failed: ${validation.error}`);
|
|
3771
5994
|
throw new Error(validation.error);
|
|
3772
5995
|
}
|
|
3773
|
-
spinner?.succeed(`
|
|
5996
|
+
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
3774
5997
|
state.agentName = validation.agent.name;
|
|
5998
|
+
const microvmId = process.env.MICROVM_ID?.trim();
|
|
5999
|
+
if (microvmId) {
|
|
6000
|
+
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
6001
|
+
if (reported.ok) {
|
|
6002
|
+
log2(state, "Reported MicroVM identity so this runner can be resumed rather than restarted");
|
|
6003
|
+
} else {
|
|
6004
|
+
const message = `Could not report MicroVM identity (future wakes will cold-start): ${reported.error}`;
|
|
6005
|
+
log2(state, message, "warn");
|
|
6006
|
+
if (state.interactive && !state.json) {
|
|
6007
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
6008
|
+
}
|
|
6009
|
+
}
|
|
6010
|
+
} else {
|
|
6011
|
+
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
6012
|
+
}
|
|
3775
6013
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
3776
6014
|
try {
|
|
3777
6015
|
const oc = await ensureOpenCodeRunning({
|
|
@@ -3788,9 +6026,24 @@ async function run(options) {
|
|
|
3788
6026
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
3789
6027
|
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
3790
6028
|
if (versionWarning) {
|
|
3791
|
-
log2(state, versionWarning,
|
|
6029
|
+
log2(state, versionWarning, "warn");
|
|
3792
6030
|
if (state.interactive && !state.json) {
|
|
3793
|
-
logActivity(state, { type: "info", message: versionWarning });
|
|
6031
|
+
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
6032
|
+
}
|
|
6033
|
+
}
|
|
6034
|
+
const noProviderWarning = buildNoProviderWarning(await hasAnyConfiguredProvider(state.port));
|
|
6035
|
+
if (noProviderWarning) {
|
|
6036
|
+
log2(state, noProviderWarning, "warn");
|
|
6037
|
+
if (state.interactive && !state.json) {
|
|
6038
|
+
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
6039
|
+
blank();
|
|
6040
|
+
console.log(chalk6.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
6041
|
+
console.log(
|
|
6042
|
+
chalk6.dim(
|
|
6043
|
+
`Run ${chalk6.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
6044
|
+
)
|
|
6045
|
+
);
|
|
6046
|
+
blank();
|
|
3794
6047
|
}
|
|
3795
6048
|
}
|
|
3796
6049
|
} catch (error2) {
|
|
@@ -3805,11 +6058,21 @@ async function run(options) {
|
|
|
3805
6058
|
getAuthHeader: () => state.authHeader,
|
|
3806
6059
|
conversationFilter: state.conversationFilter,
|
|
3807
6060
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
6061
|
+
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
6062
|
+
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
6063
|
+
fileSyncDirectories,
|
|
6064
|
+
homeDir: homedir2(),
|
|
6065
|
+
log: (entry) => (
|
|
6066
|
+
// Thread the driver's real level straight through so `debug`/`warn`
|
|
6067
|
+
// survive the sink filter (they no longer collapse to info). `type`
|
|
6068
|
+
// stays the coarse error/non-error split the activity log renders with.
|
|
6069
|
+
logActivity(state, {
|
|
6070
|
+
type: entry.level === "error" ? "error" : "info",
|
|
6071
|
+
level: entry.level,
|
|
6072
|
+
message: entry.message,
|
|
6073
|
+
error: entry.level === "error" ? entry.message : void 0
|
|
6074
|
+
})
|
|
6075
|
+
)
|
|
3813
6076
|
});
|
|
3814
6077
|
state.channelDriver = channelDriver;
|
|
3815
6078
|
const connection = new RunnerConnection({
|
|
@@ -3823,8 +6086,20 @@ async function run(options) {
|
|
|
3823
6086
|
state.agentId = agentId;
|
|
3824
6087
|
logActivity(state, {
|
|
3825
6088
|
type: "info",
|
|
3826
|
-
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (
|
|
6089
|
+
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (runner: ${agentId})`
|
|
3827
6090
|
});
|
|
6091
|
+
if (options.tunnelReadyFile) {
|
|
6092
|
+
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
6093
|
+
if (marker.ok) {
|
|
6094
|
+
log2(state, `Wrote tunnel readiness marker to ${options.tunnelReadyFile}`, "debug");
|
|
6095
|
+
} else {
|
|
6096
|
+
log2(
|
|
6097
|
+
state,
|
|
6098
|
+
`Failed to write tunnel readiness marker to ${options.tunnelReadyFile}: ${marker.error}`,
|
|
6099
|
+
"error"
|
|
6100
|
+
);
|
|
6101
|
+
}
|
|
6102
|
+
}
|
|
3828
6103
|
emitAgentConnected(state.agentId, {
|
|
3829
6104
|
port: state.port,
|
|
3830
6105
|
cli_version: getCliVersion(),
|
|
@@ -3880,6 +6155,12 @@ async function run(options) {
|
|
|
3880
6155
|
onDrainPing: () => {
|
|
3881
6156
|
if (!state.running) return;
|
|
3882
6157
|
logActivity(state, { type: "info", message: "Drain ping received \u2014 draining" });
|
|
6158
|
+
void channelDriver.syncPendingFiles().catch(
|
|
6159
|
+
(error2) => logActivity(state, {
|
|
6160
|
+
type: "error",
|
|
6161
|
+
error: `Runner file sync failed on ping: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
6162
|
+
})
|
|
6163
|
+
);
|
|
3883
6164
|
channelDriver.drainPending().then((processed) => {
|
|
3884
6165
|
if (processed > 0) {
|
|
3885
6166
|
state.messageCount += processed;
|
|
@@ -3908,6 +6189,7 @@ async function run(options) {
|
|
|
3908
6189
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
3909
6190
|
throw error2;
|
|
3910
6191
|
}
|
|
6192
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
3911
6193
|
if (!interactive || state.json) {
|
|
3912
6194
|
log2(state, "Driving channel messages...");
|
|
3913
6195
|
}
|
|
@@ -3937,7 +6219,7 @@ async function run(options) {
|
|
|
3937
6219
|
}
|
|
3938
6220
|
telemetry.error(EventTypes.CLI_ERROR, `Run command failed: ${message}`, {
|
|
3939
6221
|
command: "run",
|
|
3940
|
-
agentId: options.agent
|
|
6222
|
+
agentId: options.runner || options.agent
|
|
3941
6223
|
});
|
|
3942
6224
|
await shutdownTelemetry();
|
|
3943
6225
|
process.exit(1);
|
|
@@ -3962,15 +6244,50 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
3962
6244
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
3963
6245
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
3964
6246
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
3965
|
-
program.command("run").description("Connect to Evident and process messages").option("
|
|
6247
|
+
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
6248
|
+
"-a, --agent [id]",
|
|
6249
|
+
"Deprecated alias for --runner (still supported; --runner wins if both are given)"
|
|
6250
|
+
).option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option(
|
|
6251
|
+
"--log-level <level>",
|
|
6252
|
+
"Log verbosity: debug | info | warn | error (default: info). Env: EVIDENT_LOG_LEVEL"
|
|
6253
|
+
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
6254
|
+
"--session-cleanup-max-age <duration>",
|
|
6255
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
6256
|
+
).option(
|
|
6257
|
+
"--session-cleanup-max-count <n>",
|
|
6258
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
6259
|
+
).option(
|
|
6260
|
+
"--session-cleanup-interval <duration>",
|
|
6261
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
6262
|
+
).option(
|
|
6263
|
+
"--enable-file-sync-to <dir>",
|
|
6264
|
+
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
6265
|
+
(value, previous) => previous.concat([value]),
|
|
6266
|
+
[]
|
|
6267
|
+
).option(
|
|
6268
|
+
"--tunnel-ready-file <path>",
|
|
6269
|
+
"Path to write once the tunnel is connected (set by the MicroVM hooks; unused on a developer machine)"
|
|
6270
|
+
).action(
|
|
3966
6271
|
(options) => {
|
|
3967
6272
|
run({
|
|
3968
6273
|
agent: options.agent,
|
|
6274
|
+
runner: options.runner,
|
|
3969
6275
|
port: parseInt(options.port, 10),
|
|
6276
|
+
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
6277
|
+
// resolveLogLevel (flag > -v > EVIDENT_LOG_LEVEL > info).
|
|
6278
|
+
logLevel: options.logLevel,
|
|
3970
6279
|
verbose: options.verbose,
|
|
3971
6280
|
conversation: options.conversation,
|
|
3972
6281
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
3973
|
-
json: options.json
|
|
6282
|
+
json: options.json,
|
|
6283
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
6284
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
6285
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
6286
|
+
sessionCleanupInterval: options.sessionCleanupInterval,
|
|
6287
|
+
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
6288
|
+
// resolveFileSyncDirectories.
|
|
6289
|
+
enableFileSyncTo: options.enableFileSyncTo,
|
|
6290
|
+
tunnelReadyFile: options.tunnelReadyFile
|
|
3974
6291
|
});
|
|
3975
6292
|
}
|
|
3976
6293
|
);
|