@evident-ai/cli 3.5.2-dev.4912967 → 3.5.2-dev.4f7b3ac
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1546 -429
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1070,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1158
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1323
|
|
|
1160
1324
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.4f7b3ac" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1326
|
function getCliVersion() {
|
|
1163
1327
|
return CLI_VERSION;
|
|
1164
1328
|
}
|
|
@@ -1172,6 +1336,23 @@ var authProvider = null;
|
|
|
1172
1336
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1337
|
authProvider = provider;
|
|
1174
1338
|
}
|
|
1339
|
+
var diagnosticsReporter = null;
|
|
1340
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1341
|
+
diagnosticsReporter = reporter;
|
|
1342
|
+
}
|
|
1343
|
+
function reportTelemetryDiagnostic(message) {
|
|
1344
|
+
if (!diagnosticsReporter) {
|
|
1345
|
+
console.error(message);
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
diagnosticsReporter(message);
|
|
1350
|
+
} catch (error2) {
|
|
1351
|
+
console.error(
|
|
1352
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1175
1356
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1357
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1358
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1418,9 @@ async function flushEvents() {
|
|
|
1237
1418
|
signal: controller.signal
|
|
1238
1419
|
});
|
|
1239
1420
|
if (!response.ok) {
|
|
1240
|
-
|
|
1421
|
+
reportTelemetryDiagnostic(
|
|
1422
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1423
|
+
);
|
|
1241
1424
|
}
|
|
1242
1425
|
} finally {
|
|
1243
1426
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1430,9 @@ async function flushEvents() {
|
|
|
1247
1430
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1431
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1432
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1433
|
+
reportTelemetryDiagnostic(
|
|
1434
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1435
|
+
);
|
|
1251
1436
|
lastFlushFailureLoggedAt = now;
|
|
1252
1437
|
suppressedFlushFailureCount = 0;
|
|
1253
1438
|
} else {
|
|
@@ -1300,6 +1485,219 @@ var EventTypes = {
|
|
|
1300
1485
|
|
|
1301
1486
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
1487
|
import { homedir as homedir2 } from "node:os";
|
|
1488
|
+
|
|
1489
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1490
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1491
|
+
import { dirname as dirname2 } from "node:path";
|
|
1492
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1493
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1494
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1495
|
+
function errorMessage(error2) {
|
|
1496
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1497
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1498
|
+
}
|
|
1499
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1500
|
+
const messages = [];
|
|
1501
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1502
|
+
let current = error2;
|
|
1503
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1504
|
+
visited.add(current);
|
|
1505
|
+
if (typeof current === "string") {
|
|
1506
|
+
messages.push(current);
|
|
1507
|
+
break;
|
|
1508
|
+
}
|
|
1509
|
+
if (current instanceof Error) {
|
|
1510
|
+
messages.push(current.name, current.message);
|
|
1511
|
+
current = current.cause;
|
|
1512
|
+
continue;
|
|
1513
|
+
}
|
|
1514
|
+
if (typeof current === "object") {
|
|
1515
|
+
const record = current;
|
|
1516
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1517
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1518
|
+
current = record.cause;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
break;
|
|
1522
|
+
}
|
|
1523
|
+
return messages.some(
|
|
1524
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1525
|
+
message
|
|
1526
|
+
)
|
|
1527
|
+
);
|
|
1528
|
+
}
|
|
1529
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1530
|
+
try {
|
|
1531
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1532
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1533
|
+
if (availableBytes < requiredBytes) {
|
|
1534
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1535
|
+
}
|
|
1536
|
+
return null;
|
|
1537
|
+
} catch (err) {
|
|
1538
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
function readLogicalBytes(db) {
|
|
1542
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1543
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1544
|
+
return pageCount * pageSize;
|
|
1545
|
+
}
|
|
1546
|
+
function readFreelistCount(db) {
|
|
1547
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1548
|
+
}
|
|
1549
|
+
function readCheckpointResult(db) {
|
|
1550
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1551
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1552
|
+
}
|
|
1553
|
+
async function probeReclaimAvailability(input) {
|
|
1554
|
+
const { dbPath, requiredBytes } = input;
|
|
1555
|
+
let sqlite;
|
|
1556
|
+
try {
|
|
1557
|
+
sqlite = await import("node:sqlite");
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1560
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1561
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1562
|
+
}
|
|
1563
|
+
let autoVacuum = null;
|
|
1564
|
+
try {
|
|
1565
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1566
|
+
try {
|
|
1567
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1568
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1569
|
+
} finally {
|
|
1570
|
+
db.close();
|
|
1571
|
+
}
|
|
1572
|
+
} catch (err) {
|
|
1573
|
+
console.warn(
|
|
1574
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1575
|
+
);
|
|
1576
|
+
}
|
|
1577
|
+
if (autoVacuum !== 0) return null;
|
|
1578
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1579
|
+
}
|
|
1580
|
+
async function reclaimSessionDbSpace(input) {
|
|
1581
|
+
const {
|
|
1582
|
+
dbPath,
|
|
1583
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1584
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1585
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1586
|
+
now = () => performance.now(),
|
|
1587
|
+
allowFullVacuum = true
|
|
1588
|
+
} = input;
|
|
1589
|
+
let sqlite;
|
|
1590
|
+
try {
|
|
1591
|
+
sqlite = await import("node:sqlite");
|
|
1592
|
+
} catch (err) {
|
|
1593
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1594
|
+
console.warn(
|
|
1595
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1596
|
+
);
|
|
1597
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1598
|
+
}
|
|
1599
|
+
const { DatabaseSync } = sqlite;
|
|
1600
|
+
let db;
|
|
1601
|
+
try {
|
|
1602
|
+
db = new DatabaseSync(dbPath);
|
|
1603
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1604
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1605
|
+
if (autoVacuum === 0) {
|
|
1606
|
+
if (!allowFullVacuum) {
|
|
1607
|
+
console.warn(
|
|
1608
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1609
|
+
);
|
|
1610
|
+
return {
|
|
1611
|
+
ok: false,
|
|
1612
|
+
skipped: "full-vacuum-blocked",
|
|
1613
|
+
detail: "a session turn is live"
|
|
1614
|
+
};
|
|
1615
|
+
}
|
|
1616
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1617
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1618
|
+
if (skipReason !== null) {
|
|
1619
|
+
console.warn(
|
|
1620
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1621
|
+
);
|
|
1622
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1623
|
+
}
|
|
1624
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1625
|
+
const freelistBefore = readFreelistCount(db);
|
|
1626
|
+
const startedAt = now();
|
|
1627
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1628
|
+
db.exec("VACUUM");
|
|
1629
|
+
const afterBytes = readLogicalBytes(db);
|
|
1630
|
+
const freelistAfter = readFreelistCount(db);
|
|
1631
|
+
const checkpoint = readCheckpointResult(db);
|
|
1632
|
+
return {
|
|
1633
|
+
ok: true,
|
|
1634
|
+
mode: "convert",
|
|
1635
|
+
beforeBytes,
|
|
1636
|
+
afterBytes,
|
|
1637
|
+
freelistBefore,
|
|
1638
|
+
freelistAfter,
|
|
1639
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1640
|
+
stoppedBy: "freelist-drained",
|
|
1641
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1642
|
+
checkpoint
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
if (autoVacuum === 2) {
|
|
1646
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1647
|
+
const freelistBefore = readFreelistCount(db);
|
|
1648
|
+
const startedAt = now();
|
|
1649
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1650
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1651
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1652
|
+
let pagesAttempted = 0;
|
|
1653
|
+
let freelistAfter = freelistBefore;
|
|
1654
|
+
let stoppedBy = "freelist-drained";
|
|
1655
|
+
while (freelistAfter > 0) {
|
|
1656
|
+
if (pagesAttempted >= pageLimit) {
|
|
1657
|
+
stoppedBy = "page-ceiling";
|
|
1658
|
+
break;
|
|
1659
|
+
}
|
|
1660
|
+
if (now() - startedAt >= budget) {
|
|
1661
|
+
stoppedBy = "time-budget";
|
|
1662
|
+
break;
|
|
1663
|
+
}
|
|
1664
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1665
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1666
|
+
pagesAttempted += pages;
|
|
1667
|
+
freelistAfter = readFreelistCount(db);
|
|
1668
|
+
}
|
|
1669
|
+
const afterBytes = readLogicalBytes(db);
|
|
1670
|
+
const checkpoint = readCheckpointResult(db);
|
|
1671
|
+
return {
|
|
1672
|
+
ok: true,
|
|
1673
|
+
mode: "incremental",
|
|
1674
|
+
beforeBytes,
|
|
1675
|
+
afterBytes,
|
|
1676
|
+
freelistBefore,
|
|
1677
|
+
freelistAfter,
|
|
1678
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1679
|
+
stoppedBy,
|
|
1680
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1681
|
+
checkpoint
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1684
|
+
console.warn(
|
|
1685
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1686
|
+
);
|
|
1687
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1688
|
+
} catch (err) {
|
|
1689
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1690
|
+
return {
|
|
1691
|
+
ok: false,
|
|
1692
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1693
|
+
detail: errorMessage(err)
|
|
1694
|
+
};
|
|
1695
|
+
} finally {
|
|
1696
|
+
db?.close();
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1303
1701
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1304
1702
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1305
1703
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1316,26 +1714,85 @@ function parseOpenCodeLogLine(line) {
|
|
|
1316
1714
|
if (!levelMatch) return null;
|
|
1317
1715
|
const level = levelMatch[1].toUpperCase();
|
|
1318
1716
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1319
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1320
|
-
|
|
1717
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1718
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1719
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1720
|
+
const parsed = {
|
|
1721
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1722
|
+
sessionID: sessionMatch?.[1],
|
|
1723
|
+
messageID: messageIDMatch?.[1]
|
|
1724
|
+
};
|
|
1725
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1726
|
+
if (message !== void 0) parsed.message = message;
|
|
1727
|
+
return parsed;
|
|
1321
1728
|
}
|
|
1322
1729
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1323
|
-
|
|
1730
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1731
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1324
1732
|
let buffer = Buffer.alloc(0);
|
|
1325
|
-
const
|
|
1326
|
-
|
|
1327
|
-
|
|
1733
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1734
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1735
|
+
const metadata = {
|
|
1736
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1737
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1738
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1739
|
+
};
|
|
1328
1740
|
forwardRunnerActivity(
|
|
1329
1741
|
{
|
|
1330
1742
|
level: parsed.level,
|
|
1331
1743
|
error: line,
|
|
1332
|
-
metadata:
|
|
1744
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1333
1745
|
source: "opencode"
|
|
1334
1746
|
},
|
|
1335
1747
|
getContext()
|
|
1336
1748
|
);
|
|
1337
1749
|
};
|
|
1338
|
-
|
|
1750
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1751
|
+
for (const [key, state] of repeatStates) {
|
|
1752
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1753
|
+
continue;
|
|
1754
|
+
}
|
|
1755
|
+
if (state.repeatsSuppressed > 0) {
|
|
1756
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1757
|
+
}
|
|
1758
|
+
repeatStates.delete(key);
|
|
1759
|
+
}
|
|
1760
|
+
};
|
|
1761
|
+
const flushLine = (line) => {
|
|
1762
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1763
|
+
try {
|
|
1764
|
+
onSessionStoreWriteFailure();
|
|
1765
|
+
} catch (err) {
|
|
1766
|
+
reportTelemetryDiagnostic(
|
|
1767
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1768
|
+
);
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1772
|
+
if (!parsed) return;
|
|
1773
|
+
const now = Date.now();
|
|
1774
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1775
|
+
expireRepeatStates(now, key);
|
|
1776
|
+
if (key === void 0) {
|
|
1777
|
+
forwardLine(line, parsed);
|
|
1778
|
+
return;
|
|
1779
|
+
}
|
|
1780
|
+
const previous = repeatStates.get(key);
|
|
1781
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1782
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1783
|
+
repeatStates.set(key, {
|
|
1784
|
+
line,
|
|
1785
|
+
parsed,
|
|
1786
|
+
lastForwardedAt: now,
|
|
1787
|
+
repeatsSuppressed: 0
|
|
1788
|
+
});
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
previous.line = line;
|
|
1792
|
+
previous.parsed = parsed;
|
|
1793
|
+
previous.repeatsSuppressed++;
|
|
1794
|
+
};
|
|
1795
|
+
const forward = ((chunk) => {
|
|
1339
1796
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1340
1797
|
let newlineIndex;
|
|
1341
1798
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1346,7 +1803,16 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1346
1803
|
flushLine(buffer.toString("utf-8"));
|
|
1347
1804
|
buffer = Buffer.alloc(0);
|
|
1348
1805
|
}
|
|
1806
|
+
});
|
|
1807
|
+
forward.flush = () => {
|
|
1808
|
+
for (const state of repeatStates.values()) {
|
|
1809
|
+
if (state.repeatsSuppressed > 0) {
|
|
1810
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1811
|
+
}
|
|
1812
|
+
}
|
|
1813
|
+
repeatStates.clear();
|
|
1349
1814
|
};
|
|
1815
|
+
return forward;
|
|
1350
1816
|
}
|
|
1351
1817
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1352
1818
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
@@ -1372,7 +1838,7 @@ function redactHomeDirectory(message) {
|
|
|
1372
1838
|
}
|
|
1373
1839
|
return result + message.slice(searchFrom);
|
|
1374
1840
|
}
|
|
1375
|
-
function
|
|
1841
|
+
function redactDiagnosticText(message) {
|
|
1376
1842
|
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1377
1843
|
return redactHomeDirectory(redacted);
|
|
1378
1844
|
}
|
|
@@ -1384,14 +1850,17 @@ function sanitiseMetadata(metadata) {
|
|
|
1384
1850
|
if (!metadata) return {};
|
|
1385
1851
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1386
1852
|
if (Object.keys(metadata).length > entries.length) {
|
|
1387
|
-
|
|
1388
|
-
`
|
|
1853
|
+
reportTelemetryDiagnostic(
|
|
1854
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1389
1855
|
);
|
|
1390
1856
|
}
|
|
1391
1857
|
const sanitised = [];
|
|
1392
1858
|
for (const [key, value] of entries) {
|
|
1393
1859
|
if (typeof value === "string") {
|
|
1394
|
-
sanitised.push([
|
|
1860
|
+
sanitised.push([
|
|
1861
|
+
key,
|
|
1862
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1863
|
+
]);
|
|
1395
1864
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1396
1865
|
sanitised.push([key, value]);
|
|
1397
1866
|
}
|
|
@@ -1409,8 +1878,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1409
1878
|
}
|
|
1410
1879
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1411
1880
|
if (window.windowDroppedCount > 0) {
|
|
1412
|
-
|
|
1413
|
-
`
|
|
1881
|
+
reportTelemetryDiagnostic(
|
|
1882
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1414
1883
|
);
|
|
1415
1884
|
}
|
|
1416
1885
|
window.windowStartedAt = now;
|
|
@@ -1420,8 +1889,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1420
1889
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1421
1890
|
window.windowDroppedCount++;
|
|
1422
1891
|
if (window.windowDroppedCount === 1) {
|
|
1423
|
-
|
|
1424
|
-
`
|
|
1892
|
+
reportTelemetryDiagnostic(
|
|
1893
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1425
1894
|
);
|
|
1426
1895
|
}
|
|
1427
1896
|
return false;
|
|
@@ -1436,7 +1905,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1436
1905
|
if (!context.agentId || !context.authHeader) return;
|
|
1437
1906
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1438
1907
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1439
|
-
const message = truncate(
|
|
1908
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1440
1909
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1441
1910
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1442
1911
|
message,
|
|
@@ -1444,8 +1913,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1444
1913
|
agentId: context.agentId
|
|
1445
1914
|
});
|
|
1446
1915
|
} catch (err) {
|
|
1447
|
-
|
|
1448
|
-
`
|
|
1916
|
+
reportTelemetryDiagnostic(
|
|
1917
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1449
1918
|
);
|
|
1450
1919
|
}
|
|
1451
1920
|
}
|
|
@@ -1585,12 +2054,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1585
2054
|
metadata: withoutContractFields(record),
|
|
1586
2055
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1587
2056
|
};
|
|
1588
|
-
case "session_db_boot_refused":
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
2057
|
+
case "session_db_boot_refused": {
|
|
2058
|
+
const message = (() => {
|
|
2059
|
+
switch (record.reason) {
|
|
2060
|
+
case "local_discard_failed":
|
|
2061
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2062
|
+
case "replica_separation_unproven":
|
|
2063
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2064
|
+
case "verification_timeout_before_separation":
|
|
2065
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2066
|
+
case "verification_timeout_during_separation":
|
|
2067
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2068
|
+
default:
|
|
2069
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2070
|
+
}
|
|
2071
|
+
})();
|
|
2072
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2073
|
+
}
|
|
1594
2074
|
case "schema_provenance_mismatch":
|
|
1595
2075
|
return {
|
|
1596
2076
|
level,
|
|
@@ -1735,8 +2215,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1735
2215
|
|
|
1736
2216
|
// src/lib/http-timeout.ts
|
|
1737
2217
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2218
|
+
var RequestTimeoutError = class extends Error {
|
|
2219
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2220
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2221
|
+
this.operation = operation;
|
|
2222
|
+
this.timeoutMs = timeoutMs;
|
|
2223
|
+
this.elapsedMs = elapsedMs;
|
|
2224
|
+
this.name = "TimeoutError";
|
|
2225
|
+
}
|
|
2226
|
+
};
|
|
2227
|
+
function describeRequestOperation(input, init) {
|
|
2228
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2229
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2230
|
+
let path;
|
|
2231
|
+
try {
|
|
2232
|
+
path = new URL(rawUrl).pathname;
|
|
2233
|
+
} catch (error2) {
|
|
2234
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2235
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2236
|
+
}
|
|
2237
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2238
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2239
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2240
|
+
return `${method} ${maskedPath}`;
|
|
2241
|
+
}
|
|
1738
2242
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1739
|
-
return ((input, init) =>
|
|
2243
|
+
return (async (input, init) => {
|
|
2244
|
+
const startedAt = Date.now();
|
|
2245
|
+
try {
|
|
2246
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2247
|
+
} catch (error2) {
|
|
2248
|
+
if (error2?.name === "TimeoutError") {
|
|
2249
|
+
throw new RequestTimeoutError(
|
|
2250
|
+
describeRequestOperation(input, init),
|
|
2251
|
+
timeoutMs,
|
|
2252
|
+
Date.now() - startedAt
|
|
2253
|
+
);
|
|
2254
|
+
}
|
|
2255
|
+
throw error2;
|
|
2256
|
+
}
|
|
2257
|
+
});
|
|
1740
2258
|
}
|
|
1741
2259
|
|
|
1742
2260
|
// src/lib/opencode/client.ts
|
|
@@ -1802,7 +2320,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1802
2320
|
return null;
|
|
1803
2321
|
}
|
|
1804
2322
|
}
|
|
1805
|
-
function
|
|
2323
|
+
function roleOf2(m) {
|
|
1806
2324
|
if (!m || typeof m !== "object") return void 0;
|
|
1807
2325
|
if (typeof m.role === "string") return m.role;
|
|
1808
2326
|
const infoRole = m.info?.role;
|
|
@@ -1816,13 +2334,13 @@ function createdOf(m) {
|
|
|
1816
2334
|
if (!m || typeof m !== "object") return void 0;
|
|
1817
2335
|
return m.info?.time?.created ?? m.time?.created;
|
|
1818
2336
|
}
|
|
1819
|
-
function
|
|
2337
|
+
function idOf2(m) {
|
|
1820
2338
|
if (!m || typeof m !== "object") return void 0;
|
|
1821
2339
|
if (typeof m.id === "string") return m.id;
|
|
1822
2340
|
const infoId = m.info?.id;
|
|
1823
2341
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1824
2342
|
}
|
|
1825
|
-
function
|
|
2343
|
+
function parentIdOf2(m) {
|
|
1826
2344
|
if (!m || typeof m !== "object") return void 0;
|
|
1827
2345
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1828
2346
|
const infoParent = m.info?.parentID;
|
|
@@ -1838,8 +2356,12 @@ function errorOf(m) {
|
|
|
1838
2356
|
if (!m || typeof m !== "object") return void 0;
|
|
1839
2357
|
return m.info?.error ?? m.error;
|
|
1840
2358
|
}
|
|
2359
|
+
function isTerminalStopAnswer(m) {
|
|
2360
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2361
|
+
}
|
|
1841
2362
|
function isAssistantInFlight(m) {
|
|
1842
2363
|
if (completedOf(m) == null) return true;
|
|
2364
|
+
if (errorOf(m) != null) return false;
|
|
1843
2365
|
return finishOf(m) === "tool-calls";
|
|
1844
2366
|
}
|
|
1845
2367
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -1887,9 +2409,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1887
2409
|
function isSessionActivelyGenerating(messages) {
|
|
1888
2410
|
if (!messages || messages.length === 0) return false;
|
|
1889
2411
|
const last = messages[messages.length - 1];
|
|
1890
|
-
if (
|
|
2412
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1891
2413
|
return completedOf(last) == null;
|
|
1892
2414
|
}
|
|
2415
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1893
2416
|
function sessionLastActivityMs(session) {
|
|
1894
2417
|
const candidates = [
|
|
1895
2418
|
session.time?.updated,
|
|
@@ -1904,30 +2427,60 @@ function sessionLastActivityMs(session) {
|
|
|
1904
2427
|
}
|
|
1905
2428
|
return null;
|
|
1906
2429
|
}
|
|
1907
|
-
|
|
1908
|
-
|
|
2430
|
+
function sessionListPath(options) {
|
|
2431
|
+
const params = new URLSearchParams();
|
|
2432
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2433
|
+
const query = params.toString();
|
|
2434
|
+
return query ? `/session?${query}` : "/session";
|
|
2435
|
+
}
|
|
2436
|
+
async function listSessionPage(port, client, options) {
|
|
2437
|
+
const path = sessionListPath(options);
|
|
1909
2438
|
try {
|
|
1910
|
-
const res = await requestWithClient(port, client,
|
|
2439
|
+
const res = await requestWithClient(port, client, path);
|
|
1911
2440
|
if (!res.ok) return null;
|
|
1912
2441
|
const body = await res.json();
|
|
1913
2442
|
return Array.isArray(body) ? body : null;
|
|
1914
2443
|
} catch (error2) {
|
|
1915
2444
|
console.error(
|
|
1916
|
-
`[
|
|
2445
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1917
2446
|
);
|
|
1918
2447
|
return null;
|
|
1919
2448
|
}
|
|
1920
2449
|
}
|
|
2450
|
+
async function listSessions(port, client, options) {
|
|
2451
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2452
|
+
return listSessionPage(port, client, options);
|
|
2453
|
+
}
|
|
2454
|
+
async function listAllSessions(port, client) {
|
|
2455
|
+
if (client?.version === "v2") {
|
|
2456
|
+
const sessions2 = await listV2Sessions(client);
|
|
2457
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2458
|
+
}
|
|
2459
|
+
const sessions = await listSessionPage(port, client, {
|
|
2460
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2461
|
+
});
|
|
2462
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2463
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2464
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2465
|
+
}
|
|
2466
|
+
return { sessions };
|
|
2467
|
+
}
|
|
1921
2468
|
async function deleteSession(port, id, client) {
|
|
1922
2469
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1923
2470
|
try {
|
|
1924
|
-
const res = await requestWithClient(
|
|
1925
|
-
|
|
2471
|
+
const res = await requestWithClient(
|
|
2472
|
+
port,
|
|
2473
|
+
client,
|
|
2474
|
+
`/session/${id}`,
|
|
2475
|
+
{ method: "DELETE" },
|
|
2476
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2477
|
+
);
|
|
2478
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1926
2479
|
} catch (error2) {
|
|
1927
2480
|
console.error(
|
|
1928
2481
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1929
2482
|
);
|
|
1930
|
-
return false;
|
|
2483
|
+
return { ok: false, status: "threw" };
|
|
1931
2484
|
}
|
|
1932
2485
|
}
|
|
1933
2486
|
async function sessionExists(port, id, client) {
|
|
@@ -2165,11 +2718,31 @@ function messageText(m) {
|
|
|
2165
2718
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2166
2719
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2167
2720
|
}
|
|
2168
|
-
|
|
2721
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2722
|
+
if (!messages || messages.length === 0) return null;
|
|
2723
|
+
let best = null;
|
|
2724
|
+
for (const message of messages) {
|
|
2725
|
+
if (roleOf2(message) !== "user") continue;
|
|
2726
|
+
const id = idOf2(message);
|
|
2727
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2728
|
+
if (messageText(message) !== content) continue;
|
|
2729
|
+
const created = createdOf(message) ?? 0;
|
|
2730
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2731
|
+
}
|
|
2732
|
+
return best?.id ?? null;
|
|
2733
|
+
}
|
|
2734
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2169
2735
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2170
2736
|
const knownUserIds = new Set(
|
|
2171
|
-
(before ?? []).filter((m) =>
|
|
2737
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2172
2738
|
);
|
|
2739
|
+
if (callbacks) {
|
|
2740
|
+
try {
|
|
2741
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2742
|
+
} catch (error2) {
|
|
2743
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2744
|
+
}
|
|
2745
|
+
}
|
|
2173
2746
|
const parts = [{ type: "text", text: content }];
|
|
2174
2747
|
let pendingOutcomes = null;
|
|
2175
2748
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2208,22 +2781,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2208
2781
|
const READ_BACK_DELAY_MS = 150;
|
|
2209
2782
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2210
2783
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
const id = idOf(m);
|
|
2216
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2217
|
-
if (messageText(m) !== content) continue;
|
|
2218
|
-
const created = createdOf(m) ?? 0;
|
|
2219
|
-
if (best === null || created > best.created) {
|
|
2220
|
-
best = { id, created };
|
|
2221
|
-
}
|
|
2222
|
-
}
|
|
2223
|
-
if (best) {
|
|
2224
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2225
|
-
return best.id;
|
|
2226
|
-
}
|
|
2784
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2785
|
+
if (dispatchedUserMessageId !== null) {
|
|
2786
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2787
|
+
return dispatchedUserMessageId;
|
|
2227
2788
|
}
|
|
2228
2789
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2229
2790
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2234,57 +2795,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2234
2795
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2235
2796
|
if (!messages || messages.length === 0) return null;
|
|
2236
2797
|
const byParent = messages.find(
|
|
2237
|
-
(m) =>
|
|
2798
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2238
2799
|
);
|
|
2239
2800
|
if (byParent) return byParent;
|
|
2240
|
-
const userIndex = messages.findIndex((m) =>
|
|
2801
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2241
2802
|
if (userIndex === -1) return null;
|
|
2242
2803
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2243
|
-
if (
|
|
2804
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2244
2805
|
}
|
|
2245
2806
|
return null;
|
|
2246
2807
|
}
|
|
2247
2808
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2248
2809
|
if (!messages || messages.length === 0) return null;
|
|
2249
2810
|
let lastCorrelated = null;
|
|
2250
|
-
let lastNonErrored = null;
|
|
2251
2811
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2252
2812
|
const m = messages[i];
|
|
2253
|
-
if (
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2813
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2814
|
+
lastCorrelated = m;
|
|
2815
|
+
break;
|
|
2816
|
+
}
|
|
2817
|
+
if (lastCorrelated) {
|
|
2818
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2819
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2820
|
+
const m = messages[i];
|
|
2821
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2822
|
+
return m;
|
|
2823
|
+
}
|
|
2258
2824
|
}
|
|
2825
|
+
return lastCorrelated;
|
|
2259
2826
|
}
|
|
2260
|
-
|
|
2261
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2827
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2262
2828
|
if (userIndex === -1) return null;
|
|
2263
2829
|
let last = null;
|
|
2264
|
-
let
|
|
2830
|
+
let lastAssistantIndex = -1;
|
|
2265
2831
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2266
|
-
const role =
|
|
2832
|
+
const role = roleOf2(messages[i]);
|
|
2267
2833
|
if (role === "user") break;
|
|
2268
2834
|
if (role === "assistant") {
|
|
2269
2835
|
last = messages[i];
|
|
2270
|
-
|
|
2836
|
+
lastAssistantIndex = i;
|
|
2271
2837
|
}
|
|
2272
2838
|
}
|
|
2273
|
-
|
|
2839
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2840
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2841
|
+
const m = messages[i];
|
|
2842
|
+
if (roleOf2(m) === "user") break;
|
|
2843
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2844
|
+
}
|
|
2845
|
+
return last;
|
|
2274
2846
|
}
|
|
2275
2847
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2276
2848
|
if (!messages || messages.length === 0) return [];
|
|
2277
2849
|
const byParent = messages.filter(
|
|
2278
|
-
(message) =>
|
|
2850
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2279
2851
|
);
|
|
2280
2852
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2281
2853
|
if (assistants.length === 0) {
|
|
2282
|
-
const userIndex = messages.findIndex((message) =>
|
|
2854
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2283
2855
|
if (userIndex === -1) return [];
|
|
2284
2856
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2285
2857
|
const message = messages[i];
|
|
2286
|
-
if (
|
|
2287
|
-
if (
|
|
2858
|
+
if (roleOf2(message) === "user") break;
|
|
2859
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2288
2860
|
}
|
|
2289
2861
|
}
|
|
2290
2862
|
const refs = [];
|
|
@@ -2324,7 +2896,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2324
2896
|
if (!messages || messages.length === 0) return [];
|
|
2325
2897
|
const calls = [];
|
|
2326
2898
|
for (const message of messages) {
|
|
2327
|
-
if (
|
|
2899
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2328
2900
|
for (const part of message.parts ?? []) {
|
|
2329
2901
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2330
2902
|
continue;
|
|
@@ -2354,7 +2926,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2354
2926
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2355
2927
|
const unattributed = [];
|
|
2356
2928
|
for (const message of messages ?? []) {
|
|
2357
|
-
if (
|
|
2929
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2358
2930
|
const created = finiteNumber(createdOf(message));
|
|
2359
2931
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2360
2932
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2376,8 +2948,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2376
2948
|
}
|
|
2377
2949
|
function sumAssistantUsage(messages) {
|
|
2378
2950
|
if (!messages || messages.length === 0) return null;
|
|
2379
|
-
const
|
|
2380
|
-
const
|
|
2951
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
2952
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
2953
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2381
2954
|
let sawAnyUsage = false;
|
|
2382
2955
|
let inputSum = 0;
|
|
2383
2956
|
let outputSum = 0;
|
|
@@ -2431,13 +3004,11 @@ function sumAssistantUsage(messages) {
|
|
|
2431
3004
|
function messageUsage(messages, userMessageId) {
|
|
2432
3005
|
if (!messages || messages.length === 0) return null;
|
|
2433
3006
|
const byParentAll = messages.filter(
|
|
2434
|
-
(m) =>
|
|
3007
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2435
3008
|
);
|
|
2436
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2437
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2438
3009
|
let correlated;
|
|
2439
|
-
if (
|
|
2440
|
-
correlated =
|
|
3010
|
+
if (byParentAll.length > 0) {
|
|
3011
|
+
correlated = byParentAll;
|
|
2441
3012
|
} else {
|
|
2442
3013
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2443
3014
|
correlated = reply ? [reply] : [];
|
|
@@ -2447,7 +3018,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2447
3018
|
}
|
|
2448
3019
|
function messageRunState(messages, userMessageId) {
|
|
2449
3020
|
if (!messages || messages.length === 0) return "unknown";
|
|
2450
|
-
const hasUser = messages.some((m) =>
|
|
3021
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2451
3022
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2452
3023
|
if (!hasUser) {
|
|
2453
3024
|
if (!reply) return "unknown";
|
|
@@ -2483,14 +3054,26 @@ function messageError(messages, userMessageId) {
|
|
|
2483
3054
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2484
3055
|
const error2 = errorOf(reply);
|
|
2485
3056
|
if (error2 == null) return null;
|
|
2486
|
-
|
|
2487
|
-
if (typeof error2 === "
|
|
3057
|
+
let rendered;
|
|
3058
|
+
if (typeof error2 === "string") {
|
|
3059
|
+
rendered = error2;
|
|
3060
|
+
} else if (typeof error2 === "object") {
|
|
2488
3061
|
const e = error2;
|
|
2489
3062
|
const dataMessage = e.data?.message;
|
|
2490
|
-
if (typeof dataMessage === "string")
|
|
2491
|
-
|
|
3063
|
+
if (typeof dataMessage === "string") {
|
|
3064
|
+
rendered = dataMessage;
|
|
3065
|
+
} else if (typeof e.message === "string") {
|
|
3066
|
+
rendered = e.message;
|
|
3067
|
+
} else {
|
|
3068
|
+
rendered = "The agent run failed.";
|
|
3069
|
+
}
|
|
3070
|
+
} else {
|
|
3071
|
+
rendered = "The agent run failed.";
|
|
2492
3072
|
}
|
|
2493
|
-
|
|
3073
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3074
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3075
|
+
) : void 0;
|
|
3076
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
2494
3077
|
}
|
|
2495
3078
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2496
3079
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2539,7 +3122,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2539
3122
|
if (!messages || messages.length === 0) return null;
|
|
2540
3123
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2541
3124
|
const message = messages[i];
|
|
2542
|
-
if (
|
|
3125
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2543
3126
|
const created = createdOf(message);
|
|
2544
3127
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2545
3128
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2571,25 +3154,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2571
3154
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2572
3155
|
if (!messages || messages.length === 0) return false;
|
|
2573
3156
|
return messages.some(
|
|
2574
|
-
(m) =>
|
|
3157
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2575
3158
|
);
|
|
2576
3159
|
}
|
|
2577
3160
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2578
3161
|
if (!messages || messages.length === 0) return false;
|
|
2579
|
-
const userIndex = messages.findIndex((message) =>
|
|
3162
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2580
3163
|
if (userIndex === -1) return false;
|
|
2581
3164
|
let hasLaterUser = false;
|
|
2582
3165
|
let hasStartedLaterUser = false;
|
|
2583
3166
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2584
3167
|
const message = messages[i];
|
|
2585
|
-
if (
|
|
3168
|
+
if (roleOf2(message) !== "user") continue;
|
|
2586
3169
|
hasLaterUser = true;
|
|
2587
|
-
const laterUserMessageId =
|
|
3170
|
+
const laterUserMessageId = idOf2(message);
|
|
2588
3171
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2589
3172
|
return false;
|
|
2590
3173
|
}
|
|
2591
3174
|
if (messages.some(
|
|
2592
|
-
(candidate) =>
|
|
3175
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2593
3176
|
)) {
|
|
2594
3177
|
hasStartedLaterUser = true;
|
|
2595
3178
|
}
|
|
@@ -3060,13 +3643,17 @@ async function listV2Sessions(client) {
|
|
|
3060
3643
|
}
|
|
3061
3644
|
async function deleteV2Session(client, sessionId) {
|
|
3062
3645
|
try {
|
|
3063
|
-
await client.request(
|
|
3064
|
-
|
|
3646
|
+
const response = await client.request(
|
|
3647
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3648
|
+
{ method: "DELETE" },
|
|
3649
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3650
|
+
);
|
|
3651
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3065
3652
|
} catch (error2) {
|
|
3066
3653
|
console.error(
|
|
3067
3654
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3068
3655
|
);
|
|
3069
|
-
return false;
|
|
3656
|
+
return { ok: false, status: "threw" };
|
|
3070
3657
|
}
|
|
3071
3658
|
}
|
|
3072
3659
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3397,16 +3984,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3397
3984
|
].some((name) => headers.has(name));
|
|
3398
3985
|
}
|
|
3399
3986
|
async function getOpenAiUsage(port, client) {
|
|
3987
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3400
3988
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3401
3989
|
if (!credentials2) {
|
|
3402
3990
|
throw new OpenAiUsageError(
|
|
3403
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3991
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3404
3992
|
"no_credentials"
|
|
3405
3993
|
);
|
|
3406
3994
|
}
|
|
3407
3995
|
if (credentials2.expiresAt < Date.now()) {
|
|
3408
3996
|
throw new OpenAiUsageError(
|
|
3409
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3997
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3410
3998
|
"credentials_expired"
|
|
3411
3999
|
);
|
|
3412
4000
|
}
|
|
@@ -3687,9 +4275,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3687
4275
|
|
|
3688
4276
|
// src/lib/opencode/session-db-boot.ts
|
|
3689
4277
|
import { spawn as spawn2 } from "node:child_process";
|
|
3690
|
-
import { mkdirSync, statSync as
|
|
4278
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3691
4279
|
import { homedir as homedir4 } from "node:os";
|
|
3692
|
-
import { dirname as
|
|
4280
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3693
4281
|
|
|
3694
4282
|
// src/lib/runner-synchroniser.ts
|
|
3695
4283
|
import { spawn } from "node:child_process";
|
|
@@ -3769,9 +4357,12 @@ function runSynchroniser(args, opts) {
|
|
|
3769
4357
|
|
|
3770
4358
|
// src/lib/opencode/session-db-boot.ts
|
|
3771
4359
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3772
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4360
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3773
4361
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3774
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4362
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4363
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4364
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4365
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3775
4366
|
function commandError(result) {
|
|
3776
4367
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3777
4368
|
}
|
|
@@ -3814,7 +4405,7 @@ function clearMarker(options) {
|
|
|
3814
4405
|
function markNoReplicate(options, message) {
|
|
3815
4406
|
if (options.noReplicateMarker) {
|
|
3816
4407
|
try {
|
|
3817
|
-
mkdirSync(
|
|
4408
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3818
4409
|
writeFileSync(options.noReplicateMarker, "");
|
|
3819
4410
|
} catch (error2) {
|
|
3820
4411
|
options.log(
|
|
@@ -3939,7 +4530,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3939
4530
|
return null;
|
|
3940
4531
|
}
|
|
3941
4532
|
try {
|
|
3942
|
-
if (
|
|
4533
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3943
4534
|
} catch (error2) {
|
|
3944
4535
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3945
4536
|
options.log(
|
|
@@ -3969,7 +4560,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3969
4560
|
return null;
|
|
3970
4561
|
}
|
|
3971
4562
|
try {
|
|
3972
|
-
mkdirSync(
|
|
4563
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3973
4564
|
writeFileSync(configPath, rendered.stdout);
|
|
3974
4565
|
} catch (error2) {
|
|
3975
4566
|
options.log(
|
|
@@ -4082,20 +4673,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4082
4673
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4083
4674
|
env: {
|
|
4084
4675
|
...env,
|
|
4085
|
-
// The synchroniser reads
|
|
4086
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4676
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4087
4677
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4088
4678
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4679
|
+
),
|
|
4680
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4681
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4682
|
+
),
|
|
4683
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4684
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4685
|
+
),
|
|
4686
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4687
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4089
4688
|
)
|
|
4090
4689
|
}
|
|
4091
4690
|
});
|
|
4092
4691
|
logSynchroniserDiagnostics(result, options);
|
|
4093
4692
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4693
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4694
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4695
|
+
options.log(
|
|
4696
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4697
|
+
"error"
|
|
4698
|
+
);
|
|
4699
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4700
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4701
|
+
}
|
|
4094
4702
|
options.log(
|
|
4095
4703
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4096
4704
|
"warn"
|
|
4097
4705
|
);
|
|
4098
|
-
return false;
|
|
4706
|
+
return { verifyFatal: false };
|
|
4099
4707
|
}
|
|
4100
4708
|
if (result.code === 34) {
|
|
4101
4709
|
reportRecord(
|
|
@@ -4105,14 +4713,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4105
4713
|
null,
|
|
4106
4714
|
options
|
|
4107
4715
|
);
|
|
4108
|
-
return true;
|
|
4716
|
+
return { verifyFatal: true };
|
|
4109
4717
|
}
|
|
4110
4718
|
if (result.code === 33) {
|
|
4111
4719
|
options.log(
|
|
4112
4720
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4113
4721
|
"warn"
|
|
4114
4722
|
);
|
|
4115
|
-
return false;
|
|
4723
|
+
return { verifyFatal: false };
|
|
4116
4724
|
}
|
|
4117
4725
|
if (result.code !== 0) {
|
|
4118
4726
|
options.log(
|
|
@@ -4120,17 +4728,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4120
4728
|
"warn"
|
|
4121
4729
|
);
|
|
4122
4730
|
}
|
|
4123
|
-
return false;
|
|
4731
|
+
return { verifyFatal: false };
|
|
4124
4732
|
}
|
|
4125
4733
|
options.log(
|
|
4126
4734
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4127
4735
|
"debug"
|
|
4128
4736
|
);
|
|
4129
|
-
return false;
|
|
4737
|
+
return { verifyFatal: false };
|
|
4130
4738
|
}
|
|
4131
4739
|
function fileExists(path) {
|
|
4132
4740
|
try {
|
|
4133
|
-
|
|
4741
|
+
statSync3(path);
|
|
4134
4742
|
return true;
|
|
4135
4743
|
} catch (error2) {
|
|
4136
4744
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4213,13 +4821,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4213
4821
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4214
4822
|
return { verifyFatal: false };
|
|
4215
4823
|
}
|
|
4216
|
-
return
|
|
4824
|
+
return verifySessionDb(options, configPath, env);
|
|
4217
4825
|
}
|
|
4218
4826
|
|
|
4219
4827
|
// src/lib/opencode/session-db-provenance.ts
|
|
4220
4828
|
import { createRequire } from "node:module";
|
|
4221
4829
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4222
|
-
import { dirname as
|
|
4830
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4223
4831
|
var require2 = createRequire(import.meta.url);
|
|
4224
4832
|
function readSessionDbMigrationIds(dbPath) {
|
|
4225
4833
|
let db;
|
|
@@ -4285,7 +4893,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4285
4893
|
}
|
|
4286
4894
|
function saveSessionDbProvenanceState(path, state) {
|
|
4287
4895
|
try {
|
|
4288
|
-
mkdirSync2(
|
|
4896
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4289
4897
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4290
4898
|
`, "utf8");
|
|
4291
4899
|
} catch (error2) {
|
|
@@ -4866,7 +5474,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4866
5474
|
toDelete.push(s.id);
|
|
4867
5475
|
}
|
|
4868
5476
|
}
|
|
4869
|
-
|
|
5477
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5478
|
+
return toDelete.sort(
|
|
5479
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5480
|
+
);
|
|
4870
5481
|
}
|
|
4871
5482
|
var DEFAULT_INTERVAL = "1h";
|
|
4872
5483
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4926,13 +5537,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4926
5537
|
}
|
|
4927
5538
|
|
|
4928
5539
|
// src/lib/opencode/session-db-size.ts
|
|
4929
|
-
import { statSync as
|
|
5540
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4930
5541
|
import { join as join6 } from "node:path";
|
|
4931
5542
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4932
5543
|
function statSessionDbBytes(homeDir) {
|
|
4933
5544
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4934
5545
|
try {
|
|
4935
|
-
return
|
|
5546
|
+
return statSync4(dbPath).size;
|
|
4936
5547
|
} catch (err) {
|
|
4937
5548
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4938
5549
|
if (!isMissingFile) {
|
|
@@ -4958,7 +5569,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4958
5569
|
}
|
|
4959
5570
|
|
|
4960
5571
|
// src/lib/opencode/log-tail.ts
|
|
4961
|
-
import { statSync as
|
|
5572
|
+
import { statSync as statSync5 } from "node:fs";
|
|
4962
5573
|
import { homedir as homedir5 } from "node:os";
|
|
4963
5574
|
import { join as join7 } from "node:path";
|
|
4964
5575
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -4981,7 +5592,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4981
5592
|
let inode = null;
|
|
4982
5593
|
let baselineReady = true;
|
|
4983
5594
|
try {
|
|
4984
|
-
const initial =
|
|
5595
|
+
const initial = statSync5(logPath);
|
|
4985
5596
|
offset = initial.size;
|
|
4986
5597
|
inode = initial.ino;
|
|
4987
5598
|
} catch (error2) {
|
|
@@ -5040,129 +5651,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5040
5651
|
};
|
|
5041
5652
|
}
|
|
5042
5653
|
|
|
5043
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5044
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5045
|
-
import { dirname as dirname4 } from "node:path";
|
|
5046
|
-
function errorMessage(error2) {
|
|
5047
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5048
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5049
|
-
}
|
|
5050
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5051
|
-
try {
|
|
5052
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5053
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5054
|
-
if (availableBytes < requiredBytes) {
|
|
5055
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5056
|
-
}
|
|
5057
|
-
return null;
|
|
5058
|
-
} catch (err) {
|
|
5059
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5060
|
-
}
|
|
5061
|
-
}
|
|
5062
|
-
function readLogicalBytes(db) {
|
|
5063
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5064
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5065
|
-
return pageCount * pageSize;
|
|
5066
|
-
}
|
|
5067
|
-
function readCheckpointResult(db) {
|
|
5068
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5069
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5070
|
-
}
|
|
5071
|
-
async function probeReclaimAvailability(input) {
|
|
5072
|
-
const { dbPath, requiredBytes } = input;
|
|
5073
|
-
let sqlite;
|
|
5074
|
-
try {
|
|
5075
|
-
sqlite = await import("node:sqlite");
|
|
5076
|
-
} catch (err) {
|
|
5077
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5078
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5079
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5080
|
-
}
|
|
5081
|
-
let autoVacuum = null;
|
|
5082
|
-
try {
|
|
5083
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5084
|
-
try {
|
|
5085
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5086
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5087
|
-
} finally {
|
|
5088
|
-
db.close();
|
|
5089
|
-
}
|
|
5090
|
-
} catch (err) {
|
|
5091
|
-
console.warn(
|
|
5092
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5093
|
-
);
|
|
5094
|
-
}
|
|
5095
|
-
if (autoVacuum !== 0) return null;
|
|
5096
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5097
|
-
}
|
|
5098
|
-
async function reclaimSessionDbSpace(input) {
|
|
5099
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5100
|
-
let sqlite;
|
|
5101
|
-
try {
|
|
5102
|
-
sqlite = await import("node:sqlite");
|
|
5103
|
-
} catch (err) {
|
|
5104
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5105
|
-
console.warn(
|
|
5106
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5107
|
-
);
|
|
5108
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5109
|
-
}
|
|
5110
|
-
const { DatabaseSync } = sqlite;
|
|
5111
|
-
let db;
|
|
5112
|
-
try {
|
|
5113
|
-
db = new DatabaseSync(dbPath);
|
|
5114
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5115
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5116
|
-
if (autoVacuum === 0) {
|
|
5117
|
-
if (!allowFullVacuum) {
|
|
5118
|
-
console.warn(
|
|
5119
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5120
|
-
);
|
|
5121
|
-
return {
|
|
5122
|
-
ok: false,
|
|
5123
|
-
skipped: "full-vacuum-blocked",
|
|
5124
|
-
detail: "a session turn is live"
|
|
5125
|
-
};
|
|
5126
|
-
}
|
|
5127
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5128
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5129
|
-
if (skipReason !== null) {
|
|
5130
|
-
console.warn(
|
|
5131
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5132
|
-
);
|
|
5133
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5134
|
-
}
|
|
5135
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5136
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5137
|
-
db.exec("VACUUM");
|
|
5138
|
-
const afterBytes = readLogicalBytes(db);
|
|
5139
|
-
const checkpoint = readCheckpointResult(db);
|
|
5140
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5141
|
-
}
|
|
5142
|
-
if (autoVacuum === 2) {
|
|
5143
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5144
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5145
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5146
|
-
const afterBytes = readLogicalBytes(db);
|
|
5147
|
-
const checkpoint = readCheckpointResult(db);
|
|
5148
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5149
|
-
}
|
|
5150
|
-
console.warn(
|
|
5151
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5152
|
-
);
|
|
5153
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5154
|
-
} catch (err) {
|
|
5155
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5156
|
-
return {
|
|
5157
|
-
ok: false,
|
|
5158
|
-
skipped: "reclaim-error",
|
|
5159
|
-
detail: errorMessage(err)
|
|
5160
|
-
};
|
|
5161
|
-
} finally {
|
|
5162
|
-
db?.close();
|
|
5163
|
-
}
|
|
5164
|
-
}
|
|
5165
|
-
|
|
5166
5654
|
// src/lib/tunnel/connection.ts
|
|
5167
5655
|
import WebSocket2 from "ws";
|
|
5168
5656
|
|
|
@@ -5392,6 +5880,8 @@ var StreamForwarder = class {
|
|
|
5392
5880
|
if (done) break;
|
|
5393
5881
|
const chunk = Buffer.from(value);
|
|
5394
5882
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5883
|
+
await stream.pausePromise;
|
|
5884
|
+
if (ac.signal.aborted) break;
|
|
5395
5885
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5396
5886
|
if (this.binaryFramesSupported) {
|
|
5397
5887
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5434,6 +5924,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5434
5924
|
}
|
|
5435
5925
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5436
5926
|
var BASE_RECONNECT_DELAY = 500;
|
|
5927
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5437
5928
|
function getReconnectDelay(attempt) {
|
|
5438
5929
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5439
5930
|
const jitter = Math.random() * 1e3;
|
|
@@ -5505,10 +5996,27 @@ function connectTunnel(options) {
|
|
|
5505
5996
|
ws.close();
|
|
5506
5997
|
reject(new Error("Connection timeout"));
|
|
5507
5998
|
}, 3e4);
|
|
5999
|
+
let relaySilenceTimeout;
|
|
6000
|
+
let relaySilenceTimedOut = false;
|
|
6001
|
+
const clearRelaySilenceTimeout = () => {
|
|
6002
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6003
|
+
clearTimeout(relaySilenceTimeout);
|
|
6004
|
+
relaySilenceTimeout = void 0;
|
|
6005
|
+
};
|
|
6006
|
+
const armRelaySilenceTimeout = () => {
|
|
6007
|
+
clearRelaySilenceTimeout();
|
|
6008
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6009
|
+
relaySilenceTimeout = void 0;
|
|
6010
|
+
relaySilenceTimedOut = true;
|
|
6011
|
+
ws.terminate();
|
|
6012
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6013
|
+
relaySilenceTimeout.unref?.();
|
|
6014
|
+
};
|
|
5508
6015
|
let upgradeRejection = null;
|
|
5509
6016
|
let upgradeRejectionReason = null;
|
|
5510
6017
|
ws.on("unexpected-response", (_req, res) => {
|
|
5511
6018
|
clearTimeout(connectionTimeout);
|
|
6019
|
+
clearRelaySilenceTimeout();
|
|
5512
6020
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5513
6021
|
upgradeRejectionReason = reason;
|
|
5514
6022
|
const chunks = [];
|
|
@@ -5538,6 +6046,7 @@ function connectTunnel(options) {
|
|
|
5538
6046
|
onInfo?.("WebSocket connection established");
|
|
5539
6047
|
});
|
|
5540
6048
|
ws.on("message", (data, isBinary) => {
|
|
6049
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5541
6050
|
if (isBinary) {
|
|
5542
6051
|
try {
|
|
5543
6052
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5571,6 +6080,7 @@ function connectTunnel(options) {
|
|
|
5571
6080
|
switch (message.type) {
|
|
5572
6081
|
case "connected": {
|
|
5573
6082
|
clearTimeout(connectionTimeout);
|
|
6083
|
+
armRelaySilenceTimeout();
|
|
5574
6084
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5575
6085
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5576
6086
|
onConnected?.(connectedAgentId);
|
|
@@ -5595,6 +6105,7 @@ function connectTunnel(options) {
|
|
|
5595
6105
|
});
|
|
5596
6106
|
ws.on("error", (error2) => {
|
|
5597
6107
|
clearTimeout(connectionTimeout);
|
|
6108
|
+
clearRelaySilenceTimeout();
|
|
5598
6109
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5599
6110
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5600
6111
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5606,7 +6117,8 @@ function connectTunnel(options) {
|
|
|
5606
6117
|
);
|
|
5607
6118
|
});
|
|
5608
6119
|
ws.on("close", (code, reason) => {
|
|
5609
|
-
|
|
6120
|
+
clearRelaySilenceTimeout();
|
|
6121
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5610
6122
|
forwarder.abortAll();
|
|
5611
6123
|
onDisconnected?.(code, reasonStr);
|
|
5612
6124
|
});
|
|
@@ -5614,9 +6126,10 @@ function connectTunnel(options) {
|
|
|
5614
6126
|
}
|
|
5615
6127
|
|
|
5616
6128
|
// src/lib/tunnel/runner-connection.ts
|
|
5617
|
-
var RunnerConnection = class {
|
|
6129
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5618
6130
|
opts;
|
|
5619
6131
|
sleep;
|
|
6132
|
+
now;
|
|
5620
6133
|
connection = null;
|
|
5621
6134
|
resolvedAgentId;
|
|
5622
6135
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5625,10 +6138,17 @@ var RunnerConnection = class {
|
|
|
5625
6138
|
reconnectPromise = null;
|
|
5626
6139
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5627
6140
|
reconnectAttempt = 0;
|
|
6141
|
+
generation = 0;
|
|
6142
|
+
disconnectedAtMs = null;
|
|
6143
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6144
|
+
"No pong received",
|
|
6145
|
+
"Orphaned socket"
|
|
6146
|
+
]);
|
|
5628
6147
|
constructor(opts) {
|
|
5629
6148
|
this.opts = opts;
|
|
5630
6149
|
this.resolvedAgentId = opts.agentId;
|
|
5631
6150
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6151
|
+
this.now = opts.now ?? Date.now;
|
|
5632
6152
|
}
|
|
5633
6153
|
get agentId() {
|
|
5634
6154
|
return this.resolvedAgentId;
|
|
@@ -5664,14 +6184,24 @@ var RunnerConnection = class {
|
|
|
5664
6184
|
port: this.opts.port,
|
|
5665
6185
|
openCodePassword: this.opts.openCodePassword,
|
|
5666
6186
|
onConnected: (agentId) => {
|
|
6187
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5667
6188
|
this.reconnectAttempt = 0;
|
|
5668
6189
|
this.reconnecting = false;
|
|
5669
6190
|
this.resolvedAgentId = agentId;
|
|
5670
|
-
|
|
6191
|
+
this.generation++;
|
|
6192
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6193
|
+
this.disconnectedAtMs = null;
|
|
6194
|
+
events.onConnected(agentId, isReconnect, {
|
|
6195
|
+
generation: this.generation,
|
|
6196
|
+
attempt,
|
|
6197
|
+
downtimeMs
|
|
6198
|
+
});
|
|
5671
6199
|
},
|
|
5672
6200
|
onDisconnected: (code, reason) => {
|
|
5673
|
-
|
|
5674
|
-
|
|
6201
|
+
this.disconnectedAtMs = this.now();
|
|
6202
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6203
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6204
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5675
6205
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5676
6206
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5677
6207
|
});
|
|
@@ -6010,6 +6540,83 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6010
6540
|
};
|
|
6011
6541
|
}
|
|
6012
6542
|
|
|
6543
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6544
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6545
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6546
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6547
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6548
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6549
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6550
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6551
|
+
var BAND_RANK = {
|
|
6552
|
+
low: 1,
|
|
6553
|
+
very_low: 2,
|
|
6554
|
+
critical: 3
|
|
6555
|
+
};
|
|
6556
|
+
function bandFor(freeBytes) {
|
|
6557
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6558
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6559
|
+
return "low";
|
|
6560
|
+
}
|
|
6561
|
+
function formatBytes(bytes) {
|
|
6562
|
+
if (bytes === null) return "unavailable";
|
|
6563
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6564
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6565
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6566
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6567
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6568
|
+
}
|
|
6569
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6570
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6571
|
+
}
|
|
6572
|
+
function evaluateLowDiskWarning(input) {
|
|
6573
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6574
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6575
|
+
return { warning: null, state };
|
|
6576
|
+
}
|
|
6577
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6578
|
+
if (recovered) {
|
|
6579
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6580
|
+
}
|
|
6581
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6582
|
+
if (!low) return { warning: null, state };
|
|
6583
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6584
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6585
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6586
|
+
const worse = currentRank > previousRank;
|
|
6587
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6588
|
+
const nextState = {
|
|
6589
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6590
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6591
|
+
};
|
|
6592
|
+
return {
|
|
6593
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6594
|
+
state: nextState
|
|
6595
|
+
};
|
|
6596
|
+
}
|
|
6597
|
+
|
|
6598
|
+
// src/lib/opencode/session-store-health.ts
|
|
6599
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6600
|
+
function createSessionStoreHealth(options = {}) {
|
|
6601
|
+
const now = options.now ?? (() => Date.now());
|
|
6602
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6603
|
+
let lastUnwritableAt = null;
|
|
6604
|
+
let lastWritableAt = null;
|
|
6605
|
+
return {
|
|
6606
|
+
observeUnwritable: () => {
|
|
6607
|
+
lastUnwritableAt = now();
|
|
6608
|
+
},
|
|
6609
|
+
observeWritable: () => {
|
|
6610
|
+
lastWritableAt = now();
|
|
6611
|
+
},
|
|
6612
|
+
isUnwritable: () => {
|
|
6613
|
+
if (lastUnwritableAt === null) return false;
|
|
6614
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6615
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6616
|
+
}
|
|
6617
|
+
};
|
|
6618
|
+
}
|
|
6619
|
+
|
|
6013
6620
|
// src/lib/channels/driver.ts
|
|
6014
6621
|
import { homedir as homedir6 } from "node:os";
|
|
6015
6622
|
|
|
@@ -6466,8 +7073,12 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6466
7073
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6467
7074
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6468
7075
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7076
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7077
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7078
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
6469
7079
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6470
7080
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7081
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6471
7082
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6472
7083
|
var HEARTBEAT_MS = 6e4;
|
|
6473
7084
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6529,6 +7140,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6529
7140
|
maxActiveSessions;
|
|
6530
7141
|
watcherStallMs;
|
|
6531
7142
|
wedgeWarningIntervalMs;
|
|
7143
|
+
sessionStoreHealth;
|
|
6532
7144
|
openCodeClient;
|
|
6533
7145
|
/** Cache of conversationId → opencode sessionId. */
|
|
6534
7146
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6621,6 +7233,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6621
7233
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6622
7234
|
*/
|
|
6623
7235
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7236
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6624
7237
|
/**
|
|
6625
7238
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6626
7239
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6653,10 +7266,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6653
7266
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6654
7267
|
*/
|
|
6655
7268
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7269
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7270
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6656
7271
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6657
7272
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6658
7273
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6659
7274
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7275
|
+
/**
|
|
7276
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7277
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7278
|
+
* identical-text turn, so it never produces a correlation.
|
|
7279
|
+
*/
|
|
7280
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6660
7281
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6661
7282
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6662
7283
|
/**
|
|
@@ -6742,21 +7363,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6742
7363
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6743
7364
|
*/
|
|
6744
7365
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6745
|
-
/**
|
|
6746
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6747
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6748
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6749
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6750
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6751
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6752
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6753
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6754
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6755
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6756
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6757
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6758
|
-
*/
|
|
6759
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6760
7366
|
/**
|
|
6761
7367
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6762
7368
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6774,8 +7380,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6774
7380
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6775
7381
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6776
7382
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6777
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6778
|
-
*
|
|
7383
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7384
|
+
* the terminal-report fence and is never sent again).
|
|
6779
7385
|
*/
|
|
6780
7386
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6781
7387
|
/**
|
|
@@ -6898,6 +7504,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6898
7504
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6899
7505
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6900
7506
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7507
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6901
7508
|
}
|
|
6902
7509
|
get isV2() {
|
|
6903
7510
|
return this.openCodeClient.version === "v2";
|
|
@@ -6913,7 +7520,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6913
7520
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6914
7521
|
}
|
|
6915
7522
|
async listSessions() {
|
|
6916
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7523
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7524
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7525
|
+
});
|
|
6917
7526
|
}
|
|
6918
7527
|
async sessionExists(sessionId) {
|
|
6919
7528
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7090,6 +7699,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7090
7699
|
}
|
|
7091
7700
|
return ids;
|
|
7092
7701
|
}
|
|
7702
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7703
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7704
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7705
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7706
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7707
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7708
|
+
}
|
|
7709
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7710
|
+
}
|
|
7711
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7712
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7713
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7714
|
+
this.awaitingReadopt.delete(messageId);
|
|
7715
|
+
this.sessions.delete(conversationId);
|
|
7716
|
+
this.supersede(conversationId, sessionId);
|
|
7717
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7718
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7719
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7720
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7721
|
+
this.log({
|
|
7722
|
+
level: "error",
|
|
7723
|
+
message: errorMessage3,
|
|
7724
|
+
conversation_id: conversationId,
|
|
7725
|
+
message_id: messageId
|
|
7726
|
+
});
|
|
7727
|
+
const reportError = await this.reportUntrackableAck(
|
|
7728
|
+
conversationId,
|
|
7729
|
+
messageId,
|
|
7730
|
+
null,
|
|
7731
|
+
errorMessage3
|
|
7732
|
+
);
|
|
7733
|
+
if (reportError !== null) {
|
|
7734
|
+
this.log({
|
|
7735
|
+
level: "warn",
|
|
7736
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7737
|
+
conversation_id: conversationId,
|
|
7738
|
+
message_id: messageId
|
|
7739
|
+
});
|
|
7740
|
+
}
|
|
7741
|
+
if (source === "readopt") {
|
|
7742
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7743
|
+
}
|
|
7744
|
+
}
|
|
7093
7745
|
/**
|
|
7094
7746
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7095
7747
|
*
|
|
@@ -7269,6 +7921,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7269
7921
|
}
|
|
7270
7922
|
break;
|
|
7271
7923
|
}
|
|
7924
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7925
|
+
if (correlation) {
|
|
7926
|
+
correlation.attempts += 1;
|
|
7927
|
+
let correlatedId = null;
|
|
7928
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7929
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7930
|
+
conv,
|
|
7931
|
+
message,
|
|
7932
|
+
correlation.sessionId
|
|
7933
|
+
);
|
|
7934
|
+
if (polled.ok)
|
|
7935
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7936
|
+
} else if (correlation.knownUserIds === null) {
|
|
7937
|
+
this.log({
|
|
7938
|
+
level: "debug",
|
|
7939
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7940
|
+
conversation_id: conv.id,
|
|
7941
|
+
message_id: message.id
|
|
7942
|
+
});
|
|
7943
|
+
}
|
|
7944
|
+
if (correlatedId !== null) {
|
|
7945
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7946
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7947
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7948
|
+
this.dispatched.add(message.id);
|
|
7949
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7950
|
+
dispatched += 1;
|
|
7951
|
+
this.log({
|
|
7952
|
+
level: "info",
|
|
7953
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7954
|
+
conversation_id: conv.id,
|
|
7955
|
+
message_id: message.id
|
|
7956
|
+
});
|
|
7957
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7958
|
+
opencode_message_id: correlatedId
|
|
7959
|
+
});
|
|
7960
|
+
continue;
|
|
7961
|
+
}
|
|
7962
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7963
|
+
await this.handoffUntrackableCorrelation(
|
|
7964
|
+
conv.id,
|
|
7965
|
+
message.id,
|
|
7966
|
+
correlation.sessionId,
|
|
7967
|
+
correlation.attempts,
|
|
7968
|
+
"fresh"
|
|
7969
|
+
);
|
|
7970
|
+
break;
|
|
7971
|
+
}
|
|
7972
|
+
break;
|
|
7973
|
+
}
|
|
7272
7974
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7273
7975
|
if (effectiveOpencodeMessageId) {
|
|
7274
7976
|
const outcome = await this.resolveRedrive(
|
|
@@ -7290,6 +7992,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7290
7992
|
model: message.opencode_model ?? void 0
|
|
7291
7993
|
};
|
|
7292
7994
|
let opencodeMessageId;
|
|
7995
|
+
let preDispatchUserIds = null;
|
|
7293
7996
|
try {
|
|
7294
7997
|
this.log({
|
|
7295
7998
|
level: "info",
|
|
@@ -7319,7 +8022,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7319
8022
|
message.content,
|
|
7320
8023
|
options,
|
|
7321
8024
|
sendAttachments,
|
|
7322
|
-
this.openCodeClient
|
|
8025
|
+
this.openCodeClient,
|
|
8026
|
+
{
|
|
8027
|
+
onKnownUserIds: (ids) => {
|
|
8028
|
+
preDispatchUserIds = ids;
|
|
8029
|
+
},
|
|
8030
|
+
onKnownUserIdsError: (error2) => {
|
|
8031
|
+
this.log({
|
|
8032
|
+
level: "warn",
|
|
8033
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8034
|
+
conversation_id: conv.id,
|
|
8035
|
+
message_id: message.id
|
|
8036
|
+
});
|
|
8037
|
+
}
|
|
8038
|
+
}
|
|
7323
8039
|
)
|
|
7324
8040
|
);
|
|
7325
8041
|
} catch (err) {
|
|
@@ -7327,6 +8043,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7327
8043
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7328
8044
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7329
8045
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8046
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7330
8047
|
this.log({
|
|
7331
8048
|
level: "error",
|
|
7332
8049
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7402,39 +8119,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7402
8119
|
if (this.isV2) {
|
|
7403
8120
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7404
8121
|
}
|
|
7405
|
-
|
|
7406
|
-
|
|
7407
|
-
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
message_id: message.id
|
|
7412
|
-
});
|
|
7413
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7414
|
-
continue;
|
|
7415
|
-
}
|
|
7416
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7417
|
-
this.sessions.delete(conv.id);
|
|
7418
|
-
this.supersede(conv.id, sessionId);
|
|
7419
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8122
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8123
|
+
sessionId,
|
|
8124
|
+
content: message.content,
|
|
8125
|
+
attempts: 0,
|
|
8126
|
+
knownUserIds: preDispatchUserIds
|
|
8127
|
+
});
|
|
7420
8128
|
this.log({
|
|
7421
|
-
level: "
|
|
7422
|
-
message:
|
|
8129
|
+
level: "warn",
|
|
8130
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7423
8131
|
conversation_id: conv.id,
|
|
7424
8132
|
message_id: message.id
|
|
7425
8133
|
});
|
|
7426
|
-
|
|
7427
|
-
this.log({
|
|
7428
|
-
level: "warn",
|
|
7429
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7430
|
-
conversation_id: conv.id,
|
|
7431
|
-
message_id: message.id
|
|
7432
|
-
});
|
|
7433
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7434
|
-
});
|
|
8134
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7435
8135
|
break;
|
|
7436
8136
|
}
|
|
7437
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7438
8137
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7439
8138
|
this.releasedOpencodeIds.delete(message.id);
|
|
7440
8139
|
this.dispatched.add(message.id);
|
|
@@ -8039,23 +8738,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8039
8738
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8040
8739
|
return 1;
|
|
8041
8740
|
}
|
|
8042
|
-
/**
|
|
8043
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8044
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8045
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8046
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8047
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8048
|
-
* count, since a new session is a genuinely different attempt.
|
|
8049
|
-
*/
|
|
8050
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8051
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8052
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8053
|
-
existing.count += 1;
|
|
8054
|
-
return existing.count;
|
|
8055
|
-
}
|
|
8056
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8057
|
-
return 1;
|
|
8058
|
-
}
|
|
8059
8741
|
/** Record one queued give-up toward the never-started bound. */
|
|
8060
8742
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8061
8743
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8367,8 +9049,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8367
9049
|
b2LastDescendantCheckMs: 0,
|
|
8368
9050
|
b2RootOngoingHeldLogged: false,
|
|
8369
9051
|
b2AbandonedSignalled: false,
|
|
9052
|
+
b2AbandonedWithoutAnswer: false,
|
|
8370
9053
|
ambiguousPinnedSinceMs: 0,
|
|
8371
9054
|
ambiguousResolved: false,
|
|
9055
|
+
sessionStoreFailurePending: false,
|
|
8372
9056
|
neverStartedResolved: false
|
|
8373
9057
|
});
|
|
8374
9058
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8390,6 +9074,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8390
9074
|
this.bufferedSessionErrors.delete(oldest);
|
|
8391
9075
|
}
|
|
8392
9076
|
}
|
|
9077
|
+
retainSessionError(event) {
|
|
9078
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9079
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9080
|
+
reason: event.reason,
|
|
9081
|
+
observedAt: this.now()
|
|
9082
|
+
});
|
|
9083
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9084
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9085
|
+
if (typeof oldest !== "string") break;
|
|
9086
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9087
|
+
}
|
|
9088
|
+
}
|
|
9089
|
+
readRetainedSessionError(sessionId) {
|
|
9090
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9091
|
+
if (!retained) return null;
|
|
9092
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9093
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9094
|
+
return null;
|
|
9095
|
+
}
|
|
9096
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9097
|
+
return retained.reason;
|
|
9098
|
+
}
|
|
8393
9099
|
/**
|
|
8394
9100
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8395
9101
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8485,8 +9191,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8485
9191
|
b2LastDescendantCheckMs: 0,
|
|
8486
9192
|
b2RootOngoingHeldLogged: false,
|
|
8487
9193
|
b2AbandonedSignalled: false,
|
|
9194
|
+
b2AbandonedWithoutAnswer: false,
|
|
8488
9195
|
ambiguousPinnedSinceMs: 0,
|
|
8489
9196
|
ambiguousResolved: false,
|
|
9197
|
+
sessionStoreFailurePending: false,
|
|
8490
9198
|
neverStartedResolved: false
|
|
8491
9199
|
});
|
|
8492
9200
|
}
|
|
@@ -8714,9 +9422,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8714
9422
|
return;
|
|
8715
9423
|
}
|
|
8716
9424
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9425
|
+
this.retainSessionError(event);
|
|
8717
9426
|
this.log({
|
|
8718
|
-
level: "
|
|
8719
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9427
|
+
level: "warn",
|
|
9428
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8720
9429
|
conversation_id: watcher.conv.id
|
|
8721
9430
|
});
|
|
8722
9431
|
return;
|
|
@@ -8746,9 +9455,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8746
9455
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8747
9456
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8748
9457
|
if (state !== "queued") {
|
|
9458
|
+
this.retainSessionError(event);
|
|
8749
9459
|
this.log({
|
|
8750
|
-
level: "
|
|
8751
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9460
|
+
level: "warn",
|
|
9461
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8752
9462
|
conversation_id: watcher.conv.id,
|
|
8753
9463
|
message_id: inFlight.evidentMessageId
|
|
8754
9464
|
});
|
|
@@ -8900,6 +9610,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8900
9610
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8901
9611
|
this.neverStartedAttempts.delete(id);
|
|
8902
9612
|
}
|
|
9613
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9614
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9615
|
+
return;
|
|
9616
|
+
}
|
|
8903
9617
|
if (inFlight.neverStartedResolved) {
|
|
8904
9618
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8905
9619
|
inFlight.neverStartedResolved = false;
|
|
@@ -8973,7 +9687,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8973
9687
|
if (state === "failed") {
|
|
8974
9688
|
this.anchorDeliveryDeadline(inFlight);
|
|
8975
9689
|
if (!inFlight.done) {
|
|
8976
|
-
const
|
|
9690
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9691
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9692
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9693
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9694
|
+
const runnerDetails = [retainedError, storeError].filter(
|
|
9695
|
+
(detail) => detail !== null
|
|
9696
|
+
);
|
|
9697
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
8977
9698
|
this.log({
|
|
8978
9699
|
level: "error",
|
|
8979
9700
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9059,10 +9780,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9059
9780
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9060
9781
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9061
9782
|
inFlight.b2AbandonedSignalled = false;
|
|
9783
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9062
9784
|
}
|
|
9063
9785
|
} else {
|
|
9064
9786
|
if (inFlight.b2AbandonedSignalled) {
|
|
9065
|
-
|
|
9787
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9788
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9789
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9790
|
+
sessionId,
|
|
9791
|
+
watcher,
|
|
9792
|
+
inFlight,
|
|
9793
|
+
messages,
|
|
9794
|
+
reply?.info?.finish ?? reply?.finish
|
|
9795
|
+
);
|
|
9796
|
+
} else {
|
|
9797
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9798
|
+
}
|
|
9066
9799
|
return;
|
|
9067
9800
|
}
|
|
9068
9801
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9079,17 +9812,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9079
9812
|
descendantOngoing,
|
|
9080
9813
|
rootOngoing
|
|
9081
9814
|
})) {
|
|
9815
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9816
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9817
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9818
|
+
watched_for_ms: pinnedForMs,
|
|
9819
|
+
run_state: "failed",
|
|
9820
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9821
|
+
});
|
|
9822
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9823
|
+
return;
|
|
9824
|
+
}
|
|
9082
9825
|
inFlight.b2AbandonedSignalled = true;
|
|
9826
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9827
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9828
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9829
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9830
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9831
|
+
watched_for_ms: pinnedForMs,
|
|
9832
|
+
run_state: "failed",
|
|
9833
|
+
finish,
|
|
9834
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9835
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9836
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9837
|
+
});
|
|
9838
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9839
|
+
sessionId,
|
|
9840
|
+
watcher,
|
|
9841
|
+
inFlight,
|
|
9842
|
+
messages,
|
|
9843
|
+
finish
|
|
9844
|
+
);
|
|
9845
|
+
return;
|
|
9846
|
+
}
|
|
9083
9847
|
this.log({
|
|
9084
9848
|
level: "warn",
|
|
9085
9849
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9086
9850
|
conversation_id: conv.id,
|
|
9087
9851
|
message_id: id
|
|
9088
9852
|
});
|
|
9089
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9090
9853
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9091
9854
|
watched_for_ms: pinnedForMs,
|
|
9092
|
-
finish
|
|
9855
|
+
finish,
|
|
9093
9856
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9094
9857
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9095
9858
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9137,6 +9900,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9137
9900
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9138
9901
|
sessionOngoing: ongoing
|
|
9139
9902
|
})) {
|
|
9903
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9904
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9905
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9906
|
+
watched_for_ms: pinnedForMs,
|
|
9907
|
+
run_state: "failed",
|
|
9908
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9909
|
+
});
|
|
9910
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9911
|
+
return;
|
|
9912
|
+
}
|
|
9140
9913
|
inFlight.ambiguousResolved = true;
|
|
9141
9914
|
this.log({
|
|
9142
9915
|
level: "warn",
|
|
@@ -9361,6 +10134,74 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9361
10134
|
}
|
|
9362
10135
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9363
10136
|
}
|
|
10137
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10138
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10139
|
+
const conv = watcher.conv;
|
|
10140
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
10141
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10142
|
+
this.log({
|
|
10143
|
+
level: "warn",
|
|
10144
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10145
|
+
conversation_id: conv.id,
|
|
10146
|
+
message_id: inFlight.evidentMessageId
|
|
10147
|
+
});
|
|
10148
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10149
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10150
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10151
|
+
messages,
|
|
10152
|
+
inFlight.opencodeMessageId,
|
|
10153
|
+
inFlight.evidentMessageId
|
|
10154
|
+
);
|
|
10155
|
+
try {
|
|
10156
|
+
await this.markFailed(
|
|
10157
|
+
conv.id,
|
|
10158
|
+
inFlight.evidentMessageId,
|
|
10159
|
+
sessionId,
|
|
10160
|
+
error2,
|
|
10161
|
+
usage,
|
|
10162
|
+
void 0,
|
|
10163
|
+
usageAgentName,
|
|
10164
|
+
subagentInvocations
|
|
10165
|
+
);
|
|
10166
|
+
} catch (err) {
|
|
10167
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10168
|
+
if (err instanceof ChannelTerminalError) {
|
|
10169
|
+
this.log({
|
|
10170
|
+
level: "warn",
|
|
10171
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10172
|
+
conversation_id: conv.id,
|
|
10173
|
+
message_id: inFlight.evidentMessageId
|
|
10174
|
+
});
|
|
10175
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10176
|
+
return;
|
|
10177
|
+
}
|
|
10178
|
+
if (this.now() >= inFlight.deadline) {
|
|
10179
|
+
this.log({
|
|
10180
|
+
level: "warn",
|
|
10181
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10182
|
+
conversation_id: conv.id,
|
|
10183
|
+
message_id: inFlight.evidentMessageId
|
|
10184
|
+
});
|
|
10185
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10186
|
+
return;
|
|
10187
|
+
}
|
|
10188
|
+
this.log({
|
|
10189
|
+
level: "warn",
|
|
10190
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10191
|
+
conversation_id: conv.id,
|
|
10192
|
+
message_id: inFlight.evidentMessageId
|
|
10193
|
+
});
|
|
10194
|
+
return;
|
|
10195
|
+
}
|
|
10196
|
+
inFlight.done = true;
|
|
10197
|
+
await this.reportSubagentAuthFailures(
|
|
10198
|
+
watcher.conv.id,
|
|
10199
|
+
inFlight.opencodeMessageId,
|
|
10200
|
+
inFlight.evidentMessageId,
|
|
10201
|
+
messages
|
|
10202
|
+
);
|
|
10203
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10204
|
+
}
|
|
9364
10205
|
/**
|
|
9365
10206
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9366
10207
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9416,6 +10257,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9416
10257
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9417
10258
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9418
10259
|
}
|
|
10260
|
+
/**
|
|
10261
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10262
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10263
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10264
|
+
*/
|
|
10265
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10266
|
+
const conv = watcher.conv;
|
|
10267
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10268
|
+
this.log({
|
|
10269
|
+
level: "error",
|
|
10270
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10271
|
+
conversation_id: conv.id,
|
|
10272
|
+
message_id: inFlight.evidentMessageId
|
|
10273
|
+
});
|
|
10274
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10275
|
+
try {
|
|
10276
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10277
|
+
} catch (err) {
|
|
10278
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10279
|
+
if (err instanceof ChannelTerminalError) {
|
|
10280
|
+
this.log({
|
|
10281
|
+
level: "warn",
|
|
10282
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10283
|
+
conversation_id: conv.id,
|
|
10284
|
+
message_id: inFlight.evidentMessageId
|
|
10285
|
+
});
|
|
10286
|
+
this.recordReleasedOpencodeId(
|
|
10287
|
+
inFlight.evidentMessageId,
|
|
10288
|
+
sessionId,
|
|
10289
|
+
inFlight.opencodeMessageId
|
|
10290
|
+
);
|
|
10291
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10292
|
+
return;
|
|
10293
|
+
}
|
|
10294
|
+
if (this.now() >= inFlight.deadline) {
|
|
10295
|
+
this.log({
|
|
10296
|
+
level: "warn",
|
|
10297
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10298
|
+
conversation_id: conv.id,
|
|
10299
|
+
message_id: inFlight.evidentMessageId
|
|
10300
|
+
});
|
|
10301
|
+
this.recordReleasedOpencodeId(
|
|
10302
|
+
inFlight.evidentMessageId,
|
|
10303
|
+
sessionId,
|
|
10304
|
+
inFlight.opencodeMessageId
|
|
10305
|
+
);
|
|
10306
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10307
|
+
return;
|
|
10308
|
+
}
|
|
10309
|
+
this.log({
|
|
10310
|
+
level: "warn",
|
|
10311
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10312
|
+
conversation_id: conv.id,
|
|
10313
|
+
message_id: inFlight.evidentMessageId
|
|
10314
|
+
});
|
|
10315
|
+
return;
|
|
10316
|
+
}
|
|
10317
|
+
inFlight.done = true;
|
|
10318
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10319
|
+
}
|
|
9419
10320
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9420
10321
|
/**
|
|
9421
10322
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9454,11 +10355,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9454
10355
|
}
|
|
9455
10356
|
}
|
|
9456
10357
|
}
|
|
9457
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10358
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9458
10359
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10360
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10361
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10362
|
+
this.awaitingCorrelation.delete(id);
|
|
10363
|
+
this.log({
|
|
10364
|
+
level: "debug",
|
|
10365
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10366
|
+
message_id: id
|
|
10367
|
+
});
|
|
10368
|
+
}
|
|
10369
|
+
}
|
|
9459
10370
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9460
10371
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9461
10372
|
this.untrackableAck.delete(id);
|
|
10373
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9462
10374
|
this.untrackableAckSignalled.delete(id);
|
|
9463
10375
|
this.untrackableAckWarned.delete(id);
|
|
9464
10376
|
this.log({
|
|
@@ -9498,7 +10410,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9498
10410
|
bySession.set(row.opencode_session_id, list);
|
|
9499
10411
|
}
|
|
9500
10412
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9501
|
-
let messages;
|
|
10413
|
+
let messages = null;
|
|
9502
10414
|
try {
|
|
9503
10415
|
if (this.isV2) {
|
|
9504
10416
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9507,9 +10419,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9507
10419
|
level: "warn",
|
|
9508
10420
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9509
10421
|
});
|
|
9510
|
-
|
|
10422
|
+
} else {
|
|
10423
|
+
messages = snapshot;
|
|
9511
10424
|
}
|
|
9512
|
-
messages = snapshot;
|
|
9513
10425
|
} else {
|
|
9514
10426
|
const polled = await pollSessionMessagesForRedrive(
|
|
9515
10427
|
this.port,
|
|
@@ -9522,15 +10434,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9522
10434
|
level: "warn",
|
|
9523
10435
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9524
10436
|
});
|
|
9525
|
-
|
|
10437
|
+
} else {
|
|
10438
|
+
messages = polled.messages;
|
|
9526
10439
|
}
|
|
9527
|
-
messages = polled.messages;
|
|
9528
10440
|
}
|
|
9529
10441
|
} catch (err) {
|
|
9530
10442
|
this.log({
|
|
9531
10443
|
level: "warn",
|
|
9532
10444
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9533
10445
|
});
|
|
10446
|
+
}
|
|
10447
|
+
if (messages === null) {
|
|
10448
|
+
for (const row of sessionRows) {
|
|
10449
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10450
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10451
|
+
}
|
|
10452
|
+
}
|
|
9534
10453
|
continue;
|
|
9535
10454
|
}
|
|
9536
10455
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9540,6 +10459,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9540
10459
|
}
|
|
9541
10460
|
}
|
|
9542
10461
|
}
|
|
10462
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10463
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10464
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10465
|
+
if (!correlation) return false;
|
|
10466
|
+
correlation.attempts += 1;
|
|
10467
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10468
|
+
if (correlatedId !== null) {
|
|
10469
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10470
|
+
const conv = this.convForRow(sessionId, row);
|
|
10471
|
+
const message = this.queuedMessageForRow(row);
|
|
10472
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10473
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10474
|
+
this.dispatched.add(row.id);
|
|
10475
|
+
this.readopted.add(row.id);
|
|
10476
|
+
this.ensureWatcherRunning(sessionId);
|
|
10477
|
+
this.log({
|
|
10478
|
+
level: "info",
|
|
10479
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10480
|
+
conversation_id: row.conversation_id,
|
|
10481
|
+
message_id: row.id
|
|
10482
|
+
});
|
|
10483
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10484
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10485
|
+
opencode_message_id: correlatedId
|
|
10486
|
+
});
|
|
10487
|
+
return true;
|
|
10488
|
+
}
|
|
10489
|
+
if (correlation.knownUserIds === null) {
|
|
10490
|
+
this.log({
|
|
10491
|
+
level: "debug",
|
|
10492
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10493
|
+
conversation_id: row.conversation_id,
|
|
10494
|
+
message_id: row.id
|
|
10495
|
+
});
|
|
10496
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10497
|
+
this.log({
|
|
10498
|
+
level: "warn",
|
|
10499
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10500
|
+
conversation_id: row.conversation_id,
|
|
10501
|
+
message_id: row.id
|
|
10502
|
+
});
|
|
10503
|
+
}
|
|
10504
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10505
|
+
await this.handoffUntrackableCorrelation(
|
|
10506
|
+
row.conversation_id,
|
|
10507
|
+
row.id,
|
|
10508
|
+
correlation.sessionId,
|
|
10509
|
+
correlation.attempts,
|
|
10510
|
+
"readopt"
|
|
10511
|
+
);
|
|
10512
|
+
}
|
|
10513
|
+
return true;
|
|
10514
|
+
}
|
|
9543
10515
|
/**
|
|
9544
10516
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9545
10517
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9547,8 +10519,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9547
10519
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9548
10520
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9549
10521
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9550
|
-
* before the restart)
|
|
9551
|
-
*
|
|
10522
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10523
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10524
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10525
|
+
* `forceReadoptRun`):
|
|
9552
10526
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9553
10527
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9554
10528
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9557,7 +10531,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9557
10531
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9558
10532
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9559
10533
|
* tracking the stored id so the reply correlates by it;
|
|
9560
|
-
* - `unknown`/null id →
|
|
10534
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10535
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9561
10536
|
*
|
|
9562
10537
|
* Only `ChannelAuthError` propagates.
|
|
9563
10538
|
*/
|
|
@@ -9571,6 +10546,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9571
10546
|
});
|
|
9572
10547
|
return;
|
|
9573
10548
|
}
|
|
10549
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9574
10550
|
const ocId = row.opencode_message_id;
|
|
9575
10551
|
const state = messageRunState(messages, ocId ?? "");
|
|
9576
10552
|
if (state === "done") {
|
|
@@ -9637,12 +10613,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9637
10613
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9638
10614
|
this.dontRedispatch.delete(row.id);
|
|
9639
10615
|
this.untrackableAck.delete(row.id);
|
|
10616
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9640
10617
|
this.untrackableAckSignalled.delete(row.id);
|
|
9641
10618
|
this.untrackableAckWarned.delete(row.id);
|
|
9642
10619
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9643
10620
|
return;
|
|
9644
10621
|
}
|
|
9645
|
-
if (this.
|
|
10622
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10623
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10624
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10625
|
+
this.untrackableAckWarned.add(row.id);
|
|
10626
|
+
this.log({
|
|
10627
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10628
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10629
|
+
conversation_id: row.conversation_id,
|
|
10630
|
+
message_id: row.id
|
|
10631
|
+
});
|
|
10632
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10633
|
+
const reportError = await this.reportUntrackableAck(
|
|
10634
|
+
row.conversation_id,
|
|
10635
|
+
row.id,
|
|
10636
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10637
|
+
errorMessage3
|
|
10638
|
+
);
|
|
10639
|
+
if (reportError !== null) {
|
|
10640
|
+
this.log({
|
|
10641
|
+
level: "warn",
|
|
10642
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10643
|
+
conversation_id: row.conversation_id,
|
|
10644
|
+
message_id: row.id
|
|
10645
|
+
});
|
|
10646
|
+
}
|
|
10647
|
+
} else {
|
|
10648
|
+
this.log({
|
|
10649
|
+
level: "debug",
|
|
10650
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10651
|
+
conversation_id: row.conversation_id,
|
|
10652
|
+
message_id: row.id
|
|
10653
|
+
});
|
|
10654
|
+
}
|
|
10655
|
+
return;
|
|
10656
|
+
}
|
|
10657
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9646
10658
|
this.log({
|
|
9647
10659
|
level: "debug",
|
|
9648
10660
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9820,6 +10832,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9820
10832
|
}
|
|
9821
10833
|
this.dontRedispatch.delete(row.id);
|
|
9822
10834
|
this.untrackableAck.delete(row.id);
|
|
10835
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9823
10836
|
this.untrackableAckSignalled.delete(row.id);
|
|
9824
10837
|
this.untrackableAckWarned.delete(row.id);
|
|
9825
10838
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9839,8 +10852,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9839
10852
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9840
10853
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9841
10854
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9842
|
-
* re-entry)
|
|
9843
|
-
*
|
|
10855
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10856
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10857
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9844
10858
|
*
|
|
9845
10859
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9846
10860
|
* `processed_at` (Invariant 1).
|
|
@@ -9903,6 +10917,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9903
10917
|
);
|
|
9904
10918
|
}
|
|
9905
10919
|
let ocId;
|
|
10920
|
+
let preDispatchUserIds = null;
|
|
9906
10921
|
try {
|
|
9907
10922
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9908
10923
|
sessionId,
|
|
@@ -9912,7 +10927,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9912
10927
|
row.content,
|
|
9913
10928
|
options,
|
|
9914
10929
|
sendAttachments,
|
|
9915
|
-
this.openCodeClient
|
|
10930
|
+
this.openCodeClient,
|
|
10931
|
+
{
|
|
10932
|
+
onKnownUserIds: (ids) => {
|
|
10933
|
+
preDispatchUserIds = ids;
|
|
10934
|
+
},
|
|
10935
|
+
onKnownUserIdsError: (error2) => {
|
|
10936
|
+
this.log({
|
|
10937
|
+
level: "warn",
|
|
10938
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10939
|
+
conversation_id: row.conversation_id,
|
|
10940
|
+
message_id: row.id
|
|
10941
|
+
});
|
|
10942
|
+
}
|
|
10943
|
+
}
|
|
9916
10944
|
)
|
|
9917
10945
|
);
|
|
9918
10946
|
} catch (err) {
|
|
@@ -9945,6 +10973,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9945
10973
|
}
|
|
9946
10974
|
} else {
|
|
9947
10975
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
10976
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
9948
10977
|
}
|
|
9949
10978
|
this.log({
|
|
9950
10979
|
level: "error",
|
|
@@ -9993,39 +11022,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9993
11022
|
}
|
|
9994
11023
|
if (ocId === null) {
|
|
9995
11024
|
this.awaitingReadopt.delete(row.id);
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
this.log({
|
|
10003
|
-
level: "error",
|
|
10004
|
-
message: errorMessage3,
|
|
10005
|
-
conversation_id: row.conversation_id,
|
|
10006
|
-
message_id: row.id
|
|
10007
|
-
});
|
|
10008
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10009
|
-
this.log({
|
|
10010
|
-
level: "warn",
|
|
10011
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10012
|
-
conversation_id: row.conversation_id,
|
|
10013
|
-
message_id: row.id
|
|
10014
|
-
});
|
|
10015
|
-
});
|
|
10016
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10017
|
-
return;
|
|
10018
|
-
}
|
|
11025
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11026
|
+
sessionId,
|
|
11027
|
+
content: row.content,
|
|
11028
|
+
attempts: 0,
|
|
11029
|
+
knownUserIds: preDispatchUserIds
|
|
11030
|
+
});
|
|
10019
11031
|
this.log({
|
|
10020
11032
|
level: "warn",
|
|
10021
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11033
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10022
11034
|
conversation_id: row.conversation_id,
|
|
10023
11035
|
message_id: row.id
|
|
10024
11036
|
});
|
|
10025
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10026
11037
|
return;
|
|
10027
11038
|
}
|
|
10028
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10029
11039
|
const processedAtMs = this.processedAtMs(row);
|
|
10030
11040
|
this.registerReadopted(
|
|
10031
11041
|
readoptConv,
|
|
@@ -12147,6 +13157,7 @@ function createCredentialSync({
|
|
|
12147
13157
|
|
|
12148
13158
|
// src/commands/run.ts
|
|
12149
13159
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
13160
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12150
13161
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12151
13162
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12152
13163
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12365,7 +13376,9 @@ function displayStatus(state) {
|
|
|
12365
13376
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12366
13377
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12367
13378
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12368
|
-
const detail = last ? chalk7.dim(
|
|
13379
|
+
const detail = last ? chalk7.dim(
|
|
13380
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13381
|
+
) : "";
|
|
12369
13382
|
const agent = state.agentName ?? state.agentId;
|
|
12370
13383
|
console.log(
|
|
12371
13384
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12384,7 +13397,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12384
13397
|
logActivity(state, {
|
|
12385
13398
|
type: "info",
|
|
12386
13399
|
level: "warn",
|
|
12387
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13400
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12388
13401
|
});
|
|
12389
13402
|
}
|
|
12390
13403
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12617,9 +13630,9 @@ async function driveChannels(state, driver) {
|
|
|
12617
13630
|
}
|
|
12618
13631
|
}
|
|
12619
13632
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12620
|
-
var
|
|
13633
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12621
13634
|
function shouldWarnForReclaimSkip(reason) {
|
|
12622
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13635
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12623
13636
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12624
13637
|
if (!version2) return false;
|
|
12625
13638
|
const major = Number(version2[1]);
|
|
@@ -12676,61 +13689,96 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12676
13689
|
async function runSweep(state, driver, config) {
|
|
12677
13690
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12678
13691
|
try {
|
|
12679
|
-
const
|
|
12680
|
-
|
|
12681
|
-
|
|
13692
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13693
|
+
const sessions = enumeration.sessions;
|
|
13694
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13695
|
+
if (enumerationReason !== null) {
|
|
13696
|
+
logActivity(state, {
|
|
12682
13697
|
type: "info",
|
|
12683
|
-
|
|
12684
|
-
|
|
12685
|
-
outcome: "sweep-skipped",
|
|
12686
|
-
reason: "opencode-unreachable"
|
|
12687
|
-
}
|
|
13698
|
+
level: "warn",
|
|
13699
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12688
13700
|
});
|
|
12689
|
-
return;
|
|
12690
13701
|
}
|
|
12691
|
-
const toDelete = selectSessionsToDelete(
|
|
12692
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12693
|
-
{
|
|
12694
|
-
maxAgeMs: config.maxAgeMs,
|
|
12695
|
-
maxCount: config.maxCount,
|
|
12696
|
-
nowMs: Date.now(),
|
|
12697
|
-
protectedIds: driver.protectedSessionIds()
|
|
12698
|
-
}
|
|
12699
|
-
);
|
|
12700
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12701
13702
|
let deleted = 0;
|
|
12702
13703
|
let failed = 0;
|
|
13704
|
+
let failed4xx = 0;
|
|
13705
|
+
let failed5xx = 0;
|
|
13706
|
+
let failedOtherStatus = 0;
|
|
13707
|
+
let failedThrew = 0;
|
|
12703
13708
|
let skippedNewlyActive = 0;
|
|
12704
|
-
|
|
12705
|
-
|
|
12706
|
-
|
|
12707
|
-
|
|
12708
|
-
|
|
12709
|
-
|
|
12710
|
-
|
|
12711
|
-
|
|
13709
|
+
let remaining = 0;
|
|
13710
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13711
|
+
if (sessions !== null) {
|
|
13712
|
+
const toDelete = selectSessionsToDelete(
|
|
13713
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13714
|
+
{
|
|
13715
|
+
maxAgeMs: config.maxAgeMs,
|
|
13716
|
+
maxCount: config.maxCount,
|
|
13717
|
+
nowMs: Date.now(),
|
|
13718
|
+
protectedIds: protectedNow
|
|
13719
|
+
}
|
|
13720
|
+
);
|
|
13721
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13722
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13723
|
+
protectedNow = driver.protectedSessionIds();
|
|
13724
|
+
for (const id of deleteCandidates) {
|
|
13725
|
+
if (protectedNow.has(id)) {
|
|
13726
|
+
skippedNewlyActive++;
|
|
13727
|
+
logActivity(state, {
|
|
13728
|
+
type: "info",
|
|
13729
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13730
|
+
});
|
|
13731
|
+
continue;
|
|
13732
|
+
}
|
|
13733
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13734
|
+
if (outcome.ok) {
|
|
13735
|
+
deleted++;
|
|
13736
|
+
} else if (outcome.status === "threw") {
|
|
13737
|
+
failed++;
|
|
13738
|
+
failedThrew++;
|
|
13739
|
+
} else {
|
|
13740
|
+
failed++;
|
|
13741
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13742
|
+
if (statusClass === 4) failed4xx++;
|
|
13743
|
+
else if (statusClass === 5) failed5xx++;
|
|
13744
|
+
else failedOtherStatus++;
|
|
13745
|
+
}
|
|
12712
13746
|
}
|
|
12713
|
-
|
|
12714
|
-
|
|
13747
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13748
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13749
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13750
|
+
logActivity(state, {
|
|
13751
|
+
type: "info",
|
|
13752
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13753
|
+
});
|
|
12715
13754
|
}
|
|
12716
|
-
const
|
|
12717
|
-
|
|
12718
|
-
|
|
12719
|
-
|
|
12720
|
-
|
|
12721
|
-
|
|
13755
|
+
const sessionMetadata = {
|
|
13756
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13757
|
+
sessions_deleted: deleted,
|
|
13758
|
+
sessions_failed: failed,
|
|
13759
|
+
sessions_failed_4xx: failed4xx,
|
|
13760
|
+
sessions_failed_5xx: failed5xx,
|
|
13761
|
+
sessions_failed_other: failedOtherStatus,
|
|
13762
|
+
sessions_failed_threw: failedThrew,
|
|
13763
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13764
|
+
sessions_remaining: remaining,
|
|
13765
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13766
|
+
};
|
|
12722
13767
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12723
13768
|
dbPath: sessionDbPath(),
|
|
12724
|
-
|
|
13769
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13770
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13771
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12725
13772
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12726
13773
|
});
|
|
12727
13774
|
if (reclaimResult.ok) {
|
|
13775
|
+
state.sessionStoreHealth.observeWritable();
|
|
12728
13776
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12729
13777
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12730
13778
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12731
13779
|
logSessionCleanupActivity(state, "sweep", {
|
|
12732
13780
|
type: "info",
|
|
12733
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13781
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12734
13782
|
metadata: {
|
|
12735
13783
|
outcome: "reclaimed",
|
|
12736
13784
|
mode: reclaimResult.mode,
|
|
@@ -12738,25 +13786,27 @@ async function runSweep(state, driver, config) {
|
|
|
12738
13786
|
after_bytes: reclaimResult.afterBytes,
|
|
12739
13787
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12740
13788
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12741
|
-
|
|
12742
|
-
|
|
12743
|
-
|
|
12744
|
-
|
|
13789
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13790
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13791
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13792
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13793
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13794
|
+
...sessionMetadata
|
|
12745
13795
|
}
|
|
12746
13796
|
});
|
|
12747
13797
|
} else {
|
|
13798
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13799
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13800
|
+
}
|
|
12748
13801
|
logSessionCleanupActivity(state, "sweep", {
|
|
12749
13802
|
type: "info",
|
|
12750
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13803
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12751
13804
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12752
13805
|
metadata: {
|
|
12753
13806
|
outcome: "reclaim-skipped",
|
|
12754
13807
|
reason: reclaimResult.skipped,
|
|
12755
13808
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12756
|
-
|
|
12757
|
-
sessions_deleted: deleted,
|
|
12758
|
-
sessions_failed: failed,
|
|
12759
|
-
sessions_skipped_active: skippedNewlyActive
|
|
13809
|
+
...sessionMetadata
|
|
12760
13810
|
}
|
|
12761
13811
|
});
|
|
12762
13812
|
}
|
|
@@ -13013,6 +14063,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13013
14063
|
message: `Resource usage collection: ${warning2}`
|
|
13014
14064
|
});
|
|
13015
14065
|
}
|
|
14066
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14067
|
+
...usage,
|
|
14068
|
+
nowMs: Date.now(),
|
|
14069
|
+
state: state.lowDiskWarningState
|
|
14070
|
+
});
|
|
14071
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14072
|
+
if (lowDiskDecision.warning !== null) {
|
|
14073
|
+
logActivity(state, {
|
|
14074
|
+
type: "info",
|
|
14075
|
+
level: "warn",
|
|
14076
|
+
message: lowDiskDecision.warning
|
|
14077
|
+
});
|
|
14078
|
+
}
|
|
13016
14079
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13017
14080
|
if (result.ok) {
|
|
13018
14081
|
if (consecutiveFailures > 0) {
|
|
@@ -13259,6 +14322,7 @@ async function run(options) {
|
|
|
13259
14322
|
opencodeClient: null,
|
|
13260
14323
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13261
14324
|
sessionDbProvenanceAnomaly: false,
|
|
14325
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13262
14326
|
opencodeProcess: null,
|
|
13263
14327
|
stopOpenCodeLogTail: null,
|
|
13264
14328
|
litestreamProcess: null,
|
|
@@ -13276,10 +14340,12 @@ async function run(options) {
|
|
|
13276
14340
|
openaiUsageRearm: null,
|
|
13277
14341
|
resourceUsageTimer: null,
|
|
13278
14342
|
stopResourceUsageSampling: null,
|
|
14343
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13279
14344
|
credentialSync: null,
|
|
13280
14345
|
authHeader: ""
|
|
13281
14346
|
};
|
|
13282
14347
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14348
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13283
14349
|
if (options.credentialSyncMarker) {
|
|
13284
14350
|
state.credentialSync = createCredentialSync({
|
|
13285
14351
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13448,6 +14514,34 @@ async function run(options) {
|
|
|
13448
14514
|
} else {
|
|
13449
14515
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13450
14516
|
}
|
|
14517
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14518
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14519
|
+
log2(state, message, level);
|
|
14520
|
+
if (state.interactive && !state.json) {
|
|
14521
|
+
logActivity(state, { type: "info", level, message });
|
|
14522
|
+
}
|
|
14523
|
+
};
|
|
14524
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14525
|
+
if (vaultResolution.error) {
|
|
14526
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14527
|
+
logCredentialVaultResolution(
|
|
14528
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14529
|
+
"warn"
|
|
14530
|
+
);
|
|
14531
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14532
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14533
|
+
logCredentialVaultResolution(
|
|
14534
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14535
|
+
"info"
|
|
14536
|
+
);
|
|
14537
|
+
} else {
|
|
14538
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14539
|
+
logCredentialVaultResolution(
|
|
14540
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14541
|
+
"debug"
|
|
14542
|
+
);
|
|
14543
|
+
}
|
|
14544
|
+
}
|
|
13451
14545
|
if (options.restoreRunnerCredentials) {
|
|
13452
14546
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13453
14547
|
const credentialContext = {
|
|
@@ -13465,14 +14559,23 @@ async function run(options) {
|
|
|
13465
14559
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13466
14560
|
}
|
|
13467
14561
|
state.credentialSync?.arm();
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
createOpenCodeActivityForwarder(() => ({
|
|
14562
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14563
|
+
() => ({
|
|
13471
14564
|
agentId: state.agentId,
|
|
13472
14565
|
authHeader: state.authHeader
|
|
13473
|
-
})
|
|
13474
|
-
|
|
14566
|
+
}),
|
|
14567
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14568
|
+
);
|
|
14569
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14570
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
14571
|
+
openCodeActivityForwarder
|
|
14572
|
+
);
|
|
14573
|
+
state.stopOpenCodeLogTail = () => {
|
|
14574
|
+
openCodeActivityForwarder.flush();
|
|
14575
|
+
openCodeLogTail.stop();
|
|
14576
|
+
};
|
|
13475
14577
|
let sessionDbVerifyFatal = false;
|
|
14578
|
+
let sessionDbVerifyFatalReason;
|
|
13476
14579
|
if (!options.restoreSessionDb) {
|
|
13477
14580
|
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
13478
14581
|
} else {
|
|
@@ -13505,12 +14608,13 @@ async function run(options) {
|
|
|
13505
14608
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13506
14609
|
});
|
|
13507
14610
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14611
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13508
14612
|
}
|
|
13509
14613
|
}
|
|
13510
14614
|
reportSessionDbRecovery(state);
|
|
13511
14615
|
if (sessionDbVerifyFatal) {
|
|
13512
14616
|
throw new Error(
|
|
13513
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14617
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13514
14618
|
);
|
|
13515
14619
|
}
|
|
13516
14620
|
applyRunnerOpenCodeConfig({
|
|
@@ -13755,6 +14859,7 @@ async function run(options) {
|
|
|
13755
14859
|
getAuthHeader: () => state.authHeader,
|
|
13756
14860
|
conversationFilter: state.conversationFilter,
|
|
13757
14861
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14862
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13758
14863
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13759
14864
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13760
14865
|
fileSyncDirectories,
|
|
@@ -13780,12 +14885,13 @@ async function run(options) {
|
|
|
13780
14885
|
openCodePassword: state.opencodePassword,
|
|
13781
14886
|
isRunning: () => state.running,
|
|
13782
14887
|
events: {
|
|
13783
|
-
onConnected: (agentId, isReconnect) => {
|
|
14888
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13784
14889
|
state.connected = true;
|
|
13785
14890
|
state.agentId = agentId;
|
|
14891
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13786
14892
|
logActivity(state, {
|
|
13787
14893
|
type: "info",
|
|
13788
|
-
message:
|
|
14894
|
+
message: connectionMessage
|
|
13789
14895
|
});
|
|
13790
14896
|
if (options.tunnelReadyFile) {
|
|
13791
14897
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13806,7 +14912,12 @@ async function run(options) {
|
|
|
13806
14912
|
version: state.opencodeVersion,
|
|
13807
14913
|
major: state.opencodeApiVersion,
|
|
13808
14914
|
connected: state.opencodeConnected
|
|
13809
|
-
})
|
|
14915
|
+
}),
|
|
14916
|
+
connection_generation: transition.generation,
|
|
14917
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
14918
|
+
reconnect_attempt: transition.attempt,
|
|
14919
|
+
downtime_ms: transition.downtimeMs
|
|
14920
|
+
} : {}
|
|
13810
14921
|
});
|
|
13811
14922
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13812
14923
|
if (state.interactive) displayStatus(state);
|
|
@@ -13828,13 +14939,18 @@ async function run(options) {
|
|
|
13828
14939
|
if (state.interactive) displayStatus(state);
|
|
13829
14940
|
});
|
|
13830
14941
|
},
|
|
13831
|
-
onDisconnected: (code, reason) => {
|
|
14942
|
+
onDisconnected: (code, reason, transition) => {
|
|
13832
14943
|
state.connected = false;
|
|
14944
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13833
14945
|
logActivity(state, {
|
|
13834
14946
|
type: "info",
|
|
13835
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
14947
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14948
|
+
});
|
|
14949
|
+
emitAgentDisconnected(state.agentId, {
|
|
14950
|
+
code,
|
|
14951
|
+
reason: safeReason2,
|
|
14952
|
+
connection_generation: transition.generation
|
|
13836
14953
|
});
|
|
13837
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13838
14954
|
if (state.interactive) displayStatus(state);
|
|
13839
14955
|
},
|
|
13840
14956
|
onError: (error2) => {
|
|
@@ -13934,7 +15050,7 @@ async function run(options) {
|
|
|
13934
15050
|
},
|
|
13935
15051
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13936
15052
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13937
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
15053
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13938
15054
|
firstDelayMs: firstReportDelayMs,
|
|
13939
15055
|
nextDelayMs: usageReportDelayMs,
|
|
13940
15056
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13977,7 +15093,7 @@ async function run(options) {
|
|
|
13977
15093
|
}
|
|
13978
15094
|
|
|
13979
15095
|
// src/index.ts
|
|
13980
|
-
var
|
|
15096
|
+
var version = getCliVersion();
|
|
13981
15097
|
var program = new Command();
|
|
13982
15098
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13983
15099
|
"--endpoint <url>",
|
|
@@ -13995,6 +15111,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
13995
15111
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
13996
15112
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
13997
15113
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15114
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
13998
15115
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
13999
15116
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14000
15117
|
"-a, --agent [id]",
|