@evident-ai/cli 3.5.2-dev.43244a2 → 3.5.2-dev.4f7b3ac
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1718 -512
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1063,13 +1157,83 @@ async function claudeUsage() {
|
|
|
1063
1157
|
|
|
1064
1158
|
// src/commands/run.ts
|
|
1065
1159
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
|
-
import { homedir as
|
|
1160
|
+
import { homedir as homedir7 } from "node:os";
|
|
1067
1161
|
import { isAbsolute as isAbsolute3, join as join11, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1162
|
import chalk7 from "chalk";
|
|
1069
1163
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1158
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1323
|
|
|
1160
1324
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.4f7b3ac" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1326
|
function getCliVersion() {
|
|
1163
1327
|
return CLI_VERSION;
|
|
1164
1328
|
}
|
|
@@ -1172,6 +1336,23 @@ var authProvider = null;
|
|
|
1172
1336
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1337
|
authProvider = provider;
|
|
1174
1338
|
}
|
|
1339
|
+
var diagnosticsReporter = null;
|
|
1340
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1341
|
+
diagnosticsReporter = reporter;
|
|
1342
|
+
}
|
|
1343
|
+
function reportTelemetryDiagnostic(message) {
|
|
1344
|
+
if (!diagnosticsReporter) {
|
|
1345
|
+
console.error(message);
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
diagnosticsReporter(message);
|
|
1350
|
+
} catch (error2) {
|
|
1351
|
+
console.error(
|
|
1352
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1175
1356
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1357
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1358
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1418,9 @@ async function flushEvents() {
|
|
|
1237
1418
|
signal: controller.signal
|
|
1238
1419
|
});
|
|
1239
1420
|
if (!response.ok) {
|
|
1240
|
-
|
|
1421
|
+
reportTelemetryDiagnostic(
|
|
1422
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1423
|
+
);
|
|
1241
1424
|
}
|
|
1242
1425
|
} finally {
|
|
1243
1426
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1430,9 @@ async function flushEvents() {
|
|
|
1247
1430
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1431
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1432
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1433
|
+
reportTelemetryDiagnostic(
|
|
1434
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1435
|
+
);
|
|
1251
1436
|
lastFlushFailureLoggedAt = now;
|
|
1252
1437
|
suppressedFlushFailureCount = 0;
|
|
1253
1438
|
} else {
|
|
@@ -1299,8 +1484,227 @@ var EventTypes = {
|
|
|
1299
1484
|
};
|
|
1300
1485
|
|
|
1301
1486
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
|
-
|
|
1487
|
+
import { homedir as homedir2 } from "node:os";
|
|
1488
|
+
|
|
1489
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1490
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1491
|
+
import { dirname as dirname2 } from "node:path";
|
|
1492
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1493
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1494
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1495
|
+
function errorMessage(error2) {
|
|
1496
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1497
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1498
|
+
}
|
|
1499
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1500
|
+
const messages = [];
|
|
1501
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1502
|
+
let current = error2;
|
|
1503
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1504
|
+
visited.add(current);
|
|
1505
|
+
if (typeof current === "string") {
|
|
1506
|
+
messages.push(current);
|
|
1507
|
+
break;
|
|
1508
|
+
}
|
|
1509
|
+
if (current instanceof Error) {
|
|
1510
|
+
messages.push(current.name, current.message);
|
|
1511
|
+
current = current.cause;
|
|
1512
|
+
continue;
|
|
1513
|
+
}
|
|
1514
|
+
if (typeof current === "object") {
|
|
1515
|
+
const record = current;
|
|
1516
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1517
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1518
|
+
current = record.cause;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
break;
|
|
1522
|
+
}
|
|
1523
|
+
return messages.some(
|
|
1524
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1525
|
+
message
|
|
1526
|
+
)
|
|
1527
|
+
);
|
|
1528
|
+
}
|
|
1529
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1530
|
+
try {
|
|
1531
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1532
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1533
|
+
if (availableBytes < requiredBytes) {
|
|
1534
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1535
|
+
}
|
|
1536
|
+
return null;
|
|
1537
|
+
} catch (err) {
|
|
1538
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
function readLogicalBytes(db) {
|
|
1542
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1543
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1544
|
+
return pageCount * pageSize;
|
|
1545
|
+
}
|
|
1546
|
+
function readFreelistCount(db) {
|
|
1547
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1548
|
+
}
|
|
1549
|
+
function readCheckpointResult(db) {
|
|
1550
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1551
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1552
|
+
}
|
|
1553
|
+
async function probeReclaimAvailability(input) {
|
|
1554
|
+
const { dbPath, requiredBytes } = input;
|
|
1555
|
+
let sqlite;
|
|
1556
|
+
try {
|
|
1557
|
+
sqlite = await import("node:sqlite");
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1560
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1561
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1562
|
+
}
|
|
1563
|
+
let autoVacuum = null;
|
|
1564
|
+
try {
|
|
1565
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1566
|
+
try {
|
|
1567
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1568
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1569
|
+
} finally {
|
|
1570
|
+
db.close();
|
|
1571
|
+
}
|
|
1572
|
+
} catch (err) {
|
|
1573
|
+
console.warn(
|
|
1574
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1575
|
+
);
|
|
1576
|
+
}
|
|
1577
|
+
if (autoVacuum !== 0) return null;
|
|
1578
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1579
|
+
}
|
|
1580
|
+
async function reclaimSessionDbSpace(input) {
|
|
1581
|
+
const {
|
|
1582
|
+
dbPath,
|
|
1583
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1584
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1585
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1586
|
+
now = () => performance.now(),
|
|
1587
|
+
allowFullVacuum = true
|
|
1588
|
+
} = input;
|
|
1589
|
+
let sqlite;
|
|
1590
|
+
try {
|
|
1591
|
+
sqlite = await import("node:sqlite");
|
|
1592
|
+
} catch (err) {
|
|
1593
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1594
|
+
console.warn(
|
|
1595
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1596
|
+
);
|
|
1597
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1598
|
+
}
|
|
1599
|
+
const { DatabaseSync } = sqlite;
|
|
1600
|
+
let db;
|
|
1601
|
+
try {
|
|
1602
|
+
db = new DatabaseSync(dbPath);
|
|
1603
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1604
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1605
|
+
if (autoVacuum === 0) {
|
|
1606
|
+
if (!allowFullVacuum) {
|
|
1607
|
+
console.warn(
|
|
1608
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1609
|
+
);
|
|
1610
|
+
return {
|
|
1611
|
+
ok: false,
|
|
1612
|
+
skipped: "full-vacuum-blocked",
|
|
1613
|
+
detail: "a session turn is live"
|
|
1614
|
+
};
|
|
1615
|
+
}
|
|
1616
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1617
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1618
|
+
if (skipReason !== null) {
|
|
1619
|
+
console.warn(
|
|
1620
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1621
|
+
);
|
|
1622
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1623
|
+
}
|
|
1624
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1625
|
+
const freelistBefore = readFreelistCount(db);
|
|
1626
|
+
const startedAt = now();
|
|
1627
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1628
|
+
db.exec("VACUUM");
|
|
1629
|
+
const afterBytes = readLogicalBytes(db);
|
|
1630
|
+
const freelistAfter = readFreelistCount(db);
|
|
1631
|
+
const checkpoint = readCheckpointResult(db);
|
|
1632
|
+
return {
|
|
1633
|
+
ok: true,
|
|
1634
|
+
mode: "convert",
|
|
1635
|
+
beforeBytes,
|
|
1636
|
+
afterBytes,
|
|
1637
|
+
freelistBefore,
|
|
1638
|
+
freelistAfter,
|
|
1639
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1640
|
+
stoppedBy: "freelist-drained",
|
|
1641
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1642
|
+
checkpoint
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
if (autoVacuum === 2) {
|
|
1646
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1647
|
+
const freelistBefore = readFreelistCount(db);
|
|
1648
|
+
const startedAt = now();
|
|
1649
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1650
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1651
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1652
|
+
let pagesAttempted = 0;
|
|
1653
|
+
let freelistAfter = freelistBefore;
|
|
1654
|
+
let stoppedBy = "freelist-drained";
|
|
1655
|
+
while (freelistAfter > 0) {
|
|
1656
|
+
if (pagesAttempted >= pageLimit) {
|
|
1657
|
+
stoppedBy = "page-ceiling";
|
|
1658
|
+
break;
|
|
1659
|
+
}
|
|
1660
|
+
if (now() - startedAt >= budget) {
|
|
1661
|
+
stoppedBy = "time-budget";
|
|
1662
|
+
break;
|
|
1663
|
+
}
|
|
1664
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1665
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1666
|
+
pagesAttempted += pages;
|
|
1667
|
+
freelistAfter = readFreelistCount(db);
|
|
1668
|
+
}
|
|
1669
|
+
const afterBytes = readLogicalBytes(db);
|
|
1670
|
+
const checkpoint = readCheckpointResult(db);
|
|
1671
|
+
return {
|
|
1672
|
+
ok: true,
|
|
1673
|
+
mode: "incremental",
|
|
1674
|
+
beforeBytes,
|
|
1675
|
+
afterBytes,
|
|
1676
|
+
freelistBefore,
|
|
1677
|
+
freelistAfter,
|
|
1678
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1679
|
+
stoppedBy,
|
|
1680
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1681
|
+
checkpoint
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1684
|
+
console.warn(
|
|
1685
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1686
|
+
);
|
|
1687
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1688
|
+
} catch (err) {
|
|
1689
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1690
|
+
return {
|
|
1691
|
+
ok: false,
|
|
1692
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1693
|
+
detail: errorMessage(err)
|
|
1694
|
+
};
|
|
1695
|
+
} finally {
|
|
1696
|
+
db?.close();
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1701
|
+
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1702
|
+
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1703
|
+
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1704
|
+
"cli.session-cleanup": /* @__PURE__ */ new Set(["info", "warn", "error"])
|
|
1705
|
+
};
|
|
1303
1706
|
var SEVERITY_BY_LEVEL = {
|
|
1707
|
+
info: "info",
|
|
1304
1708
|
warn: "warning",
|
|
1305
1709
|
error: "error"
|
|
1306
1710
|
};
|
|
@@ -1310,26 +1714,85 @@ function parseOpenCodeLogLine(line) {
|
|
|
1310
1714
|
if (!levelMatch) return null;
|
|
1311
1715
|
const level = levelMatch[1].toUpperCase();
|
|
1312
1716
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1313
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1314
|
-
|
|
1717
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1718
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1719
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1720
|
+
const parsed = {
|
|
1721
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1722
|
+
sessionID: sessionMatch?.[1],
|
|
1723
|
+
messageID: messageIDMatch?.[1]
|
|
1724
|
+
};
|
|
1725
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1726
|
+
if (message !== void 0) parsed.message = message;
|
|
1727
|
+
return parsed;
|
|
1315
1728
|
}
|
|
1316
1729
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1317
|
-
|
|
1730
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1731
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1318
1732
|
let buffer = Buffer.alloc(0);
|
|
1319
|
-
const
|
|
1320
|
-
|
|
1321
|
-
|
|
1733
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1734
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1735
|
+
const metadata = {
|
|
1736
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1737
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1738
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1739
|
+
};
|
|
1322
1740
|
forwardRunnerActivity(
|
|
1323
1741
|
{
|
|
1324
1742
|
level: parsed.level,
|
|
1325
1743
|
error: line,
|
|
1326
|
-
metadata:
|
|
1744
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1327
1745
|
source: "opencode"
|
|
1328
1746
|
},
|
|
1329
1747
|
getContext()
|
|
1330
1748
|
);
|
|
1331
1749
|
};
|
|
1332
|
-
|
|
1750
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1751
|
+
for (const [key, state] of repeatStates) {
|
|
1752
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1753
|
+
continue;
|
|
1754
|
+
}
|
|
1755
|
+
if (state.repeatsSuppressed > 0) {
|
|
1756
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1757
|
+
}
|
|
1758
|
+
repeatStates.delete(key);
|
|
1759
|
+
}
|
|
1760
|
+
};
|
|
1761
|
+
const flushLine = (line) => {
|
|
1762
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1763
|
+
try {
|
|
1764
|
+
onSessionStoreWriteFailure();
|
|
1765
|
+
} catch (err) {
|
|
1766
|
+
reportTelemetryDiagnostic(
|
|
1767
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1768
|
+
);
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1772
|
+
if (!parsed) return;
|
|
1773
|
+
const now = Date.now();
|
|
1774
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1775
|
+
expireRepeatStates(now, key);
|
|
1776
|
+
if (key === void 0) {
|
|
1777
|
+
forwardLine(line, parsed);
|
|
1778
|
+
return;
|
|
1779
|
+
}
|
|
1780
|
+
const previous = repeatStates.get(key);
|
|
1781
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1782
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1783
|
+
repeatStates.set(key, {
|
|
1784
|
+
line,
|
|
1785
|
+
parsed,
|
|
1786
|
+
lastForwardedAt: now,
|
|
1787
|
+
repeatsSuppressed: 0
|
|
1788
|
+
});
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
previous.line = line;
|
|
1792
|
+
previous.parsed = parsed;
|
|
1793
|
+
previous.repeatsSuppressed++;
|
|
1794
|
+
};
|
|
1795
|
+
const forward = ((chunk) => {
|
|
1333
1796
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1334
1797
|
let newlineIndex;
|
|
1335
1798
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1340,14 +1803,44 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1340
1803
|
flushLine(buffer.toString("utf-8"));
|
|
1341
1804
|
buffer = Buffer.alloc(0);
|
|
1342
1805
|
}
|
|
1806
|
+
});
|
|
1807
|
+
forward.flush = () => {
|
|
1808
|
+
for (const state of repeatStates.values()) {
|
|
1809
|
+
if (state.repeatsSuppressed > 0) {
|
|
1810
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1811
|
+
}
|
|
1812
|
+
}
|
|
1813
|
+
repeatStates.clear();
|
|
1343
1814
|
};
|
|
1815
|
+
return forward;
|
|
1344
1816
|
}
|
|
1345
1817
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1346
1818
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1347
1819
|
var MAX_METADATA_ENTRIES = 20;
|
|
1348
1820
|
var TRUNCATION_MARKER = "\u2026";
|
|
1349
|
-
function
|
|
1350
|
-
|
|
1821
|
+
function redactHomeDirectory(message) {
|
|
1822
|
+
const home = homedir2();
|
|
1823
|
+
if (!home || home === "/") {
|
|
1824
|
+
return message;
|
|
1825
|
+
}
|
|
1826
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1827
|
+
let result = "";
|
|
1828
|
+
let searchFrom = 0;
|
|
1829
|
+
let homeIndex;
|
|
1830
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1831
|
+
const homeEnd = homeIndex + home.length;
|
|
1832
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1833
|
+
const nextCharacter = message[homeEnd];
|
|
1834
|
+
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1835
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1836
|
+
result += atPathBoundary ? "~" : home;
|
|
1837
|
+
searchFrom = homeEnd;
|
|
1838
|
+
}
|
|
1839
|
+
return result + message.slice(searchFrom);
|
|
1840
|
+
}
|
|
1841
|
+
function redactDiagnosticText(message) {
|
|
1842
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1843
|
+
return redactHomeDirectory(redacted);
|
|
1351
1844
|
}
|
|
1352
1845
|
function truncate(message) {
|
|
1353
1846
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
@@ -1357,14 +1850,17 @@ function sanitiseMetadata(metadata) {
|
|
|
1357
1850
|
if (!metadata) return {};
|
|
1358
1851
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1359
1852
|
if (Object.keys(metadata).length > entries.length) {
|
|
1360
|
-
|
|
1361
|
-
`
|
|
1853
|
+
reportTelemetryDiagnostic(
|
|
1854
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1362
1855
|
);
|
|
1363
1856
|
}
|
|
1364
1857
|
const sanitised = [];
|
|
1365
1858
|
for (const [key, value] of entries) {
|
|
1366
1859
|
if (typeof value === "string") {
|
|
1367
|
-
sanitised.push([
|
|
1860
|
+
sanitised.push([
|
|
1861
|
+
key,
|
|
1862
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1863
|
+
]);
|
|
1368
1864
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1369
1865
|
sanitised.push([key, value]);
|
|
1370
1866
|
}
|
|
@@ -1382,8 +1878,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1382
1878
|
}
|
|
1383
1879
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1384
1880
|
if (window.windowDroppedCount > 0) {
|
|
1385
|
-
|
|
1386
|
-
`
|
|
1881
|
+
reportTelemetryDiagnostic(
|
|
1882
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1387
1883
|
);
|
|
1388
1884
|
}
|
|
1389
1885
|
window.windowStartedAt = now;
|
|
@@ -1393,8 +1889,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1393
1889
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1394
1890
|
window.windowDroppedCount++;
|
|
1395
1891
|
if (window.windowDroppedCount === 1) {
|
|
1396
|
-
|
|
1397
|
-
`
|
|
1892
|
+
reportTelemetryDiagnostic(
|
|
1893
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1398
1894
|
);
|
|
1399
1895
|
}
|
|
1400
1896
|
return false;
|
|
@@ -1404,12 +1900,12 @@ function admitUnderRateLimit(source, now) {
|
|
|
1404
1900
|
}
|
|
1405
1901
|
function forwardRunnerActivity(entry, context) {
|
|
1406
1902
|
try {
|
|
1407
|
-
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1408
|
-
if (!context.agentId || !context.authHeader) return;
|
|
1409
1903
|
const source = entry.source ?? "cli.run";
|
|
1904
|
+
if (!FORWARDED_LEVELS_BY_SOURCE[source].has(entry.level)) return;
|
|
1905
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1410
1906
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1411
1907
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1412
|
-
const message = truncate(
|
|
1908
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1413
1909
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1414
1910
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1415
1911
|
message,
|
|
@@ -1417,8 +1913,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1417
1913
|
agentId: context.agentId
|
|
1418
1914
|
});
|
|
1419
1915
|
} catch (err) {
|
|
1420
|
-
|
|
1421
|
-
`
|
|
1916
|
+
reportTelemetryDiagnostic(
|
|
1917
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1422
1918
|
);
|
|
1423
1919
|
}
|
|
1424
1920
|
}
|
|
@@ -1558,12 +2054,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1558
2054
|
metadata: withoutContractFields(record),
|
|
1559
2055
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1560
2056
|
};
|
|
1561
|
-
case "session_db_boot_refused":
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
2057
|
+
case "session_db_boot_refused": {
|
|
2058
|
+
const message = (() => {
|
|
2059
|
+
switch (record.reason) {
|
|
2060
|
+
case "local_discard_failed":
|
|
2061
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2062
|
+
case "replica_separation_unproven":
|
|
2063
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2064
|
+
case "verification_timeout_before_separation":
|
|
2065
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2066
|
+
case "verification_timeout_during_separation":
|
|
2067
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2068
|
+
default:
|
|
2069
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2070
|
+
}
|
|
2071
|
+
})();
|
|
2072
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2073
|
+
}
|
|
1567
2074
|
case "schema_provenance_mismatch":
|
|
1568
2075
|
return {
|
|
1569
2076
|
level,
|
|
@@ -1708,8 +2215,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1708
2215
|
|
|
1709
2216
|
// src/lib/http-timeout.ts
|
|
1710
2217
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2218
|
+
var RequestTimeoutError = class extends Error {
|
|
2219
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2220
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2221
|
+
this.operation = operation;
|
|
2222
|
+
this.timeoutMs = timeoutMs;
|
|
2223
|
+
this.elapsedMs = elapsedMs;
|
|
2224
|
+
this.name = "TimeoutError";
|
|
2225
|
+
}
|
|
2226
|
+
};
|
|
2227
|
+
function describeRequestOperation(input, init) {
|
|
2228
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2229
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2230
|
+
let path;
|
|
2231
|
+
try {
|
|
2232
|
+
path = new URL(rawUrl).pathname;
|
|
2233
|
+
} catch (error2) {
|
|
2234
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2235
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2236
|
+
}
|
|
2237
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2238
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2239
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2240
|
+
return `${method} ${maskedPath}`;
|
|
2241
|
+
}
|
|
1711
2242
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1712
|
-
return ((input, init) =>
|
|
2243
|
+
return (async (input, init) => {
|
|
2244
|
+
const startedAt = Date.now();
|
|
2245
|
+
try {
|
|
2246
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2247
|
+
} catch (error2) {
|
|
2248
|
+
if (error2?.name === "TimeoutError") {
|
|
2249
|
+
throw new RequestTimeoutError(
|
|
2250
|
+
describeRequestOperation(input, init),
|
|
2251
|
+
timeoutMs,
|
|
2252
|
+
Date.now() - startedAt
|
|
2253
|
+
);
|
|
2254
|
+
}
|
|
2255
|
+
throw error2;
|
|
2256
|
+
}
|
|
2257
|
+
});
|
|
1713
2258
|
}
|
|
1714
2259
|
|
|
1715
2260
|
// src/lib/opencode/client.ts
|
|
@@ -1775,7 +2320,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1775
2320
|
return null;
|
|
1776
2321
|
}
|
|
1777
2322
|
}
|
|
1778
|
-
function
|
|
2323
|
+
function roleOf2(m) {
|
|
1779
2324
|
if (!m || typeof m !== "object") return void 0;
|
|
1780
2325
|
if (typeof m.role === "string") return m.role;
|
|
1781
2326
|
const infoRole = m.info?.role;
|
|
@@ -1789,13 +2334,13 @@ function createdOf(m) {
|
|
|
1789
2334
|
if (!m || typeof m !== "object") return void 0;
|
|
1790
2335
|
return m.info?.time?.created ?? m.time?.created;
|
|
1791
2336
|
}
|
|
1792
|
-
function
|
|
2337
|
+
function idOf2(m) {
|
|
1793
2338
|
if (!m || typeof m !== "object") return void 0;
|
|
1794
2339
|
if (typeof m.id === "string") return m.id;
|
|
1795
2340
|
const infoId = m.info?.id;
|
|
1796
2341
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1797
2342
|
}
|
|
1798
|
-
function
|
|
2343
|
+
function parentIdOf2(m) {
|
|
1799
2344
|
if (!m || typeof m !== "object") return void 0;
|
|
1800
2345
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1801
2346
|
const infoParent = m.info?.parentID;
|
|
@@ -1811,8 +2356,12 @@ function errorOf(m) {
|
|
|
1811
2356
|
if (!m || typeof m !== "object") return void 0;
|
|
1812
2357
|
return m.info?.error ?? m.error;
|
|
1813
2358
|
}
|
|
2359
|
+
function isTerminalStopAnswer(m) {
|
|
2360
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2361
|
+
}
|
|
1814
2362
|
function isAssistantInFlight(m) {
|
|
1815
2363
|
if (completedOf(m) == null) return true;
|
|
2364
|
+
if (errorOf(m) != null) return false;
|
|
1816
2365
|
return finishOf(m) === "tool-calls";
|
|
1817
2366
|
}
|
|
1818
2367
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -1860,9 +2409,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1860
2409
|
function isSessionActivelyGenerating(messages) {
|
|
1861
2410
|
if (!messages || messages.length === 0) return false;
|
|
1862
2411
|
const last = messages[messages.length - 1];
|
|
1863
|
-
if (
|
|
2412
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1864
2413
|
return completedOf(last) == null;
|
|
1865
2414
|
}
|
|
2415
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1866
2416
|
function sessionLastActivityMs(session) {
|
|
1867
2417
|
const candidates = [
|
|
1868
2418
|
session.time?.updated,
|
|
@@ -1877,30 +2427,60 @@ function sessionLastActivityMs(session) {
|
|
|
1877
2427
|
}
|
|
1878
2428
|
return null;
|
|
1879
2429
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
2430
|
+
function sessionListPath(options) {
|
|
2431
|
+
const params = new URLSearchParams();
|
|
2432
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2433
|
+
const query = params.toString();
|
|
2434
|
+
return query ? `/session?${query}` : "/session";
|
|
2435
|
+
}
|
|
2436
|
+
async function listSessionPage(port, client, options) {
|
|
2437
|
+
const path = sessionListPath(options);
|
|
1882
2438
|
try {
|
|
1883
|
-
const res = await requestWithClient(port, client,
|
|
2439
|
+
const res = await requestWithClient(port, client, path);
|
|
1884
2440
|
if (!res.ok) return null;
|
|
1885
2441
|
const body = await res.json();
|
|
1886
2442
|
return Array.isArray(body) ? body : null;
|
|
1887
2443
|
} catch (error2) {
|
|
1888
2444
|
console.error(
|
|
1889
|
-
`[
|
|
2445
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1890
2446
|
);
|
|
1891
2447
|
return null;
|
|
1892
2448
|
}
|
|
1893
2449
|
}
|
|
2450
|
+
async function listSessions(port, client, options) {
|
|
2451
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2452
|
+
return listSessionPage(port, client, options);
|
|
2453
|
+
}
|
|
2454
|
+
async function listAllSessions(port, client) {
|
|
2455
|
+
if (client?.version === "v2") {
|
|
2456
|
+
const sessions2 = await listV2Sessions(client);
|
|
2457
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2458
|
+
}
|
|
2459
|
+
const sessions = await listSessionPage(port, client, {
|
|
2460
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2461
|
+
});
|
|
2462
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2463
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2464
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2465
|
+
}
|
|
2466
|
+
return { sessions };
|
|
2467
|
+
}
|
|
1894
2468
|
async function deleteSession(port, id, client) {
|
|
1895
2469
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1896
2470
|
try {
|
|
1897
|
-
const res = await requestWithClient(
|
|
1898
|
-
|
|
2471
|
+
const res = await requestWithClient(
|
|
2472
|
+
port,
|
|
2473
|
+
client,
|
|
2474
|
+
`/session/${id}`,
|
|
2475
|
+
{ method: "DELETE" },
|
|
2476
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2477
|
+
);
|
|
2478
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1899
2479
|
} catch (error2) {
|
|
1900
2480
|
console.error(
|
|
1901
2481
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1902
2482
|
);
|
|
1903
|
-
return false;
|
|
2483
|
+
return { ok: false, status: "threw" };
|
|
1904
2484
|
}
|
|
1905
2485
|
}
|
|
1906
2486
|
async function sessionExists(port, id, client) {
|
|
@@ -2138,11 +2718,31 @@ function messageText(m) {
|
|
|
2138
2718
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2139
2719
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2140
2720
|
}
|
|
2141
|
-
|
|
2721
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2722
|
+
if (!messages || messages.length === 0) return null;
|
|
2723
|
+
let best = null;
|
|
2724
|
+
for (const message of messages) {
|
|
2725
|
+
if (roleOf2(message) !== "user") continue;
|
|
2726
|
+
const id = idOf2(message);
|
|
2727
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2728
|
+
if (messageText(message) !== content) continue;
|
|
2729
|
+
const created = createdOf(message) ?? 0;
|
|
2730
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2731
|
+
}
|
|
2732
|
+
return best?.id ?? null;
|
|
2733
|
+
}
|
|
2734
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2142
2735
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2143
2736
|
const knownUserIds = new Set(
|
|
2144
|
-
(before ?? []).filter((m) =>
|
|
2737
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2145
2738
|
);
|
|
2739
|
+
if (callbacks) {
|
|
2740
|
+
try {
|
|
2741
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2742
|
+
} catch (error2) {
|
|
2743
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2744
|
+
}
|
|
2745
|
+
}
|
|
2146
2746
|
const parts = [{ type: "text", text: content }];
|
|
2147
2747
|
let pendingOutcomes = null;
|
|
2148
2748
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2181,22 +2781,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2181
2781
|
const READ_BACK_DELAY_MS = 150;
|
|
2182
2782
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2183
2783
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
const id = idOf(m);
|
|
2189
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2190
|
-
if (messageText(m) !== content) continue;
|
|
2191
|
-
const created = createdOf(m) ?? 0;
|
|
2192
|
-
if (best === null || created > best.created) {
|
|
2193
|
-
best = { id, created };
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
if (best) {
|
|
2197
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2198
|
-
return best.id;
|
|
2199
|
-
}
|
|
2784
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2785
|
+
if (dispatchedUserMessageId !== null) {
|
|
2786
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2787
|
+
return dispatchedUserMessageId;
|
|
2200
2788
|
}
|
|
2201
2789
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2202
2790
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2207,57 +2795,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2207
2795
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2208
2796
|
if (!messages || messages.length === 0) return null;
|
|
2209
2797
|
const byParent = messages.find(
|
|
2210
|
-
(m) =>
|
|
2798
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2211
2799
|
);
|
|
2212
2800
|
if (byParent) return byParent;
|
|
2213
|
-
const userIndex = messages.findIndex((m) =>
|
|
2801
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2214
2802
|
if (userIndex === -1) return null;
|
|
2215
2803
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2216
|
-
if (
|
|
2804
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2217
2805
|
}
|
|
2218
2806
|
return null;
|
|
2219
2807
|
}
|
|
2220
2808
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2221
2809
|
if (!messages || messages.length === 0) return null;
|
|
2222
2810
|
let lastCorrelated = null;
|
|
2223
|
-
let lastNonErrored = null;
|
|
2224
2811
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2225
2812
|
const m = messages[i];
|
|
2226
|
-
if (
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2813
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2814
|
+
lastCorrelated = m;
|
|
2815
|
+
break;
|
|
2816
|
+
}
|
|
2817
|
+
if (lastCorrelated) {
|
|
2818
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2819
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2820
|
+
const m = messages[i];
|
|
2821
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2822
|
+
return m;
|
|
2823
|
+
}
|
|
2231
2824
|
}
|
|
2825
|
+
return lastCorrelated;
|
|
2232
2826
|
}
|
|
2233
|
-
|
|
2234
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2827
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2235
2828
|
if (userIndex === -1) return null;
|
|
2236
2829
|
let last = null;
|
|
2237
|
-
let
|
|
2830
|
+
let lastAssistantIndex = -1;
|
|
2238
2831
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2239
|
-
const role =
|
|
2832
|
+
const role = roleOf2(messages[i]);
|
|
2240
2833
|
if (role === "user") break;
|
|
2241
2834
|
if (role === "assistant") {
|
|
2242
2835
|
last = messages[i];
|
|
2243
|
-
|
|
2836
|
+
lastAssistantIndex = i;
|
|
2244
2837
|
}
|
|
2245
2838
|
}
|
|
2246
|
-
|
|
2839
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2840
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2841
|
+
const m = messages[i];
|
|
2842
|
+
if (roleOf2(m) === "user") break;
|
|
2843
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2844
|
+
}
|
|
2845
|
+
return last;
|
|
2247
2846
|
}
|
|
2248
2847
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2249
2848
|
if (!messages || messages.length === 0) return [];
|
|
2250
2849
|
const byParent = messages.filter(
|
|
2251
|
-
(message) =>
|
|
2850
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2252
2851
|
);
|
|
2253
2852
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2254
2853
|
if (assistants.length === 0) {
|
|
2255
|
-
const userIndex = messages.findIndex((message) =>
|
|
2854
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2256
2855
|
if (userIndex === -1) return [];
|
|
2257
2856
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2258
2857
|
const message = messages[i];
|
|
2259
|
-
if (
|
|
2260
|
-
if (
|
|
2858
|
+
if (roleOf2(message) === "user") break;
|
|
2859
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2261
2860
|
}
|
|
2262
2861
|
}
|
|
2263
2862
|
const refs = [];
|
|
@@ -2297,7 +2896,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2297
2896
|
if (!messages || messages.length === 0) return [];
|
|
2298
2897
|
const calls = [];
|
|
2299
2898
|
for (const message of messages) {
|
|
2300
|
-
if (
|
|
2899
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2301
2900
|
for (const part of message.parts ?? []) {
|
|
2302
2901
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2303
2902
|
continue;
|
|
@@ -2327,7 +2926,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2327
2926
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2328
2927
|
const unattributed = [];
|
|
2329
2928
|
for (const message of messages ?? []) {
|
|
2330
|
-
if (
|
|
2929
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2331
2930
|
const created = finiteNumber(createdOf(message));
|
|
2332
2931
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2333
2932
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2349,8 +2948,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2349
2948
|
}
|
|
2350
2949
|
function sumAssistantUsage(messages) {
|
|
2351
2950
|
if (!messages || messages.length === 0) return null;
|
|
2352
|
-
const
|
|
2353
|
-
const
|
|
2951
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
2952
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
2953
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2354
2954
|
let sawAnyUsage = false;
|
|
2355
2955
|
let inputSum = 0;
|
|
2356
2956
|
let outputSum = 0;
|
|
@@ -2404,13 +3004,11 @@ function sumAssistantUsage(messages) {
|
|
|
2404
3004
|
function messageUsage(messages, userMessageId) {
|
|
2405
3005
|
if (!messages || messages.length === 0) return null;
|
|
2406
3006
|
const byParentAll = messages.filter(
|
|
2407
|
-
(m) =>
|
|
3007
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2408
3008
|
);
|
|
2409
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2410
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2411
3009
|
let correlated;
|
|
2412
|
-
if (
|
|
2413
|
-
correlated =
|
|
3010
|
+
if (byParentAll.length > 0) {
|
|
3011
|
+
correlated = byParentAll;
|
|
2414
3012
|
} else {
|
|
2415
3013
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2416
3014
|
correlated = reply ? [reply] : [];
|
|
@@ -2420,7 +3018,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2420
3018
|
}
|
|
2421
3019
|
function messageRunState(messages, userMessageId) {
|
|
2422
3020
|
if (!messages || messages.length === 0) return "unknown";
|
|
2423
|
-
const hasUser = messages.some((m) =>
|
|
3021
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2424
3022
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2425
3023
|
if (!hasUser) {
|
|
2426
3024
|
if (!reply) return "unknown";
|
|
@@ -2456,14 +3054,26 @@ function messageError(messages, userMessageId) {
|
|
|
2456
3054
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2457
3055
|
const error2 = errorOf(reply);
|
|
2458
3056
|
if (error2 == null) return null;
|
|
2459
|
-
|
|
2460
|
-
if (typeof error2 === "
|
|
3057
|
+
let rendered;
|
|
3058
|
+
if (typeof error2 === "string") {
|
|
3059
|
+
rendered = error2;
|
|
3060
|
+
} else if (typeof error2 === "object") {
|
|
2461
3061
|
const e = error2;
|
|
2462
3062
|
const dataMessage = e.data?.message;
|
|
2463
|
-
if (typeof dataMessage === "string")
|
|
2464
|
-
|
|
3063
|
+
if (typeof dataMessage === "string") {
|
|
3064
|
+
rendered = dataMessage;
|
|
3065
|
+
} else if (typeof e.message === "string") {
|
|
3066
|
+
rendered = e.message;
|
|
3067
|
+
} else {
|
|
3068
|
+
rendered = "The agent run failed.";
|
|
3069
|
+
}
|
|
3070
|
+
} else {
|
|
3071
|
+
rendered = "The agent run failed.";
|
|
2465
3072
|
}
|
|
2466
|
-
|
|
3073
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3074
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3075
|
+
) : void 0;
|
|
3076
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
2467
3077
|
}
|
|
2468
3078
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2469
3079
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2512,7 +3122,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2512
3122
|
if (!messages || messages.length === 0) return null;
|
|
2513
3123
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2514
3124
|
const message = messages[i];
|
|
2515
|
-
if (
|
|
3125
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2516
3126
|
const created = createdOf(message);
|
|
2517
3127
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2518
3128
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2544,25 +3154,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2544
3154
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2545
3155
|
if (!messages || messages.length === 0) return false;
|
|
2546
3156
|
return messages.some(
|
|
2547
|
-
(m) =>
|
|
3157
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2548
3158
|
);
|
|
2549
3159
|
}
|
|
2550
3160
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2551
3161
|
if (!messages || messages.length === 0) return false;
|
|
2552
|
-
const userIndex = messages.findIndex((message) =>
|
|
3162
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2553
3163
|
if (userIndex === -1) return false;
|
|
2554
3164
|
let hasLaterUser = false;
|
|
2555
3165
|
let hasStartedLaterUser = false;
|
|
2556
3166
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2557
3167
|
const message = messages[i];
|
|
2558
|
-
if (
|
|
3168
|
+
if (roleOf2(message) !== "user") continue;
|
|
2559
3169
|
hasLaterUser = true;
|
|
2560
|
-
const laterUserMessageId =
|
|
3170
|
+
const laterUserMessageId = idOf2(message);
|
|
2561
3171
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2562
3172
|
return false;
|
|
2563
3173
|
}
|
|
2564
3174
|
if (messages.some(
|
|
2565
|
-
(candidate) =>
|
|
3175
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2566
3176
|
)) {
|
|
2567
3177
|
hasStartedLaterUser = true;
|
|
2568
3178
|
}
|
|
@@ -3033,13 +3643,17 @@ async function listV2Sessions(client) {
|
|
|
3033
3643
|
}
|
|
3034
3644
|
async function deleteV2Session(client, sessionId) {
|
|
3035
3645
|
try {
|
|
3036
|
-
await client.request(
|
|
3037
|
-
|
|
3646
|
+
const response = await client.request(
|
|
3647
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3648
|
+
{ method: "DELETE" },
|
|
3649
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3650
|
+
);
|
|
3651
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3038
3652
|
} catch (error2) {
|
|
3039
3653
|
console.error(
|
|
3040
3654
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3041
3655
|
);
|
|
3042
|
-
return false;
|
|
3656
|
+
return { ok: false, status: "threw" };
|
|
3043
3657
|
}
|
|
3044
3658
|
}
|
|
3045
3659
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3214,7 +3828,7 @@ import { join as join4 } from "node:path";
|
|
|
3214
3828
|
|
|
3215
3829
|
// src/lib/openai-usage.ts
|
|
3216
3830
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3217
|
-
import { homedir as
|
|
3831
|
+
import { homedir as homedir3 } from "node:os";
|
|
3218
3832
|
import { join as join3 } from "node:path";
|
|
3219
3833
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3220
3834
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
@@ -3229,7 +3843,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3229
3843
|
}
|
|
3230
3844
|
function readOpenCodeChatGptCredentials() {
|
|
3231
3845
|
try {
|
|
3232
|
-
const raw = readFileSync3(join3(
|
|
3846
|
+
const raw = readFileSync3(join3(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3233
3847
|
let parsed;
|
|
3234
3848
|
try {
|
|
3235
3849
|
parsed = JSON.parse(raw);
|
|
@@ -3370,16 +3984,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3370
3984
|
].some((name) => headers.has(name));
|
|
3371
3985
|
}
|
|
3372
3986
|
async function getOpenAiUsage(port, client) {
|
|
3987
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3373
3988
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3374
3989
|
if (!credentials2) {
|
|
3375
3990
|
throw new OpenAiUsageError(
|
|
3376
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3991
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3377
3992
|
"no_credentials"
|
|
3378
3993
|
);
|
|
3379
3994
|
}
|
|
3380
3995
|
if (credentials2.expiresAt < Date.now()) {
|
|
3381
3996
|
throw new OpenAiUsageError(
|
|
3382
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3997
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3383
3998
|
"credentials_expired"
|
|
3384
3999
|
);
|
|
3385
4000
|
}
|
|
@@ -3660,9 +4275,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3660
4275
|
|
|
3661
4276
|
// src/lib/opencode/session-db-boot.ts
|
|
3662
4277
|
import { spawn as spawn2 } from "node:child_process";
|
|
3663
|
-
import { mkdirSync, statSync as
|
|
3664
|
-
import { homedir as
|
|
3665
|
-
import { dirname as
|
|
4278
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
4279
|
+
import { homedir as homedir4 } from "node:os";
|
|
4280
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3666
4281
|
|
|
3667
4282
|
// src/lib/runner-synchroniser.ts
|
|
3668
4283
|
import { spawn } from "node:child_process";
|
|
@@ -3742,9 +4357,12 @@ function runSynchroniser(args, opts) {
|
|
|
3742
4357
|
|
|
3743
4358
|
// src/lib/opencode/session-db-boot.ts
|
|
3744
4359
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3745
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4360
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3746
4361
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3747
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4362
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4363
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4364
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4365
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3748
4366
|
function commandError(result) {
|
|
3749
4367
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3750
4368
|
}
|
|
@@ -3787,7 +4405,7 @@ function clearMarker(options) {
|
|
|
3787
4405
|
function markNoReplicate(options, message) {
|
|
3788
4406
|
if (options.noReplicateMarker) {
|
|
3789
4407
|
try {
|
|
3790
|
-
mkdirSync(
|
|
4408
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3791
4409
|
writeFileSync(options.noReplicateMarker, "");
|
|
3792
4410
|
} catch (error2) {
|
|
3793
4411
|
options.log(
|
|
@@ -3912,7 +4530,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3912
4530
|
return null;
|
|
3913
4531
|
}
|
|
3914
4532
|
try {
|
|
3915
|
-
if (
|
|
4533
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3916
4534
|
} catch (error2) {
|
|
3917
4535
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3918
4536
|
options.log(
|
|
@@ -3942,7 +4560,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3942
4560
|
return null;
|
|
3943
4561
|
}
|
|
3944
4562
|
try {
|
|
3945
|
-
mkdirSync(
|
|
4563
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3946
4564
|
writeFileSync(configPath, rendered.stdout);
|
|
3947
4565
|
} catch (error2) {
|
|
3948
4566
|
options.log(
|
|
@@ -4017,14 +4635,14 @@ async function restoreSessionDb(options, configPath, env) {
|
|
|
4017
4635
|
case 0:
|
|
4018
4636
|
return;
|
|
4019
4637
|
case 31:
|
|
4020
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4638
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4021
4639
|
options.log(
|
|
4022
4640
|
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
4023
4641
|
"warn"
|
|
4024
4642
|
);
|
|
4025
4643
|
return;
|
|
4026
4644
|
case 32:
|
|
4027
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4645
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4028
4646
|
discardSessionDbDebris(options);
|
|
4029
4647
|
markNoReplicate(
|
|
4030
4648
|
options,
|
|
@@ -4055,20 +4673,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4055
4673
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4056
4674
|
env: {
|
|
4057
4675
|
...env,
|
|
4058
|
-
// The synchroniser reads
|
|
4059
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4676
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4060
4677
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4061
4678
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4679
|
+
),
|
|
4680
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4681
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4682
|
+
),
|
|
4683
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4684
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4685
|
+
),
|
|
4686
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4687
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4062
4688
|
)
|
|
4063
4689
|
}
|
|
4064
4690
|
});
|
|
4065
4691
|
logSynchroniserDiagnostics(result, options);
|
|
4066
4692
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4693
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4694
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4695
|
+
options.log(
|
|
4696
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4697
|
+
"error"
|
|
4698
|
+
);
|
|
4699
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4700
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4701
|
+
}
|
|
4067
4702
|
options.log(
|
|
4068
4703
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4069
4704
|
"warn"
|
|
4070
4705
|
);
|
|
4071
|
-
return false;
|
|
4706
|
+
return { verifyFatal: false };
|
|
4072
4707
|
}
|
|
4073
4708
|
if (result.code === 34) {
|
|
4074
4709
|
reportRecord(
|
|
@@ -4078,14 +4713,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4078
4713
|
null,
|
|
4079
4714
|
options
|
|
4080
4715
|
);
|
|
4081
|
-
return true;
|
|
4716
|
+
return { verifyFatal: true };
|
|
4082
4717
|
}
|
|
4083
4718
|
if (result.code === 33) {
|
|
4084
4719
|
options.log(
|
|
4085
4720
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4086
4721
|
"warn"
|
|
4087
4722
|
);
|
|
4088
|
-
return false;
|
|
4723
|
+
return { verifyFatal: false };
|
|
4089
4724
|
}
|
|
4090
4725
|
if (result.code !== 0) {
|
|
4091
4726
|
options.log(
|
|
@@ -4093,17 +4728,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4093
4728
|
"warn"
|
|
4094
4729
|
);
|
|
4095
4730
|
}
|
|
4096
|
-
return false;
|
|
4731
|
+
return { verifyFatal: false };
|
|
4097
4732
|
}
|
|
4098
4733
|
options.log(
|
|
4099
4734
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4100
4735
|
"debug"
|
|
4101
4736
|
);
|
|
4102
|
-
return false;
|
|
4737
|
+
return { verifyFatal: false };
|
|
4103
4738
|
}
|
|
4104
4739
|
function fileExists(path) {
|
|
4105
4740
|
try {
|
|
4106
|
-
|
|
4741
|
+
statSync3(path);
|
|
4107
4742
|
return true;
|
|
4108
4743
|
} catch (error2) {
|
|
4109
4744
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4113,7 +4748,7 @@ function fileExists(path) {
|
|
|
4113
4748
|
async function restoreAndVerifySessionDb(options) {
|
|
4114
4749
|
const env = options.env ?? process.env;
|
|
4115
4750
|
clearMarker(options);
|
|
4116
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4751
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4117
4752
|
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
4118
4753
|
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
4119
4754
|
env
|
|
@@ -4186,13 +4821,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4186
4821
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4187
4822
|
return { verifyFatal: false };
|
|
4188
4823
|
}
|
|
4189
|
-
return
|
|
4824
|
+
return verifySessionDb(options, configPath, env);
|
|
4190
4825
|
}
|
|
4191
4826
|
|
|
4192
4827
|
// src/lib/opencode/session-db-provenance.ts
|
|
4193
4828
|
import { createRequire } from "node:module";
|
|
4194
4829
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4195
|
-
import { dirname as
|
|
4830
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4196
4831
|
var require2 = createRequire(import.meta.url);
|
|
4197
4832
|
function readSessionDbMigrationIds(dbPath) {
|
|
4198
4833
|
let db;
|
|
@@ -4258,7 +4893,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4258
4893
|
}
|
|
4259
4894
|
function saveSessionDbProvenanceState(path, state) {
|
|
4260
4895
|
try {
|
|
4261
|
-
mkdirSync2(
|
|
4896
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4262
4897
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4263
4898
|
`, "utf8");
|
|
4264
4899
|
} catch (error2) {
|
|
@@ -4493,12 +5128,15 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4493
5128
|
}
|
|
4494
5129
|
} catch {
|
|
4495
5130
|
try {
|
|
4496
|
-
const psOutput = execSync(
|
|
5131
|
+
const psOutput = execSync("ps aux", {
|
|
4497
5132
|
encoding: "utf-8",
|
|
4498
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
5133
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
5134
|
+
// The whole table reaches Node; the 1 MiB default would surface a spurious failure.
|
|
5135
|
+
maxBuffer: 16 * 1024 * 1024
|
|
4499
5136
|
}).trim();
|
|
4500
5137
|
if (psOutput) {
|
|
4501
5138
|
for (const line of psOutput.split("\n")) {
|
|
5139
|
+
if (!psPattern.test(line)) continue;
|
|
4502
5140
|
const parts = line.trim().split(/\s+/);
|
|
4503
5141
|
if (parts.length >= 2) {
|
|
4504
5142
|
const pid = parseInt(parts[1], 10);
|
|
@@ -4507,8 +5145,9 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4507
5145
|
}
|
|
4508
5146
|
}
|
|
4509
5147
|
} catch (err) {
|
|
5148
|
+
const reason = (typeof err === "object" && err !== null && "stderr" in err && typeof err.stderr === "string" ? err.stderr : "").split("\n").map((line) => line.trim()).find(Boolean) ?? (err instanceof Error ? err.message : String(err)).split("\n").map((line) => line.trim()).find(Boolean) ?? "no error detail available";
|
|
4510
5149
|
console.warn(
|
|
4511
|
-
`
|
|
5150
|
+
`Could not list running processes, so an already-running OpenCode may not be detected: ${reason}`
|
|
4512
5151
|
);
|
|
4513
5152
|
}
|
|
4514
5153
|
}
|
|
@@ -4540,7 +5179,7 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4540
5179
|
return instances;
|
|
4541
5180
|
}
|
|
4542
5181
|
function findOpenCodeProcesses() {
|
|
4543
|
-
return findProcessesByPattern("opencode serve|opencode-serve",
|
|
5182
|
+
return findProcessesByPattern("opencode serve|opencode-serve", /opencode (serve|--port)/);
|
|
4544
5183
|
}
|
|
4545
5184
|
async function scanPortsForOpenCode() {
|
|
4546
5185
|
const instances = [];
|
|
@@ -4835,7 +5474,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4835
5474
|
toDelete.push(s.id);
|
|
4836
5475
|
}
|
|
4837
5476
|
}
|
|
4838
|
-
|
|
5477
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5478
|
+
return toDelete.sort(
|
|
5479
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5480
|
+
);
|
|
4839
5481
|
}
|
|
4840
5482
|
var DEFAULT_INTERVAL = "1h";
|
|
4841
5483
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4895,13 +5537,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4895
5537
|
}
|
|
4896
5538
|
|
|
4897
5539
|
// src/lib/opencode/session-db-size.ts
|
|
4898
|
-
import { statSync as
|
|
5540
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4899
5541
|
import { join as join6 } from "node:path";
|
|
4900
5542
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4901
5543
|
function statSessionDbBytes(homeDir) {
|
|
4902
5544
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4903
5545
|
try {
|
|
4904
|
-
return
|
|
5546
|
+
return statSync4(dbPath).size;
|
|
4905
5547
|
} catch (err) {
|
|
4906
5548
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4907
5549
|
if (!isMissingFile) {
|
|
@@ -4927,12 +5569,12 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4927
5569
|
}
|
|
4928
5570
|
|
|
4929
5571
|
// src/lib/opencode/log-tail.ts
|
|
4930
|
-
import { statSync as
|
|
4931
|
-
import { homedir as
|
|
5572
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5573
|
+
import { homedir as homedir5 } from "node:os";
|
|
4932
5574
|
import { join as join7 } from "node:path";
|
|
4933
5575
|
import { open as open2, stat } from "node:fs/promises";
|
|
4934
5576
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
4935
|
-
function resolveOpenCodeLogPath(homeDir =
|
|
5577
|
+
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
4936
5578
|
const dataDir = env.XDG_DATA_HOME || join7(homeDir, ".local", "share");
|
|
4937
5579
|
return join7(dataDir, "opencode", "log", "opencode.log");
|
|
4938
5580
|
}
|
|
@@ -4944,188 +5586,69 @@ function reportFailure(operation, logPath, error2) {
|
|
|
4944
5586
|
`[opencode-log-tail] ${operation} failed for ${logPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4945
5587
|
);
|
|
4946
5588
|
}
|
|
4947
|
-
function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
4948
|
-
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
4949
|
-
let offset = 0;
|
|
4950
|
-
let inode = null;
|
|
4951
|
-
let baselineReady = true;
|
|
4952
|
-
try {
|
|
4953
|
-
const initial = statSync4(logPath);
|
|
4954
|
-
offset = initial.size;
|
|
4955
|
-
inode = initial.ino;
|
|
4956
|
-
} catch (error2) {
|
|
4957
|
-
if (!isEnoent(error2)) {
|
|
4958
|
-
reportFailure("initial stat", logPath, error2);
|
|
4959
|
-
baselineReady = false;
|
|
4960
|
-
}
|
|
4961
|
-
}
|
|
4962
|
-
let polling = false;
|
|
4963
|
-
let stopped = false;
|
|
4964
|
-
const poll = async () => {
|
|
4965
|
-
if (polling || stopped) return;
|
|
4966
|
-
polling = true;
|
|
4967
|
-
try {
|
|
4968
|
-
let current;
|
|
4969
|
-
try {
|
|
4970
|
-
current = await stat(logPath);
|
|
4971
|
-
} catch (error2) {
|
|
4972
|
-
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
4973
|
-
return;
|
|
4974
|
-
}
|
|
4975
|
-
if (!baselineReady) {
|
|
4976
|
-
offset = current.size;
|
|
4977
|
-
inode = current.ino;
|
|
4978
|
-
baselineReady = true;
|
|
4979
|
-
return;
|
|
4980
|
-
}
|
|
4981
|
-
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
4982
|
-
offset = 0;
|
|
4983
|
-
}
|
|
4984
|
-
inode = current.ino;
|
|
4985
|
-
if (current.size === offset) return;
|
|
4986
|
-
const length = current.size - offset;
|
|
4987
|
-
const fh = await open2(logPath, "r");
|
|
4988
|
-
try {
|
|
4989
|
-
const buf = Buffer.alloc(length);
|
|
4990
|
-
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
4991
|
-
offset += bytesRead;
|
|
4992
|
-
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
4993
|
-
} finally {
|
|
4994
|
-
await fh.close();
|
|
4995
|
-
}
|
|
4996
|
-
} catch (error2) {
|
|
4997
|
-
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
4998
|
-
} finally {
|
|
4999
|
-
polling = false;
|
|
5000
|
-
}
|
|
5001
|
-
};
|
|
5002
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5003
|
-
void poll();
|
|
5004
|
-
return {
|
|
5005
|
-
stop: () => {
|
|
5006
|
-
stopped = true;
|
|
5007
|
-
clearInterval(interval);
|
|
5008
|
-
}
|
|
5009
|
-
};
|
|
5010
|
-
}
|
|
5011
|
-
|
|
5012
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5013
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5014
|
-
import { dirname as dirname4 } from "node:path";
|
|
5015
|
-
function errorMessage(error2) {
|
|
5016
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5017
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5018
|
-
}
|
|
5019
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5020
|
-
try {
|
|
5021
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5022
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5023
|
-
if (availableBytes < requiredBytes) {
|
|
5024
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5025
|
-
}
|
|
5026
|
-
return null;
|
|
5027
|
-
} catch (err) {
|
|
5028
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5029
|
-
}
|
|
5030
|
-
}
|
|
5031
|
-
function readLogicalBytes(db) {
|
|
5032
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5033
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5034
|
-
return pageCount * pageSize;
|
|
5035
|
-
}
|
|
5036
|
-
function readCheckpointResult(db) {
|
|
5037
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5038
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5039
|
-
}
|
|
5040
|
-
async function probeReclaimAvailability(input) {
|
|
5041
|
-
const { dbPath, requiredBytes } = input;
|
|
5042
|
-
let sqlite;
|
|
5043
|
-
try {
|
|
5044
|
-
sqlite = await import("node:sqlite");
|
|
5045
|
-
} catch (err) {
|
|
5046
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5047
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5048
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5049
|
-
}
|
|
5050
|
-
let autoVacuum = null;
|
|
5051
|
-
try {
|
|
5052
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5053
|
-
try {
|
|
5054
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5055
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5056
|
-
} finally {
|
|
5057
|
-
db.close();
|
|
5058
|
-
}
|
|
5059
|
-
} catch (err) {
|
|
5060
|
-
console.warn(
|
|
5061
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5062
|
-
);
|
|
5063
|
-
}
|
|
5064
|
-
if (autoVacuum !== 0) return null;
|
|
5065
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5066
|
-
}
|
|
5067
|
-
async function reclaimSessionDbSpace(input) {
|
|
5068
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5069
|
-
let sqlite;
|
|
5589
|
+
function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
5590
|
+
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
5591
|
+
let offset = 0;
|
|
5592
|
+
let inode = null;
|
|
5593
|
+
let baselineReady = true;
|
|
5070
5594
|
try {
|
|
5071
|
-
|
|
5072
|
-
|
|
5073
|
-
|
|
5074
|
-
|
|
5075
|
-
|
|
5076
|
-
|
|
5077
|
-
|
|
5595
|
+
const initial = statSync5(logPath);
|
|
5596
|
+
offset = initial.size;
|
|
5597
|
+
inode = initial.ino;
|
|
5598
|
+
} catch (error2) {
|
|
5599
|
+
if (!isEnoent(error2)) {
|
|
5600
|
+
reportFailure("initial stat", logPath, error2);
|
|
5601
|
+
baselineReady = false;
|
|
5602
|
+
}
|
|
5078
5603
|
}
|
|
5079
|
-
|
|
5080
|
-
let
|
|
5081
|
-
|
|
5082
|
-
|
|
5083
|
-
|
|
5084
|
-
|
|
5085
|
-
|
|
5086
|
-
|
|
5087
|
-
|
|
5088
|
-
|
|
5089
|
-
);
|
|
5090
|
-
return
|
|
5604
|
+
let polling = false;
|
|
5605
|
+
let stopped = false;
|
|
5606
|
+
const poll = async () => {
|
|
5607
|
+
if (polling || stopped) return;
|
|
5608
|
+
polling = true;
|
|
5609
|
+
try {
|
|
5610
|
+
let current;
|
|
5611
|
+
try {
|
|
5612
|
+
current = await stat(logPath);
|
|
5613
|
+
} catch (error2) {
|
|
5614
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5615
|
+
return;
|
|
5091
5616
|
}
|
|
5092
|
-
|
|
5093
|
-
|
|
5094
|
-
|
|
5095
|
-
|
|
5096
|
-
|
|
5097
|
-
);
|
|
5098
|
-
return { ok: false, skipped: "insufficient-disk-space" };
|
|
5617
|
+
if (!baselineReady) {
|
|
5618
|
+
offset = current.size;
|
|
5619
|
+
inode = current.ino;
|
|
5620
|
+
baselineReady = true;
|
|
5621
|
+
return;
|
|
5099
5622
|
}
|
|
5100
|
-
|
|
5101
|
-
|
|
5102
|
-
|
|
5103
|
-
|
|
5104
|
-
|
|
5105
|
-
|
|
5623
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5624
|
+
offset = 0;
|
|
5625
|
+
}
|
|
5626
|
+
inode = current.ino;
|
|
5627
|
+
if (current.size === offset) return;
|
|
5628
|
+
const length = current.size - offset;
|
|
5629
|
+
const fh = await open2(logPath, "r");
|
|
5630
|
+
try {
|
|
5631
|
+
const buf = Buffer.alloc(length);
|
|
5632
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5633
|
+
offset += bytesRead;
|
|
5634
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5635
|
+
} finally {
|
|
5636
|
+
await fh.close();
|
|
5637
|
+
}
|
|
5638
|
+
} catch (error2) {
|
|
5639
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5640
|
+
} finally {
|
|
5641
|
+
polling = false;
|
|
5106
5642
|
}
|
|
5107
|
-
|
|
5108
|
-
|
|
5109
|
-
|
|
5110
|
-
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
|
-
|
|
5643
|
+
};
|
|
5644
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5645
|
+
void poll();
|
|
5646
|
+
return {
|
|
5647
|
+
stop: () => {
|
|
5648
|
+
stopped = true;
|
|
5649
|
+
clearInterval(interval);
|
|
5114
5650
|
}
|
|
5115
|
-
|
|
5116
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5117
|
-
);
|
|
5118
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5119
|
-
} catch (err) {
|
|
5120
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5121
|
-
return {
|
|
5122
|
-
ok: false,
|
|
5123
|
-
skipped: "reclaim-error",
|
|
5124
|
-
detail: errorMessage(err)
|
|
5125
|
-
};
|
|
5126
|
-
} finally {
|
|
5127
|
-
db?.close();
|
|
5128
|
-
}
|
|
5651
|
+
};
|
|
5129
5652
|
}
|
|
5130
5653
|
|
|
5131
5654
|
// src/lib/tunnel/connection.ts
|
|
@@ -5357,6 +5880,8 @@ var StreamForwarder = class {
|
|
|
5357
5880
|
if (done) break;
|
|
5358
5881
|
const chunk = Buffer.from(value);
|
|
5359
5882
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5883
|
+
await stream.pausePromise;
|
|
5884
|
+
if (ac.signal.aborted) break;
|
|
5360
5885
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5361
5886
|
if (this.binaryFramesSupported) {
|
|
5362
5887
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5399,6 +5924,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5399
5924
|
}
|
|
5400
5925
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5401
5926
|
var BASE_RECONNECT_DELAY = 500;
|
|
5927
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5402
5928
|
function getReconnectDelay(attempt) {
|
|
5403
5929
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5404
5930
|
const jitter = Math.random() * 1e3;
|
|
@@ -5470,10 +5996,27 @@ function connectTunnel(options) {
|
|
|
5470
5996
|
ws.close();
|
|
5471
5997
|
reject(new Error("Connection timeout"));
|
|
5472
5998
|
}, 3e4);
|
|
5999
|
+
let relaySilenceTimeout;
|
|
6000
|
+
let relaySilenceTimedOut = false;
|
|
6001
|
+
const clearRelaySilenceTimeout = () => {
|
|
6002
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6003
|
+
clearTimeout(relaySilenceTimeout);
|
|
6004
|
+
relaySilenceTimeout = void 0;
|
|
6005
|
+
};
|
|
6006
|
+
const armRelaySilenceTimeout = () => {
|
|
6007
|
+
clearRelaySilenceTimeout();
|
|
6008
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6009
|
+
relaySilenceTimeout = void 0;
|
|
6010
|
+
relaySilenceTimedOut = true;
|
|
6011
|
+
ws.terminate();
|
|
6012
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6013
|
+
relaySilenceTimeout.unref?.();
|
|
6014
|
+
};
|
|
5473
6015
|
let upgradeRejection = null;
|
|
5474
6016
|
let upgradeRejectionReason = null;
|
|
5475
6017
|
ws.on("unexpected-response", (_req, res) => {
|
|
5476
6018
|
clearTimeout(connectionTimeout);
|
|
6019
|
+
clearRelaySilenceTimeout();
|
|
5477
6020
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5478
6021
|
upgradeRejectionReason = reason;
|
|
5479
6022
|
const chunks = [];
|
|
@@ -5503,6 +6046,7 @@ function connectTunnel(options) {
|
|
|
5503
6046
|
onInfo?.("WebSocket connection established");
|
|
5504
6047
|
});
|
|
5505
6048
|
ws.on("message", (data, isBinary) => {
|
|
6049
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5506
6050
|
if (isBinary) {
|
|
5507
6051
|
try {
|
|
5508
6052
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5536,6 +6080,7 @@ function connectTunnel(options) {
|
|
|
5536
6080
|
switch (message.type) {
|
|
5537
6081
|
case "connected": {
|
|
5538
6082
|
clearTimeout(connectionTimeout);
|
|
6083
|
+
armRelaySilenceTimeout();
|
|
5539
6084
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5540
6085
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5541
6086
|
onConnected?.(connectedAgentId);
|
|
@@ -5560,6 +6105,7 @@ function connectTunnel(options) {
|
|
|
5560
6105
|
});
|
|
5561
6106
|
ws.on("error", (error2) => {
|
|
5562
6107
|
clearTimeout(connectionTimeout);
|
|
6108
|
+
clearRelaySilenceTimeout();
|
|
5563
6109
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5564
6110
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5565
6111
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5571,7 +6117,8 @@ function connectTunnel(options) {
|
|
|
5571
6117
|
);
|
|
5572
6118
|
});
|
|
5573
6119
|
ws.on("close", (code, reason) => {
|
|
5574
|
-
|
|
6120
|
+
clearRelaySilenceTimeout();
|
|
6121
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5575
6122
|
forwarder.abortAll();
|
|
5576
6123
|
onDisconnected?.(code, reasonStr);
|
|
5577
6124
|
});
|
|
@@ -5579,9 +6126,10 @@ function connectTunnel(options) {
|
|
|
5579
6126
|
}
|
|
5580
6127
|
|
|
5581
6128
|
// src/lib/tunnel/runner-connection.ts
|
|
5582
|
-
var RunnerConnection = class {
|
|
6129
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5583
6130
|
opts;
|
|
5584
6131
|
sleep;
|
|
6132
|
+
now;
|
|
5585
6133
|
connection = null;
|
|
5586
6134
|
resolvedAgentId;
|
|
5587
6135
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5590,10 +6138,17 @@ var RunnerConnection = class {
|
|
|
5590
6138
|
reconnectPromise = null;
|
|
5591
6139
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5592
6140
|
reconnectAttempt = 0;
|
|
6141
|
+
generation = 0;
|
|
6142
|
+
disconnectedAtMs = null;
|
|
6143
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6144
|
+
"No pong received",
|
|
6145
|
+
"Orphaned socket"
|
|
6146
|
+
]);
|
|
5593
6147
|
constructor(opts) {
|
|
5594
6148
|
this.opts = opts;
|
|
5595
6149
|
this.resolvedAgentId = opts.agentId;
|
|
5596
6150
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6151
|
+
this.now = opts.now ?? Date.now;
|
|
5597
6152
|
}
|
|
5598
6153
|
get agentId() {
|
|
5599
6154
|
return this.resolvedAgentId;
|
|
@@ -5629,14 +6184,24 @@ var RunnerConnection = class {
|
|
|
5629
6184
|
port: this.opts.port,
|
|
5630
6185
|
openCodePassword: this.opts.openCodePassword,
|
|
5631
6186
|
onConnected: (agentId) => {
|
|
6187
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5632
6188
|
this.reconnectAttempt = 0;
|
|
5633
6189
|
this.reconnecting = false;
|
|
5634
6190
|
this.resolvedAgentId = agentId;
|
|
5635
|
-
|
|
6191
|
+
this.generation++;
|
|
6192
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6193
|
+
this.disconnectedAtMs = null;
|
|
6194
|
+
events.onConnected(agentId, isReconnect, {
|
|
6195
|
+
generation: this.generation,
|
|
6196
|
+
attempt,
|
|
6197
|
+
downtimeMs
|
|
6198
|
+
});
|
|
5636
6199
|
},
|
|
5637
6200
|
onDisconnected: (code, reason) => {
|
|
5638
|
-
|
|
5639
|
-
|
|
6201
|
+
this.disconnectedAtMs = this.now();
|
|
6202
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6203
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6204
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5640
6205
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5641
6206
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5642
6207
|
});
|
|
@@ -5975,8 +6540,85 @@ function createResourceUsageCollector(homeDir) {
|
|
|
5975
6540
|
};
|
|
5976
6541
|
}
|
|
5977
6542
|
|
|
6543
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6544
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6545
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6546
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6547
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6548
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6549
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6550
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6551
|
+
var BAND_RANK = {
|
|
6552
|
+
low: 1,
|
|
6553
|
+
very_low: 2,
|
|
6554
|
+
critical: 3
|
|
6555
|
+
};
|
|
6556
|
+
function bandFor(freeBytes) {
|
|
6557
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6558
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6559
|
+
return "low";
|
|
6560
|
+
}
|
|
6561
|
+
function formatBytes(bytes) {
|
|
6562
|
+
if (bytes === null) return "unavailable";
|
|
6563
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6564
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6565
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6566
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6567
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6568
|
+
}
|
|
6569
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6570
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6571
|
+
}
|
|
6572
|
+
function evaluateLowDiskWarning(input) {
|
|
6573
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6574
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6575
|
+
return { warning: null, state };
|
|
6576
|
+
}
|
|
6577
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6578
|
+
if (recovered) {
|
|
6579
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6580
|
+
}
|
|
6581
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6582
|
+
if (!low) return { warning: null, state };
|
|
6583
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6584
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6585
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6586
|
+
const worse = currentRank > previousRank;
|
|
6587
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6588
|
+
const nextState = {
|
|
6589
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6590
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6591
|
+
};
|
|
6592
|
+
return {
|
|
6593
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6594
|
+
state: nextState
|
|
6595
|
+
};
|
|
6596
|
+
}
|
|
6597
|
+
|
|
6598
|
+
// src/lib/opencode/session-store-health.ts
|
|
6599
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6600
|
+
function createSessionStoreHealth(options = {}) {
|
|
6601
|
+
const now = options.now ?? (() => Date.now());
|
|
6602
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6603
|
+
let lastUnwritableAt = null;
|
|
6604
|
+
let lastWritableAt = null;
|
|
6605
|
+
return {
|
|
6606
|
+
observeUnwritable: () => {
|
|
6607
|
+
lastUnwritableAt = now();
|
|
6608
|
+
},
|
|
6609
|
+
observeWritable: () => {
|
|
6610
|
+
lastWritableAt = now();
|
|
6611
|
+
},
|
|
6612
|
+
isUnwritable: () => {
|
|
6613
|
+
if (lastUnwritableAt === null) return false;
|
|
6614
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6615
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6616
|
+
}
|
|
6617
|
+
};
|
|
6618
|
+
}
|
|
6619
|
+
|
|
5978
6620
|
// src/lib/channels/driver.ts
|
|
5979
|
-
import { homedir as
|
|
6621
|
+
import { homedir as homedir6 } from "node:os";
|
|
5980
6622
|
|
|
5981
6623
|
// src/lib/runner-file-sync.ts
|
|
5982
6624
|
import { join as join9 } from "node:path";
|
|
@@ -6431,8 +7073,12 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6431
7073
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6432
7074
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6433
7075
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7076
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7077
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7078
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
6434
7079
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6435
7080
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7081
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6436
7082
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6437
7083
|
var HEARTBEAT_MS = 6e4;
|
|
6438
7084
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6494,6 +7140,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6494
7140
|
maxActiveSessions;
|
|
6495
7141
|
watcherStallMs;
|
|
6496
7142
|
wedgeWarningIntervalMs;
|
|
7143
|
+
sessionStoreHealth;
|
|
6497
7144
|
openCodeClient;
|
|
6498
7145
|
/** Cache of conversationId → opencode sessionId. */
|
|
6499
7146
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6586,6 +7233,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6586
7233
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6587
7234
|
*/
|
|
6588
7235
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7236
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6589
7237
|
/**
|
|
6590
7238
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6591
7239
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6618,10 +7266,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6618
7266
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6619
7267
|
*/
|
|
6620
7268
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7269
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7270
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6621
7271
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6622
7272
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6623
7273
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6624
7274
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7275
|
+
/**
|
|
7276
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7277
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7278
|
+
* identical-text turn, so it never produces a correlation.
|
|
7279
|
+
*/
|
|
7280
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6625
7281
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6626
7282
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6627
7283
|
/**
|
|
@@ -6707,21 +7363,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6707
7363
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6708
7364
|
*/
|
|
6709
7365
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6710
|
-
/**
|
|
6711
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6712
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6713
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6714
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6715
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6716
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6717
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6718
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6719
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6720
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6721
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6722
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6723
|
-
*/
|
|
6724
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6725
7366
|
/**
|
|
6726
7367
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6727
7368
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6739,8 +7380,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6739
7380
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6740
7381
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6741
7382
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6742
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6743
|
-
*
|
|
7383
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7384
|
+
* the terminal-report fence and is never sent again).
|
|
6744
7385
|
*/
|
|
6745
7386
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6746
7387
|
/**
|
|
@@ -6859,10 +7500,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6859
7500
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
6860
7501
|
this.now = config.now ?? (() => Date.now());
|
|
6861
7502
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
6862
|
-
this.homeDir = config.homeDir ??
|
|
7503
|
+
this.homeDir = config.homeDir ?? homedir6();
|
|
6863
7504
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6864
7505
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6865
7506
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7507
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6866
7508
|
}
|
|
6867
7509
|
get isV2() {
|
|
6868
7510
|
return this.openCodeClient.version === "v2";
|
|
@@ -6878,7 +7520,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6878
7520
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6879
7521
|
}
|
|
6880
7522
|
async listSessions() {
|
|
6881
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7523
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7524
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7525
|
+
});
|
|
6882
7526
|
}
|
|
6883
7527
|
async sessionExists(sessionId) {
|
|
6884
7528
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7055,6 +7699,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7055
7699
|
}
|
|
7056
7700
|
return ids;
|
|
7057
7701
|
}
|
|
7702
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7703
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7704
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7705
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7706
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7707
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7708
|
+
}
|
|
7709
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7710
|
+
}
|
|
7711
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7712
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7713
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7714
|
+
this.awaitingReadopt.delete(messageId);
|
|
7715
|
+
this.sessions.delete(conversationId);
|
|
7716
|
+
this.supersede(conversationId, sessionId);
|
|
7717
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7718
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7719
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7720
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7721
|
+
this.log({
|
|
7722
|
+
level: "error",
|
|
7723
|
+
message: errorMessage3,
|
|
7724
|
+
conversation_id: conversationId,
|
|
7725
|
+
message_id: messageId
|
|
7726
|
+
});
|
|
7727
|
+
const reportError = await this.reportUntrackableAck(
|
|
7728
|
+
conversationId,
|
|
7729
|
+
messageId,
|
|
7730
|
+
null,
|
|
7731
|
+
errorMessage3
|
|
7732
|
+
);
|
|
7733
|
+
if (reportError !== null) {
|
|
7734
|
+
this.log({
|
|
7735
|
+
level: "warn",
|
|
7736
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7737
|
+
conversation_id: conversationId,
|
|
7738
|
+
message_id: messageId
|
|
7739
|
+
});
|
|
7740
|
+
}
|
|
7741
|
+
if (source === "readopt") {
|
|
7742
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7743
|
+
}
|
|
7744
|
+
}
|
|
7058
7745
|
/**
|
|
7059
7746
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7060
7747
|
*
|
|
@@ -7234,6 +7921,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7234
7921
|
}
|
|
7235
7922
|
break;
|
|
7236
7923
|
}
|
|
7924
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7925
|
+
if (correlation) {
|
|
7926
|
+
correlation.attempts += 1;
|
|
7927
|
+
let correlatedId = null;
|
|
7928
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7929
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7930
|
+
conv,
|
|
7931
|
+
message,
|
|
7932
|
+
correlation.sessionId
|
|
7933
|
+
);
|
|
7934
|
+
if (polled.ok)
|
|
7935
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7936
|
+
} else if (correlation.knownUserIds === null) {
|
|
7937
|
+
this.log({
|
|
7938
|
+
level: "debug",
|
|
7939
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7940
|
+
conversation_id: conv.id,
|
|
7941
|
+
message_id: message.id
|
|
7942
|
+
});
|
|
7943
|
+
}
|
|
7944
|
+
if (correlatedId !== null) {
|
|
7945
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7946
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7947
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7948
|
+
this.dispatched.add(message.id);
|
|
7949
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7950
|
+
dispatched += 1;
|
|
7951
|
+
this.log({
|
|
7952
|
+
level: "info",
|
|
7953
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7954
|
+
conversation_id: conv.id,
|
|
7955
|
+
message_id: message.id
|
|
7956
|
+
});
|
|
7957
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7958
|
+
opencode_message_id: correlatedId
|
|
7959
|
+
});
|
|
7960
|
+
continue;
|
|
7961
|
+
}
|
|
7962
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7963
|
+
await this.handoffUntrackableCorrelation(
|
|
7964
|
+
conv.id,
|
|
7965
|
+
message.id,
|
|
7966
|
+
correlation.sessionId,
|
|
7967
|
+
correlation.attempts,
|
|
7968
|
+
"fresh"
|
|
7969
|
+
);
|
|
7970
|
+
break;
|
|
7971
|
+
}
|
|
7972
|
+
break;
|
|
7973
|
+
}
|
|
7237
7974
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7238
7975
|
if (effectiveOpencodeMessageId) {
|
|
7239
7976
|
const outcome = await this.resolveRedrive(
|
|
@@ -7255,6 +7992,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7255
7992
|
model: message.opencode_model ?? void 0
|
|
7256
7993
|
};
|
|
7257
7994
|
let opencodeMessageId;
|
|
7995
|
+
let preDispatchUserIds = null;
|
|
7258
7996
|
try {
|
|
7259
7997
|
this.log({
|
|
7260
7998
|
level: "info",
|
|
@@ -7284,7 +8022,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7284
8022
|
message.content,
|
|
7285
8023
|
options,
|
|
7286
8024
|
sendAttachments,
|
|
7287
|
-
this.openCodeClient
|
|
8025
|
+
this.openCodeClient,
|
|
8026
|
+
{
|
|
8027
|
+
onKnownUserIds: (ids) => {
|
|
8028
|
+
preDispatchUserIds = ids;
|
|
8029
|
+
},
|
|
8030
|
+
onKnownUserIdsError: (error2) => {
|
|
8031
|
+
this.log({
|
|
8032
|
+
level: "warn",
|
|
8033
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8034
|
+
conversation_id: conv.id,
|
|
8035
|
+
message_id: message.id
|
|
8036
|
+
});
|
|
8037
|
+
}
|
|
8038
|
+
}
|
|
7288
8039
|
)
|
|
7289
8040
|
);
|
|
7290
8041
|
} catch (err) {
|
|
@@ -7292,6 +8043,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7292
8043
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7293
8044
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7294
8045
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8046
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7295
8047
|
this.log({
|
|
7296
8048
|
level: "error",
|
|
7297
8049
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7367,39 +8119,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7367
8119
|
if (this.isV2) {
|
|
7368
8120
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7369
8121
|
}
|
|
7370
|
-
|
|
7371
|
-
|
|
7372
|
-
|
|
7373
|
-
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
message_id: message.id
|
|
7377
|
-
});
|
|
7378
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7379
|
-
continue;
|
|
7380
|
-
}
|
|
7381
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7382
|
-
this.sessions.delete(conv.id);
|
|
7383
|
-
this.supersede(conv.id, sessionId);
|
|
7384
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8122
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8123
|
+
sessionId,
|
|
8124
|
+
content: message.content,
|
|
8125
|
+
attempts: 0,
|
|
8126
|
+
knownUserIds: preDispatchUserIds
|
|
8127
|
+
});
|
|
7385
8128
|
this.log({
|
|
7386
|
-
level: "
|
|
7387
|
-
message:
|
|
8129
|
+
level: "warn",
|
|
8130
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7388
8131
|
conversation_id: conv.id,
|
|
7389
8132
|
message_id: message.id
|
|
7390
8133
|
});
|
|
7391
|
-
|
|
7392
|
-
this.log({
|
|
7393
|
-
level: "warn",
|
|
7394
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7395
|
-
conversation_id: conv.id,
|
|
7396
|
-
message_id: message.id
|
|
7397
|
-
});
|
|
7398
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7399
|
-
});
|
|
8134
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7400
8135
|
break;
|
|
7401
8136
|
}
|
|
7402
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7403
8137
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7404
8138
|
this.releasedOpencodeIds.delete(message.id);
|
|
7405
8139
|
this.dispatched.add(message.id);
|
|
@@ -8004,23 +8738,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8004
8738
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8005
8739
|
return 1;
|
|
8006
8740
|
}
|
|
8007
|
-
/**
|
|
8008
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8009
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8010
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8011
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8012
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8013
|
-
* count, since a new session is a genuinely different attempt.
|
|
8014
|
-
*/
|
|
8015
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8016
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8017
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8018
|
-
existing.count += 1;
|
|
8019
|
-
return existing.count;
|
|
8020
|
-
}
|
|
8021
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8022
|
-
return 1;
|
|
8023
|
-
}
|
|
8024
8741
|
/** Record one queued give-up toward the never-started bound. */
|
|
8025
8742
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8026
8743
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8332,8 +9049,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8332
9049
|
b2LastDescendantCheckMs: 0,
|
|
8333
9050
|
b2RootOngoingHeldLogged: false,
|
|
8334
9051
|
b2AbandonedSignalled: false,
|
|
9052
|
+
b2AbandonedWithoutAnswer: false,
|
|
8335
9053
|
ambiguousPinnedSinceMs: 0,
|
|
8336
9054
|
ambiguousResolved: false,
|
|
9055
|
+
sessionStoreFailurePending: false,
|
|
8337
9056
|
neverStartedResolved: false
|
|
8338
9057
|
});
|
|
8339
9058
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8355,6 +9074,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8355
9074
|
this.bufferedSessionErrors.delete(oldest);
|
|
8356
9075
|
}
|
|
8357
9076
|
}
|
|
9077
|
+
retainSessionError(event) {
|
|
9078
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9079
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9080
|
+
reason: event.reason,
|
|
9081
|
+
observedAt: this.now()
|
|
9082
|
+
});
|
|
9083
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9084
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9085
|
+
if (typeof oldest !== "string") break;
|
|
9086
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9087
|
+
}
|
|
9088
|
+
}
|
|
9089
|
+
readRetainedSessionError(sessionId) {
|
|
9090
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9091
|
+
if (!retained) return null;
|
|
9092
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9093
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9094
|
+
return null;
|
|
9095
|
+
}
|
|
9096
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9097
|
+
return retained.reason;
|
|
9098
|
+
}
|
|
8358
9099
|
/**
|
|
8359
9100
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8360
9101
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8450,8 +9191,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8450
9191
|
b2LastDescendantCheckMs: 0,
|
|
8451
9192
|
b2RootOngoingHeldLogged: false,
|
|
8452
9193
|
b2AbandonedSignalled: false,
|
|
9194
|
+
b2AbandonedWithoutAnswer: false,
|
|
8453
9195
|
ambiguousPinnedSinceMs: 0,
|
|
8454
9196
|
ambiguousResolved: false,
|
|
9197
|
+
sessionStoreFailurePending: false,
|
|
8455
9198
|
neverStartedResolved: false
|
|
8456
9199
|
});
|
|
8457
9200
|
}
|
|
@@ -8679,9 +9422,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8679
9422
|
return;
|
|
8680
9423
|
}
|
|
8681
9424
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9425
|
+
this.retainSessionError(event);
|
|
8682
9426
|
this.log({
|
|
8683
|
-
level: "
|
|
8684
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9427
|
+
level: "warn",
|
|
9428
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8685
9429
|
conversation_id: watcher.conv.id
|
|
8686
9430
|
});
|
|
8687
9431
|
return;
|
|
@@ -8711,9 +9455,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8711
9455
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8712
9456
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8713
9457
|
if (state !== "queued") {
|
|
9458
|
+
this.retainSessionError(event);
|
|
8714
9459
|
this.log({
|
|
8715
|
-
level: "
|
|
8716
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9460
|
+
level: "warn",
|
|
9461
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8717
9462
|
conversation_id: watcher.conv.id,
|
|
8718
9463
|
message_id: inFlight.evidentMessageId
|
|
8719
9464
|
});
|
|
@@ -8865,6 +9610,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8865
9610
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8866
9611
|
this.neverStartedAttempts.delete(id);
|
|
8867
9612
|
}
|
|
9613
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9614
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9615
|
+
return;
|
|
9616
|
+
}
|
|
8868
9617
|
if (inFlight.neverStartedResolved) {
|
|
8869
9618
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8870
9619
|
inFlight.neverStartedResolved = false;
|
|
@@ -8938,7 +9687,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8938
9687
|
if (state === "failed") {
|
|
8939
9688
|
this.anchorDeliveryDeadline(inFlight);
|
|
8940
9689
|
if (!inFlight.done) {
|
|
8941
|
-
const
|
|
9690
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9691
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9692
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9693
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9694
|
+
const runnerDetails = [retainedError, storeError].filter(
|
|
9695
|
+
(detail) => detail !== null
|
|
9696
|
+
);
|
|
9697
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
8942
9698
|
this.log({
|
|
8943
9699
|
level: "error",
|
|
8944
9700
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9024,10 +9780,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9024
9780
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9025
9781
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9026
9782
|
inFlight.b2AbandonedSignalled = false;
|
|
9783
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9027
9784
|
}
|
|
9028
9785
|
} else {
|
|
9029
9786
|
if (inFlight.b2AbandonedSignalled) {
|
|
9030
|
-
|
|
9787
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9788
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9789
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9790
|
+
sessionId,
|
|
9791
|
+
watcher,
|
|
9792
|
+
inFlight,
|
|
9793
|
+
messages,
|
|
9794
|
+
reply?.info?.finish ?? reply?.finish
|
|
9795
|
+
);
|
|
9796
|
+
} else {
|
|
9797
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9798
|
+
}
|
|
9031
9799
|
return;
|
|
9032
9800
|
}
|
|
9033
9801
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9044,17 +9812,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9044
9812
|
descendantOngoing,
|
|
9045
9813
|
rootOngoing
|
|
9046
9814
|
})) {
|
|
9815
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9816
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9817
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9818
|
+
watched_for_ms: pinnedForMs,
|
|
9819
|
+
run_state: "failed",
|
|
9820
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9821
|
+
});
|
|
9822
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9823
|
+
return;
|
|
9824
|
+
}
|
|
9047
9825
|
inFlight.b2AbandonedSignalled = true;
|
|
9826
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9827
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9828
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9829
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9830
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9831
|
+
watched_for_ms: pinnedForMs,
|
|
9832
|
+
run_state: "failed",
|
|
9833
|
+
finish,
|
|
9834
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9835
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9836
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9837
|
+
});
|
|
9838
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9839
|
+
sessionId,
|
|
9840
|
+
watcher,
|
|
9841
|
+
inFlight,
|
|
9842
|
+
messages,
|
|
9843
|
+
finish
|
|
9844
|
+
);
|
|
9845
|
+
return;
|
|
9846
|
+
}
|
|
9048
9847
|
this.log({
|
|
9049
9848
|
level: "warn",
|
|
9050
9849
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9051
9850
|
conversation_id: conv.id,
|
|
9052
9851
|
message_id: id
|
|
9053
9852
|
});
|
|
9054
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9055
9853
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9056
9854
|
watched_for_ms: pinnedForMs,
|
|
9057
|
-
finish
|
|
9855
|
+
finish,
|
|
9058
9856
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9059
9857
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9060
9858
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9102,6 +9900,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9102
9900
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9103
9901
|
sessionOngoing: ongoing
|
|
9104
9902
|
})) {
|
|
9903
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9904
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9905
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9906
|
+
watched_for_ms: pinnedForMs,
|
|
9907
|
+
run_state: "failed",
|
|
9908
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9909
|
+
});
|
|
9910
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9911
|
+
return;
|
|
9912
|
+
}
|
|
9105
9913
|
inFlight.ambiguousResolved = true;
|
|
9106
9914
|
this.log({
|
|
9107
9915
|
level: "warn",
|
|
@@ -9310,20 +10118,88 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9310
10118
|
}
|
|
9311
10119
|
this.log({
|
|
9312
10120
|
level: "warn",
|
|
9313
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10121
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10122
|
+
conversation_id: conv.id,
|
|
10123
|
+
message_id: inFlight.evidentMessageId
|
|
10124
|
+
});
|
|
10125
|
+
return;
|
|
10126
|
+
}
|
|
10127
|
+
inFlight.done = true;
|
|
10128
|
+
await this.reportSubagentAuthFailures(
|
|
10129
|
+
watcher.conv.id,
|
|
10130
|
+
inFlight.opencodeMessageId,
|
|
10131
|
+
inFlight.evidentMessageId,
|
|
10132
|
+
messages
|
|
10133
|
+
);
|
|
10134
|
+
}
|
|
10135
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10136
|
+
}
|
|
10137
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10138
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10139
|
+
const conv = watcher.conv;
|
|
10140
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
10141
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10142
|
+
this.log({
|
|
10143
|
+
level: "warn",
|
|
10144
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10145
|
+
conversation_id: conv.id,
|
|
10146
|
+
message_id: inFlight.evidentMessageId
|
|
10147
|
+
});
|
|
10148
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10149
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10150
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10151
|
+
messages,
|
|
10152
|
+
inFlight.opencodeMessageId,
|
|
10153
|
+
inFlight.evidentMessageId
|
|
10154
|
+
);
|
|
10155
|
+
try {
|
|
10156
|
+
await this.markFailed(
|
|
10157
|
+
conv.id,
|
|
10158
|
+
inFlight.evidentMessageId,
|
|
10159
|
+
sessionId,
|
|
10160
|
+
error2,
|
|
10161
|
+
usage,
|
|
10162
|
+
void 0,
|
|
10163
|
+
usageAgentName,
|
|
10164
|
+
subagentInvocations
|
|
10165
|
+
);
|
|
10166
|
+
} catch (err) {
|
|
10167
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10168
|
+
if (err instanceof ChannelTerminalError) {
|
|
10169
|
+
this.log({
|
|
10170
|
+
level: "warn",
|
|
10171
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9314
10172
|
conversation_id: conv.id,
|
|
9315
10173
|
message_id: inFlight.evidentMessageId
|
|
9316
10174
|
});
|
|
10175
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9317
10176
|
return;
|
|
9318
10177
|
}
|
|
9319
|
-
inFlight.
|
|
9320
|
-
|
|
9321
|
-
|
|
9322
|
-
|
|
9323
|
-
|
|
9324
|
-
|
|
9325
|
-
|
|
10178
|
+
if (this.now() >= inFlight.deadline) {
|
|
10179
|
+
this.log({
|
|
10180
|
+
level: "warn",
|
|
10181
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10182
|
+
conversation_id: conv.id,
|
|
10183
|
+
message_id: inFlight.evidentMessageId
|
|
10184
|
+
});
|
|
10185
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10186
|
+
return;
|
|
10187
|
+
}
|
|
10188
|
+
this.log({
|
|
10189
|
+
level: "warn",
|
|
10190
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10191
|
+
conversation_id: conv.id,
|
|
10192
|
+
message_id: inFlight.evidentMessageId
|
|
10193
|
+
});
|
|
10194
|
+
return;
|
|
9326
10195
|
}
|
|
10196
|
+
inFlight.done = true;
|
|
10197
|
+
await this.reportSubagentAuthFailures(
|
|
10198
|
+
watcher.conv.id,
|
|
10199
|
+
inFlight.opencodeMessageId,
|
|
10200
|
+
inFlight.evidentMessageId,
|
|
10201
|
+
messages
|
|
10202
|
+
);
|
|
9327
10203
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9328
10204
|
}
|
|
9329
10205
|
/**
|
|
@@ -9381,6 +10257,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9381
10257
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9382
10258
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9383
10259
|
}
|
|
10260
|
+
/**
|
|
10261
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10262
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10263
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10264
|
+
*/
|
|
10265
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10266
|
+
const conv = watcher.conv;
|
|
10267
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10268
|
+
this.log({
|
|
10269
|
+
level: "error",
|
|
10270
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10271
|
+
conversation_id: conv.id,
|
|
10272
|
+
message_id: inFlight.evidentMessageId
|
|
10273
|
+
});
|
|
10274
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10275
|
+
try {
|
|
10276
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10277
|
+
} catch (err) {
|
|
10278
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10279
|
+
if (err instanceof ChannelTerminalError) {
|
|
10280
|
+
this.log({
|
|
10281
|
+
level: "warn",
|
|
10282
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10283
|
+
conversation_id: conv.id,
|
|
10284
|
+
message_id: inFlight.evidentMessageId
|
|
10285
|
+
});
|
|
10286
|
+
this.recordReleasedOpencodeId(
|
|
10287
|
+
inFlight.evidentMessageId,
|
|
10288
|
+
sessionId,
|
|
10289
|
+
inFlight.opencodeMessageId
|
|
10290
|
+
);
|
|
10291
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10292
|
+
return;
|
|
10293
|
+
}
|
|
10294
|
+
if (this.now() >= inFlight.deadline) {
|
|
10295
|
+
this.log({
|
|
10296
|
+
level: "warn",
|
|
10297
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10298
|
+
conversation_id: conv.id,
|
|
10299
|
+
message_id: inFlight.evidentMessageId
|
|
10300
|
+
});
|
|
10301
|
+
this.recordReleasedOpencodeId(
|
|
10302
|
+
inFlight.evidentMessageId,
|
|
10303
|
+
sessionId,
|
|
10304
|
+
inFlight.opencodeMessageId
|
|
10305
|
+
);
|
|
10306
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10307
|
+
return;
|
|
10308
|
+
}
|
|
10309
|
+
this.log({
|
|
10310
|
+
level: "warn",
|
|
10311
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10312
|
+
conversation_id: conv.id,
|
|
10313
|
+
message_id: inFlight.evidentMessageId
|
|
10314
|
+
});
|
|
10315
|
+
return;
|
|
10316
|
+
}
|
|
10317
|
+
inFlight.done = true;
|
|
10318
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10319
|
+
}
|
|
9384
10320
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9385
10321
|
/**
|
|
9386
10322
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9419,11 +10355,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9419
10355
|
}
|
|
9420
10356
|
}
|
|
9421
10357
|
}
|
|
9422
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10358
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9423
10359
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10360
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10361
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10362
|
+
this.awaitingCorrelation.delete(id);
|
|
10363
|
+
this.log({
|
|
10364
|
+
level: "debug",
|
|
10365
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10366
|
+
message_id: id
|
|
10367
|
+
});
|
|
10368
|
+
}
|
|
10369
|
+
}
|
|
9424
10370
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9425
10371
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9426
10372
|
this.untrackableAck.delete(id);
|
|
10373
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9427
10374
|
this.untrackableAckSignalled.delete(id);
|
|
9428
10375
|
this.untrackableAckWarned.delete(id);
|
|
9429
10376
|
this.log({
|
|
@@ -9463,7 +10410,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9463
10410
|
bySession.set(row.opencode_session_id, list);
|
|
9464
10411
|
}
|
|
9465
10412
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9466
|
-
let messages;
|
|
10413
|
+
let messages = null;
|
|
9467
10414
|
try {
|
|
9468
10415
|
if (this.isV2) {
|
|
9469
10416
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9472,9 +10419,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9472
10419
|
level: "warn",
|
|
9473
10420
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9474
10421
|
});
|
|
9475
|
-
|
|
10422
|
+
} else {
|
|
10423
|
+
messages = snapshot;
|
|
9476
10424
|
}
|
|
9477
|
-
messages = snapshot;
|
|
9478
10425
|
} else {
|
|
9479
10426
|
const polled = await pollSessionMessagesForRedrive(
|
|
9480
10427
|
this.port,
|
|
@@ -9487,15 +10434,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9487
10434
|
level: "warn",
|
|
9488
10435
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9489
10436
|
});
|
|
9490
|
-
|
|
10437
|
+
} else {
|
|
10438
|
+
messages = polled.messages;
|
|
9491
10439
|
}
|
|
9492
|
-
messages = polled.messages;
|
|
9493
10440
|
}
|
|
9494
10441
|
} catch (err) {
|
|
9495
10442
|
this.log({
|
|
9496
10443
|
level: "warn",
|
|
9497
10444
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9498
10445
|
});
|
|
10446
|
+
}
|
|
10447
|
+
if (messages === null) {
|
|
10448
|
+
for (const row of sessionRows) {
|
|
10449
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10450
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10451
|
+
}
|
|
10452
|
+
}
|
|
9499
10453
|
continue;
|
|
9500
10454
|
}
|
|
9501
10455
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9505,6 +10459,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9505
10459
|
}
|
|
9506
10460
|
}
|
|
9507
10461
|
}
|
|
10462
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10463
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10464
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10465
|
+
if (!correlation) return false;
|
|
10466
|
+
correlation.attempts += 1;
|
|
10467
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10468
|
+
if (correlatedId !== null) {
|
|
10469
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10470
|
+
const conv = this.convForRow(sessionId, row);
|
|
10471
|
+
const message = this.queuedMessageForRow(row);
|
|
10472
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10473
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10474
|
+
this.dispatched.add(row.id);
|
|
10475
|
+
this.readopted.add(row.id);
|
|
10476
|
+
this.ensureWatcherRunning(sessionId);
|
|
10477
|
+
this.log({
|
|
10478
|
+
level: "info",
|
|
10479
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10480
|
+
conversation_id: row.conversation_id,
|
|
10481
|
+
message_id: row.id
|
|
10482
|
+
});
|
|
10483
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10484
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10485
|
+
opencode_message_id: correlatedId
|
|
10486
|
+
});
|
|
10487
|
+
return true;
|
|
10488
|
+
}
|
|
10489
|
+
if (correlation.knownUserIds === null) {
|
|
10490
|
+
this.log({
|
|
10491
|
+
level: "debug",
|
|
10492
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10493
|
+
conversation_id: row.conversation_id,
|
|
10494
|
+
message_id: row.id
|
|
10495
|
+
});
|
|
10496
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10497
|
+
this.log({
|
|
10498
|
+
level: "warn",
|
|
10499
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10500
|
+
conversation_id: row.conversation_id,
|
|
10501
|
+
message_id: row.id
|
|
10502
|
+
});
|
|
10503
|
+
}
|
|
10504
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10505
|
+
await this.handoffUntrackableCorrelation(
|
|
10506
|
+
row.conversation_id,
|
|
10507
|
+
row.id,
|
|
10508
|
+
correlation.sessionId,
|
|
10509
|
+
correlation.attempts,
|
|
10510
|
+
"readopt"
|
|
10511
|
+
);
|
|
10512
|
+
}
|
|
10513
|
+
return true;
|
|
10514
|
+
}
|
|
9508
10515
|
/**
|
|
9509
10516
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9510
10517
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9512,8 +10519,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9512
10519
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9513
10520
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9514
10521
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9515
|
-
* before the restart)
|
|
9516
|
-
*
|
|
10522
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10523
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10524
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10525
|
+
* `forceReadoptRun`):
|
|
9517
10526
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9518
10527
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9519
10528
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9522,7 +10531,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9522
10531
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9523
10532
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9524
10533
|
* tracking the stored id so the reply correlates by it;
|
|
9525
|
-
* - `unknown`/null id →
|
|
10534
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10535
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9526
10536
|
*
|
|
9527
10537
|
* Only `ChannelAuthError` propagates.
|
|
9528
10538
|
*/
|
|
@@ -9536,6 +10546,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9536
10546
|
});
|
|
9537
10547
|
return;
|
|
9538
10548
|
}
|
|
10549
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9539
10550
|
const ocId = row.opencode_message_id;
|
|
9540
10551
|
const state = messageRunState(messages, ocId ?? "");
|
|
9541
10552
|
if (state === "done") {
|
|
@@ -9602,12 +10613,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9602
10613
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9603
10614
|
this.dontRedispatch.delete(row.id);
|
|
9604
10615
|
this.untrackableAck.delete(row.id);
|
|
10616
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9605
10617
|
this.untrackableAckSignalled.delete(row.id);
|
|
9606
10618
|
this.untrackableAckWarned.delete(row.id);
|
|
9607
10619
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9608
10620
|
return;
|
|
9609
10621
|
}
|
|
9610
|
-
if (this.
|
|
10622
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10623
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10624
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10625
|
+
this.untrackableAckWarned.add(row.id);
|
|
10626
|
+
this.log({
|
|
10627
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10628
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10629
|
+
conversation_id: row.conversation_id,
|
|
10630
|
+
message_id: row.id
|
|
10631
|
+
});
|
|
10632
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10633
|
+
const reportError = await this.reportUntrackableAck(
|
|
10634
|
+
row.conversation_id,
|
|
10635
|
+
row.id,
|
|
10636
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10637
|
+
errorMessage3
|
|
10638
|
+
);
|
|
10639
|
+
if (reportError !== null) {
|
|
10640
|
+
this.log({
|
|
10641
|
+
level: "warn",
|
|
10642
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10643
|
+
conversation_id: row.conversation_id,
|
|
10644
|
+
message_id: row.id
|
|
10645
|
+
});
|
|
10646
|
+
}
|
|
10647
|
+
} else {
|
|
10648
|
+
this.log({
|
|
10649
|
+
level: "debug",
|
|
10650
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10651
|
+
conversation_id: row.conversation_id,
|
|
10652
|
+
message_id: row.id
|
|
10653
|
+
});
|
|
10654
|
+
}
|
|
10655
|
+
return;
|
|
10656
|
+
}
|
|
10657
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9611
10658
|
this.log({
|
|
9612
10659
|
level: "debug",
|
|
9613
10660
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9785,6 +10832,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9785
10832
|
}
|
|
9786
10833
|
this.dontRedispatch.delete(row.id);
|
|
9787
10834
|
this.untrackableAck.delete(row.id);
|
|
10835
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9788
10836
|
this.untrackableAckSignalled.delete(row.id);
|
|
9789
10837
|
this.untrackableAckWarned.delete(row.id);
|
|
9790
10838
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9804,8 +10852,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9804
10852
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9805
10853
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9806
10854
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9807
|
-
* re-entry)
|
|
9808
|
-
*
|
|
10855
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10856
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10857
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9809
10858
|
*
|
|
9810
10859
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9811
10860
|
* `processed_at` (Invariant 1).
|
|
@@ -9868,6 +10917,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9868
10917
|
);
|
|
9869
10918
|
}
|
|
9870
10919
|
let ocId;
|
|
10920
|
+
let preDispatchUserIds = null;
|
|
9871
10921
|
try {
|
|
9872
10922
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9873
10923
|
sessionId,
|
|
@@ -9877,7 +10927,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9877
10927
|
row.content,
|
|
9878
10928
|
options,
|
|
9879
10929
|
sendAttachments,
|
|
9880
|
-
this.openCodeClient
|
|
10930
|
+
this.openCodeClient,
|
|
10931
|
+
{
|
|
10932
|
+
onKnownUserIds: (ids) => {
|
|
10933
|
+
preDispatchUserIds = ids;
|
|
10934
|
+
},
|
|
10935
|
+
onKnownUserIdsError: (error2) => {
|
|
10936
|
+
this.log({
|
|
10937
|
+
level: "warn",
|
|
10938
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10939
|
+
conversation_id: row.conversation_id,
|
|
10940
|
+
message_id: row.id
|
|
10941
|
+
});
|
|
10942
|
+
}
|
|
10943
|
+
}
|
|
9881
10944
|
)
|
|
9882
10945
|
);
|
|
9883
10946
|
} catch (err) {
|
|
@@ -9910,6 +10973,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9910
10973
|
}
|
|
9911
10974
|
} else {
|
|
9912
10975
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
10976
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
9913
10977
|
}
|
|
9914
10978
|
this.log({
|
|
9915
10979
|
level: "error",
|
|
@@ -9958,39 +11022,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9958
11022
|
}
|
|
9959
11023
|
if (ocId === null) {
|
|
9960
11024
|
this.awaitingReadopt.delete(row.id);
|
|
9961
|
-
|
|
9962
|
-
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
this.log({
|
|
9968
|
-
level: "error",
|
|
9969
|
-
message: errorMessage3,
|
|
9970
|
-
conversation_id: row.conversation_id,
|
|
9971
|
-
message_id: row.id
|
|
9972
|
-
});
|
|
9973
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
9974
|
-
this.log({
|
|
9975
|
-
level: "warn",
|
|
9976
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
9977
|
-
conversation_id: row.conversation_id,
|
|
9978
|
-
message_id: row.id
|
|
9979
|
-
});
|
|
9980
|
-
});
|
|
9981
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9982
|
-
return;
|
|
9983
|
-
}
|
|
11025
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11026
|
+
sessionId,
|
|
11027
|
+
content: row.content,
|
|
11028
|
+
attempts: 0,
|
|
11029
|
+
knownUserIds: preDispatchUserIds
|
|
11030
|
+
});
|
|
9984
11031
|
this.log({
|
|
9985
11032
|
level: "warn",
|
|
9986
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11033
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
9987
11034
|
conversation_id: row.conversation_id,
|
|
9988
11035
|
message_id: row.id
|
|
9989
11036
|
});
|
|
9990
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9991
11037
|
return;
|
|
9992
11038
|
}
|
|
9993
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
9994
11039
|
const processedAtMs = this.processedAtMs(row);
|
|
9995
11040
|
this.registerReadopted(
|
|
9996
11041
|
readoptConv,
|
|
@@ -12112,6 +13157,7 @@ function createCredentialSync({
|
|
|
12112
13157
|
|
|
12113
13158
|
// src/commands/run.ts
|
|
12114
13159
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
13160
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12115
13161
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12116
13162
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12117
13163
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12267,7 +13313,13 @@ function logActivity(state, entry) {
|
|
|
12267
13313
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
12268
13314
|
if (!meetsThreshold(state, level)) return;
|
|
12269
13315
|
forwardRunnerActivity(
|
|
12270
|
-
{
|
|
13316
|
+
{
|
|
13317
|
+
level,
|
|
13318
|
+
message: entry.message,
|
|
13319
|
+
error: entry.error,
|
|
13320
|
+
metadata: entry.metadata,
|
|
13321
|
+
...entry.source ? { source: entry.source } : {}
|
|
13322
|
+
},
|
|
12271
13323
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
12272
13324
|
);
|
|
12273
13325
|
const fullEntry = {
|
|
@@ -12289,7 +13341,7 @@ function logActivity(state, entry) {
|
|
|
12289
13341
|
}
|
|
12290
13342
|
function reportSessionDbRecovery(state) {
|
|
12291
13343
|
try {
|
|
12292
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
13344
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir7(), env: process.env });
|
|
12293
13345
|
for (const record of report.records) {
|
|
12294
13346
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
12295
13347
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -12324,7 +13376,9 @@ function displayStatus(state) {
|
|
|
12324
13376
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12325
13377
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12326
13378
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12327
|
-
const detail = last ? chalk7.dim(
|
|
13379
|
+
const detail = last ? chalk7.dim(
|
|
13380
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13381
|
+
) : "";
|
|
12328
13382
|
const agent = state.agentName ?? state.agentId;
|
|
12329
13383
|
console.log(
|
|
12330
13384
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12343,7 +13397,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12343
13397
|
logActivity(state, {
|
|
12344
13398
|
type: "info",
|
|
12345
13399
|
level: "warn",
|
|
12346
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13400
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12347
13401
|
});
|
|
12348
13402
|
}
|
|
12349
13403
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12368,7 +13422,7 @@ async function importOpenCodeV2CredentialsForState(state) {
|
|
|
12368
13422
|
}
|
|
12369
13423
|
const result = await importOpenCodeV2Credentials({
|
|
12370
13424
|
request: client.request,
|
|
12371
|
-
homeDir:
|
|
13425
|
+
homeDir: homedir7(),
|
|
12372
13426
|
directory,
|
|
12373
13427
|
guard: state.opencodeV2CredentialImportGuard,
|
|
12374
13428
|
log: (message, level = "info") => logActivity(state, { type: "info", level, message })
|
|
@@ -12576,9 +13630,9 @@ async function driveChannels(state, driver) {
|
|
|
12576
13630
|
}
|
|
12577
13631
|
}
|
|
12578
13632
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12579
|
-
var
|
|
13633
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12580
13634
|
function shouldWarnForReclaimSkip(reason) {
|
|
12581
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13635
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12582
13636
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12583
13637
|
if (!version2) return false;
|
|
12584
13638
|
const major = Number(version2[1]);
|
|
@@ -12587,7 +13641,14 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12587
13641
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12588
13642
|
}
|
|
12589
13643
|
function sessionDbPath() {
|
|
12590
|
-
return join11(
|
|
13644
|
+
return join11(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
13645
|
+
}
|
|
13646
|
+
function logSessionCleanupActivity(state, phase, entry) {
|
|
13647
|
+
logActivity(state, {
|
|
13648
|
+
...entry,
|
|
13649
|
+
source: "cli.session-cleanup",
|
|
13650
|
+
metadata: { ...entry.metadata, phase }
|
|
13651
|
+
});
|
|
12591
13652
|
}
|
|
12592
13653
|
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
12593
13654
|
const record = {
|
|
@@ -12628,69 +13689,136 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12628
13689
|
async function runSweep(state, driver, config) {
|
|
12629
13690
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12630
13691
|
try {
|
|
12631
|
-
const
|
|
12632
|
-
|
|
13692
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13693
|
+
const sessions = enumeration.sessions;
|
|
13694
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13695
|
+
if (enumerationReason !== null) {
|
|
12633
13696
|
logActivity(state, {
|
|
12634
13697
|
type: "info",
|
|
12635
|
-
|
|
13698
|
+
level: "warn",
|
|
13699
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12636
13700
|
});
|
|
12637
|
-
return;
|
|
12638
13701
|
}
|
|
12639
|
-
const toDelete = selectSessionsToDelete(
|
|
12640
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12641
|
-
{
|
|
12642
|
-
maxAgeMs: config.maxAgeMs,
|
|
12643
|
-
maxCount: config.maxCount,
|
|
12644
|
-
nowMs: Date.now(),
|
|
12645
|
-
protectedIds: driver.protectedSessionIds()
|
|
12646
|
-
}
|
|
12647
|
-
);
|
|
12648
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12649
13702
|
let deleted = 0;
|
|
12650
13703
|
let failed = 0;
|
|
13704
|
+
let failed4xx = 0;
|
|
13705
|
+
let failed5xx = 0;
|
|
13706
|
+
let failedOtherStatus = 0;
|
|
13707
|
+
let failedThrew = 0;
|
|
12651
13708
|
let skippedNewlyActive = 0;
|
|
12652
|
-
|
|
12653
|
-
|
|
12654
|
-
|
|
12655
|
-
|
|
12656
|
-
|
|
12657
|
-
|
|
12658
|
-
|
|
12659
|
-
|
|
13709
|
+
let remaining = 0;
|
|
13710
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13711
|
+
if (sessions !== null) {
|
|
13712
|
+
const toDelete = selectSessionsToDelete(
|
|
13713
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13714
|
+
{
|
|
13715
|
+
maxAgeMs: config.maxAgeMs,
|
|
13716
|
+
maxCount: config.maxCount,
|
|
13717
|
+
nowMs: Date.now(),
|
|
13718
|
+
protectedIds: protectedNow
|
|
13719
|
+
}
|
|
13720
|
+
);
|
|
13721
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13722
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13723
|
+
protectedNow = driver.protectedSessionIds();
|
|
13724
|
+
for (const id of deleteCandidates) {
|
|
13725
|
+
if (protectedNow.has(id)) {
|
|
13726
|
+
skippedNewlyActive++;
|
|
13727
|
+
logActivity(state, {
|
|
13728
|
+
type: "info",
|
|
13729
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13730
|
+
});
|
|
13731
|
+
continue;
|
|
13732
|
+
}
|
|
13733
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13734
|
+
if (outcome.ok) {
|
|
13735
|
+
deleted++;
|
|
13736
|
+
} else if (outcome.status === "threw") {
|
|
13737
|
+
failed++;
|
|
13738
|
+
failedThrew++;
|
|
13739
|
+
} else {
|
|
13740
|
+
failed++;
|
|
13741
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13742
|
+
if (statusClass === 4) failed4xx++;
|
|
13743
|
+
else if (statusClass === 5) failed5xx++;
|
|
13744
|
+
else failedOtherStatus++;
|
|
13745
|
+
}
|
|
12660
13746
|
}
|
|
12661
|
-
|
|
12662
|
-
|
|
13747
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13748
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13749
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13750
|
+
logActivity(state, {
|
|
13751
|
+
type: "info",
|
|
13752
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13753
|
+
});
|
|
12663
13754
|
}
|
|
12664
|
-
const
|
|
12665
|
-
|
|
12666
|
-
|
|
12667
|
-
|
|
12668
|
-
|
|
12669
|
-
|
|
13755
|
+
const sessionMetadata = {
|
|
13756
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13757
|
+
sessions_deleted: deleted,
|
|
13758
|
+
sessions_failed: failed,
|
|
13759
|
+
sessions_failed_4xx: failed4xx,
|
|
13760
|
+
sessions_failed_5xx: failed5xx,
|
|
13761
|
+
sessions_failed_other: failedOtherStatus,
|
|
13762
|
+
sessions_failed_threw: failedThrew,
|
|
13763
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13764
|
+
sessions_remaining: remaining,
|
|
13765
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13766
|
+
};
|
|
12670
13767
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12671
13768
|
dbPath: sessionDbPath(),
|
|
12672
|
-
|
|
13769
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13770
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13771
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12673
13772
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12674
13773
|
});
|
|
12675
13774
|
if (reclaimResult.ok) {
|
|
13775
|
+
state.sessionStoreHealth.observeWritable();
|
|
12676
13776
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12677
13777
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12678
13778
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12679
|
-
|
|
13779
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12680
13780
|
type: "info",
|
|
12681
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${
|
|
13781
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
13782
|
+
metadata: {
|
|
13783
|
+
outcome: "reclaimed",
|
|
13784
|
+
mode: reclaimResult.mode,
|
|
13785
|
+
before_bytes: reclaimResult.beforeBytes,
|
|
13786
|
+
after_bytes: reclaimResult.afterBytes,
|
|
13787
|
+
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
13788
|
+
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
13789
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13790
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13791
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13792
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13793
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13794
|
+
...sessionMetadata
|
|
13795
|
+
}
|
|
12682
13796
|
});
|
|
12683
13797
|
} else {
|
|
12684
|
-
|
|
13798
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13799
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13800
|
+
}
|
|
13801
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12685
13802
|
type: "info",
|
|
12686
|
-
|
|
13803
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13804
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
13805
|
+
metadata: {
|
|
13806
|
+
outcome: "reclaim-skipped",
|
|
13807
|
+
reason: reclaimResult.skipped,
|
|
13808
|
+
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
13809
|
+
...sessionMetadata
|
|
13810
|
+
}
|
|
12687
13811
|
});
|
|
12688
13812
|
}
|
|
12689
13813
|
} catch (error2) {
|
|
12690
13814
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
12691
|
-
|
|
13815
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12692
13816
|
type: "error",
|
|
12693
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}
|
|
13817
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`,
|
|
13818
|
+
metadata: {
|
|
13819
|
+
outcome: "sweep-failed",
|
|
13820
|
+
reason: "sweep-error"
|
|
13821
|
+
}
|
|
12694
13822
|
});
|
|
12695
13823
|
}
|
|
12696
13824
|
}
|
|
@@ -12706,14 +13834,19 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12706
13834
|
for (const warning2 of config.warnings) {
|
|
12707
13835
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
12708
13836
|
}
|
|
12709
|
-
const dbBytes = statSessionDbBytes(
|
|
13837
|
+
const dbBytes = statSessionDbBytes(homedir7());
|
|
12710
13838
|
void (async () => {
|
|
12711
13839
|
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
12712
13840
|
if (reclaimAvailability !== null) {
|
|
12713
|
-
|
|
13841
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12714
13842
|
type: "info",
|
|
12715
13843
|
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
12716
|
-
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
13844
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : ""),
|
|
13845
|
+
metadata: {
|
|
13846
|
+
outcome: "preflight-skipped",
|
|
13847
|
+
reason: reclaimAvailability.reason,
|
|
13848
|
+
...reclaimAvailability.detail !== void 0 ? { detail: reclaimAvailability.detail } : {}
|
|
13849
|
+
}
|
|
12717
13850
|
});
|
|
12718
13851
|
}
|
|
12719
13852
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -12730,9 +13863,15 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12730
13863
|
);
|
|
12731
13864
|
});
|
|
12732
13865
|
if (!config.enabled) return;
|
|
12733
|
-
|
|
13866
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12734
13867
|
type: "info",
|
|
12735
|
-
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)
|
|
13868
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`,
|
|
13869
|
+
metadata: {
|
|
13870
|
+
outcome: "armed",
|
|
13871
|
+
interval_ms: config.intervalMs,
|
|
13872
|
+
max_age_ms: config.maxAgeMs ?? null,
|
|
13873
|
+
max_count: config.maxCount ?? null
|
|
13874
|
+
}
|
|
12736
13875
|
});
|
|
12737
13876
|
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
12738
13877
|
const firstSweep = setTimeout(
|
|
@@ -12911,7 +14050,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12911
14050
|
});
|
|
12912
14051
|
return;
|
|
12913
14052
|
}
|
|
12914
|
-
const { collect, stop } = createResourceUsageCollector(
|
|
14053
|
+
const { collect, stop } = createResourceUsageCollector(homedir7());
|
|
12915
14054
|
state.stopResourceUsageSampling = stop;
|
|
12916
14055
|
let consecutiveFailures = 0;
|
|
12917
14056
|
const tick = async () => {
|
|
@@ -12924,6 +14063,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12924
14063
|
message: `Resource usage collection: ${warning2}`
|
|
12925
14064
|
});
|
|
12926
14065
|
}
|
|
14066
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14067
|
+
...usage,
|
|
14068
|
+
nowMs: Date.now(),
|
|
14069
|
+
state: state.lowDiskWarningState
|
|
14070
|
+
});
|
|
14071
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14072
|
+
if (lowDiskDecision.warning !== null) {
|
|
14073
|
+
logActivity(state, {
|
|
14074
|
+
type: "info",
|
|
14075
|
+
level: "warn",
|
|
14076
|
+
message: lowDiskDecision.warning
|
|
14077
|
+
});
|
|
14078
|
+
}
|
|
12927
14079
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
12928
14080
|
if (result.ok) {
|
|
12929
14081
|
if (consecutiveFailures > 0) {
|
|
@@ -13136,7 +14288,7 @@ async function run(options) {
|
|
|
13136
14288
|
let fileSyncDirectories;
|
|
13137
14289
|
try {
|
|
13138
14290
|
logLevel = resolveLogLevel(options);
|
|
13139
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
14291
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir7());
|
|
13140
14292
|
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
13141
14293
|
throw new Error(
|
|
13142
14294
|
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
@@ -13170,6 +14322,7 @@ async function run(options) {
|
|
|
13170
14322
|
opencodeClient: null,
|
|
13171
14323
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13172
14324
|
sessionDbProvenanceAnomaly: false,
|
|
14325
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13173
14326
|
opencodeProcess: null,
|
|
13174
14327
|
stopOpenCodeLogTail: null,
|
|
13175
14328
|
litestreamProcess: null,
|
|
@@ -13187,10 +14340,12 @@ async function run(options) {
|
|
|
13187
14340
|
openaiUsageRearm: null,
|
|
13188
14341
|
resourceUsageTimer: null,
|
|
13189
14342
|
stopResourceUsageSampling: null,
|
|
14343
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13190
14344
|
credentialSync: null,
|
|
13191
14345
|
authHeader: ""
|
|
13192
14346
|
};
|
|
13193
14347
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14348
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13194
14349
|
if (options.credentialSyncMarker) {
|
|
13195
14350
|
state.credentialSync = createCredentialSync({
|
|
13196
14351
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13359,6 +14514,34 @@ async function run(options) {
|
|
|
13359
14514
|
} else {
|
|
13360
14515
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13361
14516
|
}
|
|
14517
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14518
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14519
|
+
log2(state, message, level);
|
|
14520
|
+
if (state.interactive && !state.json) {
|
|
14521
|
+
logActivity(state, { type: "info", level, message });
|
|
14522
|
+
}
|
|
14523
|
+
};
|
|
14524
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14525
|
+
if (vaultResolution.error) {
|
|
14526
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14527
|
+
logCredentialVaultResolution(
|
|
14528
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14529
|
+
"warn"
|
|
14530
|
+
);
|
|
14531
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14532
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14533
|
+
logCredentialVaultResolution(
|
|
14534
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14535
|
+
"info"
|
|
14536
|
+
);
|
|
14537
|
+
} else {
|
|
14538
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14539
|
+
logCredentialVaultResolution(
|
|
14540
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14541
|
+
"debug"
|
|
14542
|
+
);
|
|
14543
|
+
}
|
|
14544
|
+
}
|
|
13362
14545
|
if (options.restoreRunnerCredentials) {
|
|
13363
14546
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13364
14547
|
const credentialContext = {
|
|
@@ -13376,14 +14559,23 @@ async function run(options) {
|
|
|
13376
14559
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13377
14560
|
}
|
|
13378
14561
|
state.credentialSync?.arm();
|
|
13379
|
-
|
|
13380
|
-
|
|
13381
|
-
createOpenCodeActivityForwarder(() => ({
|
|
14562
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14563
|
+
() => ({
|
|
13382
14564
|
agentId: state.agentId,
|
|
13383
14565
|
authHeader: state.authHeader
|
|
13384
|
-
})
|
|
13385
|
-
|
|
14566
|
+
}),
|
|
14567
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14568
|
+
);
|
|
14569
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14570
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
14571
|
+
openCodeActivityForwarder
|
|
14572
|
+
);
|
|
14573
|
+
state.stopOpenCodeLogTail = () => {
|
|
14574
|
+
openCodeActivityForwarder.flush();
|
|
14575
|
+
openCodeLogTail.stop();
|
|
14576
|
+
};
|
|
13386
14577
|
let sessionDbVerifyFatal = false;
|
|
14578
|
+
let sessionDbVerifyFatalReason;
|
|
13387
14579
|
if (!options.restoreSessionDb) {
|
|
13388
14580
|
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
13389
14581
|
} else {
|
|
@@ -13416,12 +14608,13 @@ async function run(options) {
|
|
|
13416
14608
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13417
14609
|
});
|
|
13418
14610
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14611
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13419
14612
|
}
|
|
13420
14613
|
}
|
|
13421
14614
|
reportSessionDbRecovery(state);
|
|
13422
14615
|
if (sessionDbVerifyFatal) {
|
|
13423
14616
|
throw new Error(
|
|
13424
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14617
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13425
14618
|
);
|
|
13426
14619
|
}
|
|
13427
14620
|
applyRunnerOpenCodeConfig({
|
|
@@ -13499,7 +14692,7 @@ async function run(options) {
|
|
|
13499
14692
|
const provenance = checkSessionDbProvenance({
|
|
13500
14693
|
dbPath: sessionDbPath(),
|
|
13501
14694
|
currentVersion: state.opencodeVersion,
|
|
13502
|
-
homeDir:
|
|
14695
|
+
homeDir: homedir7(),
|
|
13503
14696
|
env: process.env
|
|
13504
14697
|
});
|
|
13505
14698
|
if (provenance.anomaly) {
|
|
@@ -13666,10 +14859,11 @@ async function run(options) {
|
|
|
13666
14859
|
getAuthHeader: () => state.authHeader,
|
|
13667
14860
|
conversationFilter: state.conversationFilter,
|
|
13668
14861
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14862
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13669
14863
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13670
14864
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13671
14865
|
fileSyncDirectories,
|
|
13672
|
-
homeDir:
|
|
14866
|
+
homeDir: homedir7(),
|
|
13673
14867
|
maxActiveSessions,
|
|
13674
14868
|
log: (entry) => (
|
|
13675
14869
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -13691,12 +14885,13 @@ async function run(options) {
|
|
|
13691
14885
|
openCodePassword: state.opencodePassword,
|
|
13692
14886
|
isRunning: () => state.running,
|
|
13693
14887
|
events: {
|
|
13694
|
-
onConnected: (agentId, isReconnect) => {
|
|
14888
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13695
14889
|
state.connected = true;
|
|
13696
14890
|
state.agentId = agentId;
|
|
14891
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13697
14892
|
logActivity(state, {
|
|
13698
14893
|
type: "info",
|
|
13699
|
-
message:
|
|
14894
|
+
message: connectionMessage
|
|
13700
14895
|
});
|
|
13701
14896
|
if (options.tunnelReadyFile) {
|
|
13702
14897
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13717,7 +14912,12 @@ async function run(options) {
|
|
|
13717
14912
|
version: state.opencodeVersion,
|
|
13718
14913
|
major: state.opencodeApiVersion,
|
|
13719
14914
|
connected: state.opencodeConnected
|
|
13720
|
-
})
|
|
14915
|
+
}),
|
|
14916
|
+
connection_generation: transition.generation,
|
|
14917
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
14918
|
+
reconnect_attempt: transition.attempt,
|
|
14919
|
+
downtime_ms: transition.downtimeMs
|
|
14920
|
+
} : {}
|
|
13721
14921
|
});
|
|
13722
14922
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13723
14923
|
if (state.interactive) displayStatus(state);
|
|
@@ -13739,13 +14939,18 @@ async function run(options) {
|
|
|
13739
14939
|
if (state.interactive) displayStatus(state);
|
|
13740
14940
|
});
|
|
13741
14941
|
},
|
|
13742
|
-
onDisconnected: (code, reason) => {
|
|
14942
|
+
onDisconnected: (code, reason, transition) => {
|
|
13743
14943
|
state.connected = false;
|
|
14944
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13744
14945
|
logActivity(state, {
|
|
13745
14946
|
type: "info",
|
|
13746
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
14947
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14948
|
+
});
|
|
14949
|
+
emitAgentDisconnected(state.agentId, {
|
|
14950
|
+
code,
|
|
14951
|
+
reason: safeReason2,
|
|
14952
|
+
connection_generation: transition.generation
|
|
13747
14953
|
});
|
|
13748
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13749
14954
|
if (state.interactive) displayStatus(state);
|
|
13750
14955
|
},
|
|
13751
14956
|
onError: (error2) => {
|
|
@@ -13845,7 +15050,7 @@ async function run(options) {
|
|
|
13845
15050
|
},
|
|
13846
15051
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13847
15052
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13848
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
15053
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13849
15054
|
firstDelayMs: firstReportDelayMs,
|
|
13850
15055
|
nextDelayMs: usageReportDelayMs,
|
|
13851
15056
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13888,7 +15093,7 @@ async function run(options) {
|
|
|
13888
15093
|
}
|
|
13889
15094
|
|
|
13890
15095
|
// src/index.ts
|
|
13891
|
-
var
|
|
15096
|
+
var version = getCliVersion();
|
|
13892
15097
|
var program = new Command();
|
|
13893
15098
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13894
15099
|
"--endpoint <url>",
|
|
@@ -13906,6 +15111,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
13906
15111
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
13907
15112
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
13908
15113
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15114
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
13909
15115
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
13910
15116
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
13911
15117
|
"-a, --agent [id]",
|