@evident-ai/cli 3.5.2-dev.4912967 → 3.5.2-dev.4f96188
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1770 -521
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1070,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1158
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1323
|
|
|
1160
1324
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.4f96188" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1326
|
function getCliVersion() {
|
|
1163
1327
|
return CLI_VERSION;
|
|
1164
1328
|
}
|
|
@@ -1172,6 +1336,23 @@ var authProvider = null;
|
|
|
1172
1336
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1337
|
authProvider = provider;
|
|
1174
1338
|
}
|
|
1339
|
+
var diagnosticsReporter = null;
|
|
1340
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1341
|
+
diagnosticsReporter = reporter;
|
|
1342
|
+
}
|
|
1343
|
+
function reportTelemetryDiagnostic(message) {
|
|
1344
|
+
if (!diagnosticsReporter) {
|
|
1345
|
+
console.error(message);
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
diagnosticsReporter(message);
|
|
1350
|
+
} catch (error2) {
|
|
1351
|
+
console.error(
|
|
1352
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1175
1356
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1357
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1358
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1418,9 @@ async function flushEvents() {
|
|
|
1237
1418
|
signal: controller.signal
|
|
1238
1419
|
});
|
|
1239
1420
|
if (!response.ok) {
|
|
1240
|
-
|
|
1421
|
+
reportTelemetryDiagnostic(
|
|
1422
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1423
|
+
);
|
|
1241
1424
|
}
|
|
1242
1425
|
} finally {
|
|
1243
1426
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1430,9 @@ async function flushEvents() {
|
|
|
1247
1430
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1431
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1432
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1433
|
+
reportTelemetryDiagnostic(
|
|
1434
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1435
|
+
);
|
|
1251
1436
|
lastFlushFailureLoggedAt = now;
|
|
1252
1437
|
suppressedFlushFailureCount = 0;
|
|
1253
1438
|
} else {
|
|
@@ -1298,8 +1483,316 @@ var EventTypes = {
|
|
|
1298
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1299
1484
|
};
|
|
1300
1485
|
|
|
1301
|
-
// src/lib/
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1302
1491
|
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
function errorMessage(error2) {
|
|
1522
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1523
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1524
|
+
}
|
|
1525
|
+
function errorMessages(error2) {
|
|
1526
|
+
const messages = [];
|
|
1527
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1528
|
+
let current = error2;
|
|
1529
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1530
|
+
visited.add(current);
|
|
1531
|
+
if (typeof current === "string") {
|
|
1532
|
+
messages.push(current);
|
|
1533
|
+
break;
|
|
1534
|
+
}
|
|
1535
|
+
if (current instanceof Error) {
|
|
1536
|
+
messages.push(current.name, current.message);
|
|
1537
|
+
current = current.cause;
|
|
1538
|
+
continue;
|
|
1539
|
+
}
|
|
1540
|
+
if (typeof current === "object") {
|
|
1541
|
+
const record = current;
|
|
1542
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1543
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1544
|
+
current = record.cause;
|
|
1545
|
+
continue;
|
|
1546
|
+
}
|
|
1547
|
+
break;
|
|
1548
|
+
}
|
|
1549
|
+
return messages;
|
|
1550
|
+
}
|
|
1551
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1552
|
+
return errorMessages(error2).some(
|
|
1553
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1554
|
+
message
|
|
1555
|
+
)
|
|
1556
|
+
);
|
|
1557
|
+
}
|
|
1558
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1559
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1560
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1561
|
+
function isCauseBearingText(text) {
|
|
1562
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1563
|
+
}
|
|
1564
|
+
function structuredLogFields(line) {
|
|
1565
|
+
const fields = [];
|
|
1566
|
+
for (let index = 0; index < line.length; index++) {
|
|
1567
|
+
if (line[index] === '"') {
|
|
1568
|
+
index++;
|
|
1569
|
+
while (index < line.length) {
|
|
1570
|
+
if (line[index] === "\\") {
|
|
1571
|
+
index++;
|
|
1572
|
+
} else if (line[index] === '"') {
|
|
1573
|
+
break;
|
|
1574
|
+
}
|
|
1575
|
+
index++;
|
|
1576
|
+
}
|
|
1577
|
+
continue;
|
|
1578
|
+
}
|
|
1579
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1580
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1581
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1582
|
+
const valueStart = index + key.length;
|
|
1583
|
+
if (line[valueStart] !== '"') {
|
|
1584
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1585
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1586
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
let valueEnd = valueStart + 1;
|
|
1590
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1591
|
+
if (line[valueEnd] === "\\") {
|
|
1592
|
+
valueEnd++;
|
|
1593
|
+
} else if (line[valueEnd] === '"') {
|
|
1594
|
+
break;
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1598
|
+
index = valueEnd;
|
|
1599
|
+
}
|
|
1600
|
+
return fields;
|
|
1601
|
+
}
|
|
1602
|
+
function structuredCauseMessages(text) {
|
|
1603
|
+
const fields = structuredLogFields(text);
|
|
1604
|
+
if (fields.length > 0) return fields;
|
|
1605
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1606
|
+
}
|
|
1607
|
+
function causeTokenFromText(text) {
|
|
1608
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1609
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1610
|
+
const match = pattern.exec(text);
|
|
1611
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1612
|
+
return match[1];
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
return null;
|
|
1616
|
+
}
|
|
1617
|
+
function sessionStoreCauseToken(value) {
|
|
1618
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1619
|
+
const token = causeTokenFromText(message);
|
|
1620
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1621
|
+
}
|
|
1622
|
+
return null;
|
|
1623
|
+
}
|
|
1624
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1625
|
+
try {
|
|
1626
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1627
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1628
|
+
if (availableBytes < requiredBytes) {
|
|
1629
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1630
|
+
}
|
|
1631
|
+
return null;
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
function readLogicalBytes(db) {
|
|
1637
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1638
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1639
|
+
return pageCount * pageSize;
|
|
1640
|
+
}
|
|
1641
|
+
function readFreelistCount(db) {
|
|
1642
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1643
|
+
}
|
|
1644
|
+
function readCheckpointResult(db) {
|
|
1645
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1646
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1647
|
+
}
|
|
1648
|
+
async function probeReclaimAvailability(input) {
|
|
1649
|
+
const { dbPath, requiredBytes } = input;
|
|
1650
|
+
let sqlite;
|
|
1651
|
+
try {
|
|
1652
|
+
sqlite = await import("node:sqlite");
|
|
1653
|
+
} catch (err) {
|
|
1654
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1655
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1656
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1657
|
+
}
|
|
1658
|
+
let autoVacuum = null;
|
|
1659
|
+
try {
|
|
1660
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1661
|
+
try {
|
|
1662
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1663
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1664
|
+
} finally {
|
|
1665
|
+
db.close();
|
|
1666
|
+
}
|
|
1667
|
+
} catch (err) {
|
|
1668
|
+
console.warn(
|
|
1669
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
if (autoVacuum !== 0) return null;
|
|
1673
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1674
|
+
}
|
|
1675
|
+
async function reclaimSessionDbSpace(input) {
|
|
1676
|
+
const {
|
|
1677
|
+
dbPath,
|
|
1678
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1679
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1680
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1681
|
+
now = () => performance.now(),
|
|
1682
|
+
allowFullVacuum = true
|
|
1683
|
+
} = input;
|
|
1684
|
+
let sqlite;
|
|
1685
|
+
try {
|
|
1686
|
+
sqlite = await import("node:sqlite");
|
|
1687
|
+
} catch (err) {
|
|
1688
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1689
|
+
console.warn(
|
|
1690
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1691
|
+
);
|
|
1692
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1693
|
+
}
|
|
1694
|
+
const { DatabaseSync } = sqlite;
|
|
1695
|
+
let db;
|
|
1696
|
+
try {
|
|
1697
|
+
db = new DatabaseSync(dbPath);
|
|
1698
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1699
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1700
|
+
if (autoVacuum === 0) {
|
|
1701
|
+
if (!allowFullVacuum) {
|
|
1702
|
+
console.warn(
|
|
1703
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1704
|
+
);
|
|
1705
|
+
return {
|
|
1706
|
+
ok: false,
|
|
1707
|
+
skipped: "full-vacuum-blocked",
|
|
1708
|
+
detail: "a session turn is live"
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1712
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1713
|
+
if (skipReason !== null) {
|
|
1714
|
+
console.warn(
|
|
1715
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1716
|
+
);
|
|
1717
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1718
|
+
}
|
|
1719
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1720
|
+
const freelistBefore = readFreelistCount(db);
|
|
1721
|
+
const startedAt = now();
|
|
1722
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1723
|
+
db.exec("VACUUM");
|
|
1724
|
+
const afterBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistAfter = readFreelistCount(db);
|
|
1726
|
+
const checkpoint = readCheckpointResult(db);
|
|
1727
|
+
return {
|
|
1728
|
+
ok: true,
|
|
1729
|
+
mode: "convert",
|
|
1730
|
+
beforeBytes,
|
|
1731
|
+
afterBytes,
|
|
1732
|
+
freelistBefore,
|
|
1733
|
+
freelistAfter,
|
|
1734
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1735
|
+
stoppedBy: "freelist-drained",
|
|
1736
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1737
|
+
checkpoint
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
if (autoVacuum === 2) {
|
|
1741
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1742
|
+
const freelistBefore = readFreelistCount(db);
|
|
1743
|
+
const startedAt = now();
|
|
1744
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1745
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1746
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1747
|
+
let pagesAttempted = 0;
|
|
1748
|
+
let freelistAfter = freelistBefore;
|
|
1749
|
+
let stoppedBy = "freelist-drained";
|
|
1750
|
+
while (freelistAfter > 0) {
|
|
1751
|
+
if (pagesAttempted >= pageLimit) {
|
|
1752
|
+
stoppedBy = "page-ceiling";
|
|
1753
|
+
break;
|
|
1754
|
+
}
|
|
1755
|
+
if (now() - startedAt >= budget) {
|
|
1756
|
+
stoppedBy = "time-budget";
|
|
1757
|
+
break;
|
|
1758
|
+
}
|
|
1759
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1760
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1761
|
+
pagesAttempted += pages;
|
|
1762
|
+
freelistAfter = readFreelistCount(db);
|
|
1763
|
+
}
|
|
1764
|
+
const afterBytes = readLogicalBytes(db);
|
|
1765
|
+
const checkpoint = readCheckpointResult(db);
|
|
1766
|
+
return {
|
|
1767
|
+
ok: true,
|
|
1768
|
+
mode: "incremental",
|
|
1769
|
+
beforeBytes,
|
|
1770
|
+
afterBytes,
|
|
1771
|
+
freelistBefore,
|
|
1772
|
+
freelistAfter,
|
|
1773
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1774
|
+
stoppedBy,
|
|
1775
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1776
|
+
checkpoint
|
|
1777
|
+
};
|
|
1778
|
+
}
|
|
1779
|
+
console.warn(
|
|
1780
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1781
|
+
);
|
|
1782
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1783
|
+
} catch (err) {
|
|
1784
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1785
|
+
return {
|
|
1786
|
+
ok: false,
|
|
1787
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1788
|
+
detail: errorMessage(err)
|
|
1789
|
+
};
|
|
1790
|
+
} finally {
|
|
1791
|
+
db?.close();
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1303
1796
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1304
1797
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1305
1798
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1316,26 +1809,91 @@ function parseOpenCodeLogLine(line) {
|
|
|
1316
1809
|
if (!levelMatch) return null;
|
|
1317
1810
|
const level = levelMatch[1].toUpperCase();
|
|
1318
1811
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1319
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1320
|
-
|
|
1812
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1813
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1814
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1815
|
+
const parsed = {
|
|
1816
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1817
|
+
sessionID: sessionMatch?.[1],
|
|
1818
|
+
messageID: messageIDMatch?.[1]
|
|
1819
|
+
};
|
|
1820
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1821
|
+
if (message !== void 0) parsed.message = message;
|
|
1822
|
+
return parsed;
|
|
1321
1823
|
}
|
|
1322
1824
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1323
|
-
|
|
1825
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1826
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1324
1827
|
let buffer = Buffer.alloc(0);
|
|
1325
|
-
const
|
|
1326
|
-
|
|
1327
|
-
|
|
1828
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1829
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1830
|
+
const metadata = {
|
|
1831
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1832
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1833
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1834
|
+
};
|
|
1328
1835
|
forwardRunnerActivity(
|
|
1329
1836
|
{
|
|
1330
1837
|
level: parsed.level,
|
|
1331
1838
|
error: line,
|
|
1332
|
-
metadata:
|
|
1839
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1333
1840
|
source: "opencode"
|
|
1334
1841
|
},
|
|
1335
1842
|
getContext()
|
|
1336
1843
|
);
|
|
1337
1844
|
};
|
|
1338
|
-
|
|
1845
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1846
|
+
for (const [key, state] of repeatStates) {
|
|
1847
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1848
|
+
continue;
|
|
1849
|
+
}
|
|
1850
|
+
if (state.repeatsSuppressed > 0) {
|
|
1851
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1852
|
+
}
|
|
1853
|
+
repeatStates.delete(key);
|
|
1854
|
+
}
|
|
1855
|
+
};
|
|
1856
|
+
const flushLine = (line) => {
|
|
1857
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1858
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1859
|
+
const cause = sessionStoreCauseToken(line);
|
|
1860
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1861
|
+
try {
|
|
1862
|
+
onSessionStoreWriteFailure({
|
|
1863
|
+
isDiskFailure,
|
|
1864
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1865
|
+
cause
|
|
1866
|
+
});
|
|
1867
|
+
} catch (err) {
|
|
1868
|
+
reportTelemetryDiagnostic(
|
|
1869
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
if (!parsed) return;
|
|
1874
|
+
const now = Date.now();
|
|
1875
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1876
|
+
expireRepeatStates(now, key);
|
|
1877
|
+
if (key === void 0) {
|
|
1878
|
+
forwardLine(line, parsed);
|
|
1879
|
+
return;
|
|
1880
|
+
}
|
|
1881
|
+
const previous = repeatStates.get(key);
|
|
1882
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1883
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1884
|
+
repeatStates.set(key, {
|
|
1885
|
+
line,
|
|
1886
|
+
parsed,
|
|
1887
|
+
lastForwardedAt: now,
|
|
1888
|
+
repeatsSuppressed: 0
|
|
1889
|
+
});
|
|
1890
|
+
return;
|
|
1891
|
+
}
|
|
1892
|
+
previous.line = line;
|
|
1893
|
+
previous.parsed = parsed;
|
|
1894
|
+
previous.repeatsSuppressed++;
|
|
1895
|
+
};
|
|
1896
|
+
const forward = ((chunk) => {
|
|
1339
1897
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1340
1898
|
let newlineIndex;
|
|
1341
1899
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1346,52 +1904,51 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1346
1904
|
flushLine(buffer.toString("utf-8"));
|
|
1347
1905
|
buffer = Buffer.alloc(0);
|
|
1348
1906
|
}
|
|
1907
|
+
});
|
|
1908
|
+
forward.flush = () => {
|
|
1909
|
+
for (const state of repeatStates.values()) {
|
|
1910
|
+
if (state.repeatsSuppressed > 0) {
|
|
1911
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
repeatStates.clear();
|
|
1349
1915
|
};
|
|
1916
|
+
return forward;
|
|
1350
1917
|
}
|
|
1351
1918
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1352
1919
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1353
1920
|
var MAX_METADATA_ENTRIES = 20;
|
|
1354
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1355
|
-
function redactHomeDirectory(message) {
|
|
1356
|
-
const home = homedir2();
|
|
1357
|
-
if (!home || home === "/") {
|
|
1358
|
-
return message;
|
|
1359
|
-
}
|
|
1360
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1361
|
-
let result = "";
|
|
1362
|
-
let searchFrom = 0;
|
|
1363
|
-
let homeIndex;
|
|
1364
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1365
|
-
const homeEnd = homeIndex + home.length;
|
|
1366
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1367
|
-
const nextCharacter = message[homeEnd];
|
|
1368
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1369
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1370
|
-
result += atPathBoundary ? "~" : home;
|
|
1371
|
-
searchFrom = homeEnd;
|
|
1372
|
-
}
|
|
1373
|
-
return result + message.slice(searchFrom);
|
|
1374
|
-
}
|
|
1375
|
-
function redact(message) {
|
|
1376
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1377
|
-
return redactHomeDirectory(redacted);
|
|
1378
|
-
}
|
|
1379
1921
|
function truncate(message) {
|
|
1380
1922
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1381
|
-
|
|
1923
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1924
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1925
|
+
let previousDropped = -1;
|
|
1926
|
+
while (dropped !== previousDropped) {
|
|
1927
|
+
previousDropped = dropped;
|
|
1928
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1929
|
+
dropped = nextDropped;
|
|
1930
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1931
|
+
}
|
|
1932
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1933
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1934
|
+
const tailLength = remaining - headLength;
|
|
1935
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1382
1936
|
}
|
|
1383
1937
|
function sanitiseMetadata(metadata) {
|
|
1384
1938
|
if (!metadata) return {};
|
|
1385
1939
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1386
1940
|
if (Object.keys(metadata).length > entries.length) {
|
|
1387
|
-
|
|
1388
|
-
`
|
|
1941
|
+
reportTelemetryDiagnostic(
|
|
1942
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1389
1943
|
);
|
|
1390
1944
|
}
|
|
1391
1945
|
const sanitised = [];
|
|
1392
1946
|
for (const [key, value] of entries) {
|
|
1393
1947
|
if (typeof value === "string") {
|
|
1394
|
-
sanitised.push([
|
|
1948
|
+
sanitised.push([
|
|
1949
|
+
key,
|
|
1950
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1951
|
+
]);
|
|
1395
1952
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1396
1953
|
sanitised.push([key, value]);
|
|
1397
1954
|
}
|
|
@@ -1409,8 +1966,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1409
1966
|
}
|
|
1410
1967
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1411
1968
|
if (window.windowDroppedCount > 0) {
|
|
1412
|
-
|
|
1413
|
-
`
|
|
1969
|
+
reportTelemetryDiagnostic(
|
|
1970
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1414
1971
|
);
|
|
1415
1972
|
}
|
|
1416
1973
|
window.windowStartedAt = now;
|
|
@@ -1420,8 +1977,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1420
1977
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1421
1978
|
window.windowDroppedCount++;
|
|
1422
1979
|
if (window.windowDroppedCount === 1) {
|
|
1423
|
-
|
|
1424
|
-
`
|
|
1980
|
+
reportTelemetryDiagnostic(
|
|
1981
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1425
1982
|
);
|
|
1426
1983
|
}
|
|
1427
1984
|
return false;
|
|
@@ -1436,7 +1993,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1436
1993
|
if (!context.agentId || !context.authHeader) return;
|
|
1437
1994
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1438
1995
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1439
|
-
const message = truncate(
|
|
1996
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1440
1997
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1441
1998
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1442
1999
|
message,
|
|
@@ -1444,8 +2001,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1444
2001
|
agentId: context.agentId
|
|
1445
2002
|
});
|
|
1446
2003
|
} catch (err) {
|
|
1447
|
-
|
|
1448
|
-
`
|
|
2004
|
+
reportTelemetryDiagnostic(
|
|
2005
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1449
2006
|
);
|
|
1450
2007
|
}
|
|
1451
2008
|
}
|
|
@@ -1585,12 +2142,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1585
2142
|
metadata: withoutContractFields(record),
|
|
1586
2143
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1587
2144
|
};
|
|
1588
|
-
case "session_db_boot_refused":
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
2145
|
+
case "session_db_boot_refused": {
|
|
2146
|
+
const message = (() => {
|
|
2147
|
+
switch (record.reason) {
|
|
2148
|
+
case "local_discard_failed":
|
|
2149
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2150
|
+
case "replica_separation_unproven":
|
|
2151
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2152
|
+
case "verification_timeout_before_separation":
|
|
2153
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2154
|
+
case "verification_timeout_during_separation":
|
|
2155
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2156
|
+
default:
|
|
2157
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2158
|
+
}
|
|
2159
|
+
})();
|
|
2160
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2161
|
+
}
|
|
1594
2162
|
case "schema_provenance_mismatch":
|
|
1595
2163
|
return {
|
|
1596
2164
|
level,
|
|
@@ -1735,8 +2303,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1735
2303
|
|
|
1736
2304
|
// src/lib/http-timeout.ts
|
|
1737
2305
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2306
|
+
var RequestTimeoutError = class extends Error {
|
|
2307
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2308
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2309
|
+
this.operation = operation;
|
|
2310
|
+
this.timeoutMs = timeoutMs;
|
|
2311
|
+
this.elapsedMs = elapsedMs;
|
|
2312
|
+
this.name = "TimeoutError";
|
|
2313
|
+
}
|
|
2314
|
+
};
|
|
2315
|
+
function describeRequestOperation(input, init) {
|
|
2316
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2317
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2318
|
+
let path;
|
|
2319
|
+
try {
|
|
2320
|
+
path = new URL(rawUrl).pathname;
|
|
2321
|
+
} catch (error2) {
|
|
2322
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2323
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2324
|
+
}
|
|
2325
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2326
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2327
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2328
|
+
return `${method} ${maskedPath}`;
|
|
2329
|
+
}
|
|
1738
2330
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1739
|
-
return ((input, init) =>
|
|
2331
|
+
return (async (input, init) => {
|
|
2332
|
+
const startedAt = Date.now();
|
|
2333
|
+
try {
|
|
2334
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2335
|
+
} catch (error2) {
|
|
2336
|
+
if (error2?.name === "TimeoutError") {
|
|
2337
|
+
throw new RequestTimeoutError(
|
|
2338
|
+
describeRequestOperation(input, init),
|
|
2339
|
+
timeoutMs,
|
|
2340
|
+
Date.now() - startedAt
|
|
2341
|
+
);
|
|
2342
|
+
}
|
|
2343
|
+
throw error2;
|
|
2344
|
+
}
|
|
2345
|
+
});
|
|
1740
2346
|
}
|
|
1741
2347
|
|
|
1742
2348
|
// src/lib/opencode/client.ts
|
|
@@ -1802,7 +2408,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1802
2408
|
return null;
|
|
1803
2409
|
}
|
|
1804
2410
|
}
|
|
1805
|
-
function
|
|
2411
|
+
function roleOf2(m) {
|
|
1806
2412
|
if (!m || typeof m !== "object") return void 0;
|
|
1807
2413
|
if (typeof m.role === "string") return m.role;
|
|
1808
2414
|
const infoRole = m.info?.role;
|
|
@@ -1816,13 +2422,13 @@ function createdOf(m) {
|
|
|
1816
2422
|
if (!m || typeof m !== "object") return void 0;
|
|
1817
2423
|
return m.info?.time?.created ?? m.time?.created;
|
|
1818
2424
|
}
|
|
1819
|
-
function
|
|
2425
|
+
function idOf2(m) {
|
|
1820
2426
|
if (!m || typeof m !== "object") return void 0;
|
|
1821
2427
|
if (typeof m.id === "string") return m.id;
|
|
1822
2428
|
const infoId = m.info?.id;
|
|
1823
2429
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1824
2430
|
}
|
|
1825
|
-
function
|
|
2431
|
+
function parentIdOf2(m) {
|
|
1826
2432
|
if (!m || typeof m !== "object") return void 0;
|
|
1827
2433
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1828
2434
|
const infoParent = m.info?.parentID;
|
|
@@ -1838,8 +2444,12 @@ function errorOf(m) {
|
|
|
1838
2444
|
if (!m || typeof m !== "object") return void 0;
|
|
1839
2445
|
return m.info?.error ?? m.error;
|
|
1840
2446
|
}
|
|
2447
|
+
function isTerminalStopAnswer(m) {
|
|
2448
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2449
|
+
}
|
|
1841
2450
|
function isAssistantInFlight(m) {
|
|
1842
2451
|
if (completedOf(m) == null) return true;
|
|
2452
|
+
if (errorOf(m) != null) return false;
|
|
1843
2453
|
return finishOf(m) === "tool-calls";
|
|
1844
2454
|
}
|
|
1845
2455
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -1887,9 +2497,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1887
2497
|
function isSessionActivelyGenerating(messages) {
|
|
1888
2498
|
if (!messages || messages.length === 0) return false;
|
|
1889
2499
|
const last = messages[messages.length - 1];
|
|
1890
|
-
if (
|
|
2500
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1891
2501
|
return completedOf(last) == null;
|
|
1892
2502
|
}
|
|
2503
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1893
2504
|
function sessionLastActivityMs(session) {
|
|
1894
2505
|
const candidates = [
|
|
1895
2506
|
session.time?.updated,
|
|
@@ -1904,30 +2515,60 @@ function sessionLastActivityMs(session) {
|
|
|
1904
2515
|
}
|
|
1905
2516
|
return null;
|
|
1906
2517
|
}
|
|
1907
|
-
|
|
1908
|
-
|
|
2518
|
+
function sessionListPath(options) {
|
|
2519
|
+
const params = new URLSearchParams();
|
|
2520
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2521
|
+
const query = params.toString();
|
|
2522
|
+
return query ? `/session?${query}` : "/session";
|
|
2523
|
+
}
|
|
2524
|
+
async function listSessionPage(port, client, options) {
|
|
2525
|
+
const path = sessionListPath(options);
|
|
1909
2526
|
try {
|
|
1910
|
-
const res = await requestWithClient(port, client,
|
|
2527
|
+
const res = await requestWithClient(port, client, path);
|
|
1911
2528
|
if (!res.ok) return null;
|
|
1912
2529
|
const body = await res.json();
|
|
1913
2530
|
return Array.isArray(body) ? body : null;
|
|
1914
2531
|
} catch (error2) {
|
|
1915
2532
|
console.error(
|
|
1916
|
-
`[
|
|
2533
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1917
2534
|
);
|
|
1918
2535
|
return null;
|
|
1919
2536
|
}
|
|
1920
2537
|
}
|
|
2538
|
+
async function listSessions(port, client, options) {
|
|
2539
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2540
|
+
return listSessionPage(port, client, options);
|
|
2541
|
+
}
|
|
2542
|
+
async function listAllSessions(port, client) {
|
|
2543
|
+
if (client?.version === "v2") {
|
|
2544
|
+
const sessions2 = await listV2Sessions(client);
|
|
2545
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2546
|
+
}
|
|
2547
|
+
const sessions = await listSessionPage(port, client, {
|
|
2548
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2549
|
+
});
|
|
2550
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2551
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2552
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2553
|
+
}
|
|
2554
|
+
return { sessions };
|
|
2555
|
+
}
|
|
1921
2556
|
async function deleteSession(port, id, client) {
|
|
1922
2557
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1923
2558
|
try {
|
|
1924
|
-
const res = await requestWithClient(
|
|
1925
|
-
|
|
2559
|
+
const res = await requestWithClient(
|
|
2560
|
+
port,
|
|
2561
|
+
client,
|
|
2562
|
+
`/session/${id}`,
|
|
2563
|
+
{ method: "DELETE" },
|
|
2564
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2565
|
+
);
|
|
2566
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1926
2567
|
} catch (error2) {
|
|
1927
2568
|
console.error(
|
|
1928
2569
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1929
2570
|
);
|
|
1930
|
-
return false;
|
|
2571
|
+
return { ok: false, status: "threw" };
|
|
1931
2572
|
}
|
|
1932
2573
|
}
|
|
1933
2574
|
async function sessionExists(port, id, client) {
|
|
@@ -2165,11 +2806,31 @@ function messageText(m) {
|
|
|
2165
2806
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2166
2807
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2167
2808
|
}
|
|
2168
|
-
|
|
2809
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2810
|
+
if (!messages || messages.length === 0) return null;
|
|
2811
|
+
let best = null;
|
|
2812
|
+
for (const message of messages) {
|
|
2813
|
+
if (roleOf2(message) !== "user") continue;
|
|
2814
|
+
const id = idOf2(message);
|
|
2815
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2816
|
+
if (messageText(message) !== content) continue;
|
|
2817
|
+
const created = createdOf(message) ?? 0;
|
|
2818
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2819
|
+
}
|
|
2820
|
+
return best?.id ?? null;
|
|
2821
|
+
}
|
|
2822
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2169
2823
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2170
2824
|
const knownUserIds = new Set(
|
|
2171
|
-
(before ?? []).filter((m) =>
|
|
2825
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2172
2826
|
);
|
|
2827
|
+
if (callbacks) {
|
|
2828
|
+
try {
|
|
2829
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2830
|
+
} catch (error2) {
|
|
2831
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2173
2834
|
const parts = [{ type: "text", text: content }];
|
|
2174
2835
|
let pendingOutcomes = null;
|
|
2175
2836
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2208,22 +2869,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2208
2869
|
const READ_BACK_DELAY_MS = 150;
|
|
2209
2870
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2210
2871
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
const id = idOf(m);
|
|
2216
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2217
|
-
if (messageText(m) !== content) continue;
|
|
2218
|
-
const created = createdOf(m) ?? 0;
|
|
2219
|
-
if (best === null || created > best.created) {
|
|
2220
|
-
best = { id, created };
|
|
2221
|
-
}
|
|
2222
|
-
}
|
|
2223
|
-
if (best) {
|
|
2224
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2225
|
-
return best.id;
|
|
2226
|
-
}
|
|
2872
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2873
|
+
if (dispatchedUserMessageId !== null) {
|
|
2874
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2875
|
+
return dispatchedUserMessageId;
|
|
2227
2876
|
}
|
|
2228
2877
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2229
2878
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2234,57 +2883,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2234
2883
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2235
2884
|
if (!messages || messages.length === 0) return null;
|
|
2236
2885
|
const byParent = messages.find(
|
|
2237
|
-
(m) =>
|
|
2886
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2238
2887
|
);
|
|
2239
2888
|
if (byParent) return byParent;
|
|
2240
|
-
const userIndex = messages.findIndex((m) =>
|
|
2889
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2241
2890
|
if (userIndex === -1) return null;
|
|
2242
2891
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2243
|
-
if (
|
|
2892
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2244
2893
|
}
|
|
2245
2894
|
return null;
|
|
2246
2895
|
}
|
|
2247
2896
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2248
2897
|
if (!messages || messages.length === 0) return null;
|
|
2249
2898
|
let lastCorrelated = null;
|
|
2250
|
-
let lastNonErrored = null;
|
|
2251
2899
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2252
2900
|
const m = messages[i];
|
|
2253
|
-
if (
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2901
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2902
|
+
lastCorrelated = m;
|
|
2903
|
+
break;
|
|
2904
|
+
}
|
|
2905
|
+
if (lastCorrelated) {
|
|
2906
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2907
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2908
|
+
const m = messages[i];
|
|
2909
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2910
|
+
return m;
|
|
2911
|
+
}
|
|
2258
2912
|
}
|
|
2913
|
+
return lastCorrelated;
|
|
2259
2914
|
}
|
|
2260
|
-
|
|
2261
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2915
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2262
2916
|
if (userIndex === -1) return null;
|
|
2263
2917
|
let last = null;
|
|
2264
|
-
let
|
|
2918
|
+
let lastAssistantIndex = -1;
|
|
2265
2919
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2266
|
-
const role =
|
|
2920
|
+
const role = roleOf2(messages[i]);
|
|
2267
2921
|
if (role === "user") break;
|
|
2268
2922
|
if (role === "assistant") {
|
|
2269
2923
|
last = messages[i];
|
|
2270
|
-
|
|
2924
|
+
lastAssistantIndex = i;
|
|
2271
2925
|
}
|
|
2272
2926
|
}
|
|
2273
|
-
|
|
2927
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2928
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2929
|
+
const m = messages[i];
|
|
2930
|
+
if (roleOf2(m) === "user") break;
|
|
2931
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2932
|
+
}
|
|
2933
|
+
return last;
|
|
2274
2934
|
}
|
|
2275
2935
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2276
2936
|
if (!messages || messages.length === 0) return [];
|
|
2277
2937
|
const byParent = messages.filter(
|
|
2278
|
-
(message) =>
|
|
2938
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2279
2939
|
);
|
|
2280
2940
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2281
2941
|
if (assistants.length === 0) {
|
|
2282
|
-
const userIndex = messages.findIndex((message) =>
|
|
2942
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2283
2943
|
if (userIndex === -1) return [];
|
|
2284
2944
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2285
2945
|
const message = messages[i];
|
|
2286
|
-
if (
|
|
2287
|
-
if (
|
|
2946
|
+
if (roleOf2(message) === "user") break;
|
|
2947
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2288
2948
|
}
|
|
2289
2949
|
}
|
|
2290
2950
|
const refs = [];
|
|
@@ -2324,7 +2984,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2324
2984
|
if (!messages || messages.length === 0) return [];
|
|
2325
2985
|
const calls = [];
|
|
2326
2986
|
for (const message of messages) {
|
|
2327
|
-
if (
|
|
2987
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2328
2988
|
for (const part of message.parts ?? []) {
|
|
2329
2989
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2330
2990
|
continue;
|
|
@@ -2354,7 +3014,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2354
3014
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2355
3015
|
const unattributed = [];
|
|
2356
3016
|
for (const message of messages ?? []) {
|
|
2357
|
-
if (
|
|
3017
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2358
3018
|
const created = finiteNumber(createdOf(message));
|
|
2359
3019
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2360
3020
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2376,8 +3036,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2376
3036
|
}
|
|
2377
3037
|
function sumAssistantUsage(messages) {
|
|
2378
3038
|
if (!messages || messages.length === 0) return null;
|
|
2379
|
-
const
|
|
2380
|
-
const
|
|
3039
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3040
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3041
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2381
3042
|
let sawAnyUsage = false;
|
|
2382
3043
|
let inputSum = 0;
|
|
2383
3044
|
let outputSum = 0;
|
|
@@ -2431,13 +3092,11 @@ function sumAssistantUsage(messages) {
|
|
|
2431
3092
|
function messageUsage(messages, userMessageId) {
|
|
2432
3093
|
if (!messages || messages.length === 0) return null;
|
|
2433
3094
|
const byParentAll = messages.filter(
|
|
2434
|
-
(m) =>
|
|
3095
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2435
3096
|
);
|
|
2436
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2437
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2438
3097
|
let correlated;
|
|
2439
|
-
if (
|
|
2440
|
-
correlated =
|
|
3098
|
+
if (byParentAll.length > 0) {
|
|
3099
|
+
correlated = byParentAll;
|
|
2441
3100
|
} else {
|
|
2442
3101
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2443
3102
|
correlated = reply ? [reply] : [];
|
|
@@ -2447,7 +3106,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2447
3106
|
}
|
|
2448
3107
|
function messageRunState(messages, userMessageId) {
|
|
2449
3108
|
if (!messages || messages.length === 0) return "unknown";
|
|
2450
|
-
const hasUser = messages.some((m) =>
|
|
3109
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2451
3110
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2452
3111
|
if (!hasUser) {
|
|
2453
3112
|
if (!reply) return "unknown";
|
|
@@ -2483,14 +3142,38 @@ function messageError(messages, userMessageId) {
|
|
|
2483
3142
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2484
3143
|
const error2 = errorOf(reply);
|
|
2485
3144
|
if (error2 == null) return null;
|
|
2486
|
-
|
|
2487
|
-
if (typeof error2 === "
|
|
3145
|
+
let rendered;
|
|
3146
|
+
if (typeof error2 === "string") {
|
|
3147
|
+
rendered = error2;
|
|
3148
|
+
} else if (typeof error2 === "object") {
|
|
2488
3149
|
const e = error2;
|
|
2489
3150
|
const dataMessage = e.data?.message;
|
|
2490
|
-
if (typeof dataMessage === "string")
|
|
2491
|
-
|
|
3151
|
+
if (typeof dataMessage === "string") {
|
|
3152
|
+
rendered = dataMessage;
|
|
3153
|
+
} else if (typeof e.message === "string") {
|
|
3154
|
+
rendered = e.message;
|
|
3155
|
+
} else {
|
|
3156
|
+
rendered = "The agent run failed.";
|
|
3157
|
+
}
|
|
3158
|
+
} else {
|
|
3159
|
+
rendered = "The agent run failed.";
|
|
2492
3160
|
}
|
|
2493
|
-
|
|
3161
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3162
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3163
|
+
) : void 0;
|
|
3164
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3165
|
+
}
|
|
3166
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3167
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3168
|
+
if (!reply) return null;
|
|
3169
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3170
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3171
|
+
return {
|
|
3172
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3173
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3174
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3175
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3176
|
+
};
|
|
2494
3177
|
}
|
|
2495
3178
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2496
3179
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2539,7 +3222,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2539
3222
|
if (!messages || messages.length === 0) return null;
|
|
2540
3223
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2541
3224
|
const message = messages[i];
|
|
2542
|
-
if (
|
|
3225
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2543
3226
|
const created = createdOf(message);
|
|
2544
3227
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2545
3228
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2571,25 +3254,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2571
3254
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2572
3255
|
if (!messages || messages.length === 0) return false;
|
|
2573
3256
|
return messages.some(
|
|
2574
|
-
(m) =>
|
|
3257
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2575
3258
|
);
|
|
2576
3259
|
}
|
|
2577
3260
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2578
3261
|
if (!messages || messages.length === 0) return false;
|
|
2579
|
-
const userIndex = messages.findIndex((message) =>
|
|
3262
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2580
3263
|
if (userIndex === -1) return false;
|
|
2581
3264
|
let hasLaterUser = false;
|
|
2582
3265
|
let hasStartedLaterUser = false;
|
|
2583
3266
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2584
3267
|
const message = messages[i];
|
|
2585
|
-
if (
|
|
3268
|
+
if (roleOf2(message) !== "user") continue;
|
|
2586
3269
|
hasLaterUser = true;
|
|
2587
|
-
const laterUserMessageId =
|
|
3270
|
+
const laterUserMessageId = idOf2(message);
|
|
2588
3271
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2589
3272
|
return false;
|
|
2590
3273
|
}
|
|
2591
3274
|
if (messages.some(
|
|
2592
|
-
(candidate) =>
|
|
3275
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2593
3276
|
)) {
|
|
2594
3277
|
hasStartedLaterUser = true;
|
|
2595
3278
|
}
|
|
@@ -3060,13 +3743,17 @@ async function listV2Sessions(client) {
|
|
|
3060
3743
|
}
|
|
3061
3744
|
async function deleteV2Session(client, sessionId) {
|
|
3062
3745
|
try {
|
|
3063
|
-
await client.request(
|
|
3064
|
-
|
|
3746
|
+
const response = await client.request(
|
|
3747
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3748
|
+
{ method: "DELETE" },
|
|
3749
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3750
|
+
);
|
|
3751
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3065
3752
|
} catch (error2) {
|
|
3066
3753
|
console.error(
|
|
3067
3754
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3068
3755
|
);
|
|
3069
|
-
return false;
|
|
3756
|
+
return { ok: false, status: "threw" };
|
|
3070
3757
|
}
|
|
3071
3758
|
}
|
|
3072
3759
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3397,16 +4084,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3397
4084
|
].some((name) => headers.has(name));
|
|
3398
4085
|
}
|
|
3399
4086
|
async function getOpenAiUsage(port, client) {
|
|
4087
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3400
4088
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3401
4089
|
if (!credentials2) {
|
|
3402
4090
|
throw new OpenAiUsageError(
|
|
3403
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
4091
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3404
4092
|
"no_credentials"
|
|
3405
4093
|
);
|
|
3406
4094
|
}
|
|
3407
4095
|
if (credentials2.expiresAt < Date.now()) {
|
|
3408
4096
|
throw new OpenAiUsageError(
|
|
3409
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
4097
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3410
4098
|
"credentials_expired"
|
|
3411
4099
|
);
|
|
3412
4100
|
}
|
|
@@ -3687,9 +4375,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3687
4375
|
|
|
3688
4376
|
// src/lib/opencode/session-db-boot.ts
|
|
3689
4377
|
import { spawn as spawn2 } from "node:child_process";
|
|
3690
|
-
import { mkdirSync, statSync as
|
|
4378
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3691
4379
|
import { homedir as homedir4 } from "node:os";
|
|
3692
|
-
import { dirname as
|
|
4380
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3693
4381
|
|
|
3694
4382
|
// src/lib/runner-synchroniser.ts
|
|
3695
4383
|
import { spawn } from "node:child_process";
|
|
@@ -3769,9 +4457,12 @@ function runSynchroniser(args, opts) {
|
|
|
3769
4457
|
|
|
3770
4458
|
// src/lib/opencode/session-db-boot.ts
|
|
3771
4459
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3772
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4460
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3773
4461
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3774
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4462
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4463
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4464
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4465
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3775
4466
|
function commandError(result) {
|
|
3776
4467
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3777
4468
|
}
|
|
@@ -3814,7 +4505,7 @@ function clearMarker(options) {
|
|
|
3814
4505
|
function markNoReplicate(options, message) {
|
|
3815
4506
|
if (options.noReplicateMarker) {
|
|
3816
4507
|
try {
|
|
3817
|
-
mkdirSync(
|
|
4508
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3818
4509
|
writeFileSync(options.noReplicateMarker, "");
|
|
3819
4510
|
} catch (error2) {
|
|
3820
4511
|
options.log(
|
|
@@ -3939,7 +4630,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3939
4630
|
return null;
|
|
3940
4631
|
}
|
|
3941
4632
|
try {
|
|
3942
|
-
if (
|
|
4633
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3943
4634
|
} catch (error2) {
|
|
3944
4635
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3945
4636
|
options.log(
|
|
@@ -3969,7 +4660,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3969
4660
|
return null;
|
|
3970
4661
|
}
|
|
3971
4662
|
try {
|
|
3972
|
-
mkdirSync(
|
|
4663
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3973
4664
|
writeFileSync(configPath, rendered.stdout);
|
|
3974
4665
|
} catch (error2) {
|
|
3975
4666
|
options.log(
|
|
@@ -4082,20 +4773,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4082
4773
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4083
4774
|
env: {
|
|
4084
4775
|
...env,
|
|
4085
|
-
// The synchroniser reads
|
|
4086
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4776
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4087
4777
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4088
4778
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4779
|
+
),
|
|
4780
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4781
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4782
|
+
),
|
|
4783
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4784
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4785
|
+
),
|
|
4786
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4787
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4089
4788
|
)
|
|
4090
4789
|
}
|
|
4091
4790
|
});
|
|
4092
4791
|
logSynchroniserDiagnostics(result, options);
|
|
4093
4792
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4793
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4794
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4795
|
+
options.log(
|
|
4796
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4797
|
+
"error"
|
|
4798
|
+
);
|
|
4799
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4800
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4801
|
+
}
|
|
4094
4802
|
options.log(
|
|
4095
4803
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4096
4804
|
"warn"
|
|
4097
4805
|
);
|
|
4098
|
-
return false;
|
|
4806
|
+
return { verifyFatal: false };
|
|
4099
4807
|
}
|
|
4100
4808
|
if (result.code === 34) {
|
|
4101
4809
|
reportRecord(
|
|
@@ -4105,14 +4813,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4105
4813
|
null,
|
|
4106
4814
|
options
|
|
4107
4815
|
);
|
|
4108
|
-
return true;
|
|
4816
|
+
return { verifyFatal: true };
|
|
4109
4817
|
}
|
|
4110
4818
|
if (result.code === 33) {
|
|
4111
4819
|
options.log(
|
|
4112
4820
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4113
4821
|
"warn"
|
|
4114
4822
|
);
|
|
4115
|
-
return false;
|
|
4823
|
+
return { verifyFatal: false };
|
|
4116
4824
|
}
|
|
4117
4825
|
if (result.code !== 0) {
|
|
4118
4826
|
options.log(
|
|
@@ -4120,17 +4828,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4120
4828
|
"warn"
|
|
4121
4829
|
);
|
|
4122
4830
|
}
|
|
4123
|
-
return false;
|
|
4831
|
+
return { verifyFatal: false };
|
|
4124
4832
|
}
|
|
4125
4833
|
options.log(
|
|
4126
4834
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4127
4835
|
"debug"
|
|
4128
4836
|
);
|
|
4129
|
-
return false;
|
|
4837
|
+
return { verifyFatal: false };
|
|
4130
4838
|
}
|
|
4131
4839
|
function fileExists(path) {
|
|
4132
4840
|
try {
|
|
4133
|
-
|
|
4841
|
+
statSync3(path);
|
|
4134
4842
|
return true;
|
|
4135
4843
|
} catch (error2) {
|
|
4136
4844
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4213,13 +4921,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4213
4921
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4214
4922
|
return { verifyFatal: false };
|
|
4215
4923
|
}
|
|
4216
|
-
return
|
|
4924
|
+
return verifySessionDb(options, configPath, env);
|
|
4217
4925
|
}
|
|
4218
4926
|
|
|
4219
4927
|
// src/lib/opencode/session-db-provenance.ts
|
|
4220
4928
|
import { createRequire } from "node:module";
|
|
4221
4929
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4222
|
-
import { dirname as
|
|
4930
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4223
4931
|
var require2 = createRequire(import.meta.url);
|
|
4224
4932
|
function readSessionDbMigrationIds(dbPath) {
|
|
4225
4933
|
let db;
|
|
@@ -4285,7 +4993,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4285
4993
|
}
|
|
4286
4994
|
function saveSessionDbProvenanceState(path, state) {
|
|
4287
4995
|
try {
|
|
4288
|
-
mkdirSync2(
|
|
4996
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4289
4997
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4290
4998
|
`, "utf8");
|
|
4291
4999
|
} catch (error2) {
|
|
@@ -4866,7 +5574,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4866
5574
|
toDelete.push(s.id);
|
|
4867
5575
|
}
|
|
4868
5576
|
}
|
|
4869
|
-
|
|
5577
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5578
|
+
return toDelete.sort(
|
|
5579
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5580
|
+
);
|
|
4870
5581
|
}
|
|
4871
5582
|
var DEFAULT_INTERVAL = "1h";
|
|
4872
5583
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4926,13 +5637,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4926
5637
|
}
|
|
4927
5638
|
|
|
4928
5639
|
// src/lib/opencode/session-db-size.ts
|
|
4929
|
-
import { statSync as
|
|
5640
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4930
5641
|
import { join as join6 } from "node:path";
|
|
4931
5642
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4932
5643
|
function statSessionDbBytes(homeDir) {
|
|
4933
5644
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4934
5645
|
try {
|
|
4935
|
-
return
|
|
5646
|
+
return statSync4(dbPath).size;
|
|
4936
5647
|
} catch (err) {
|
|
4937
5648
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4938
5649
|
if (!isMissingFile) {
|
|
@@ -4958,7 +5669,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4958
5669
|
}
|
|
4959
5670
|
|
|
4960
5671
|
// src/lib/opencode/log-tail.ts
|
|
4961
|
-
import { statSync as
|
|
5672
|
+
import { statSync as statSync5 } from "node:fs";
|
|
4962
5673
|
import { homedir as homedir5 } from "node:os";
|
|
4963
5674
|
import { join as join7 } from "node:path";
|
|
4964
5675
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -4981,186 +5692,63 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4981
5692
|
let inode = null;
|
|
4982
5693
|
let baselineReady = true;
|
|
4983
5694
|
try {
|
|
4984
|
-
const initial =
|
|
5695
|
+
const initial = statSync5(logPath);
|
|
4985
5696
|
offset = initial.size;
|
|
4986
5697
|
inode = initial.ino;
|
|
4987
5698
|
} catch (error2) {
|
|
4988
5699
|
if (!isEnoent(error2)) {
|
|
4989
|
-
reportFailure("initial stat", logPath, error2);
|
|
4990
|
-
baselineReady = false;
|
|
4991
|
-
}
|
|
4992
|
-
}
|
|
4993
|
-
let polling = false;
|
|
4994
|
-
let stopped = false;
|
|
4995
|
-
const poll = async () => {
|
|
4996
|
-
if (polling || stopped) return;
|
|
4997
|
-
polling = true;
|
|
4998
|
-
try {
|
|
4999
|
-
let current;
|
|
5000
|
-
try {
|
|
5001
|
-
current = await stat(logPath);
|
|
5002
|
-
} catch (error2) {
|
|
5003
|
-
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5004
|
-
return;
|
|
5005
|
-
}
|
|
5006
|
-
if (!baselineReady) {
|
|
5007
|
-
offset = current.size;
|
|
5008
|
-
inode = current.ino;
|
|
5009
|
-
baselineReady = true;
|
|
5010
|
-
return;
|
|
5011
|
-
}
|
|
5012
|
-
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5013
|
-
offset = 0;
|
|
5014
|
-
}
|
|
5015
|
-
inode = current.ino;
|
|
5016
|
-
if (current.size === offset) return;
|
|
5017
|
-
const length = current.size - offset;
|
|
5018
|
-
const fh = await open2(logPath, "r");
|
|
5019
|
-
try {
|
|
5020
|
-
const buf = Buffer.alloc(length);
|
|
5021
|
-
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5022
|
-
offset += bytesRead;
|
|
5023
|
-
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5024
|
-
} finally {
|
|
5025
|
-
await fh.close();
|
|
5026
|
-
}
|
|
5027
|
-
} catch (error2) {
|
|
5028
|
-
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5029
|
-
} finally {
|
|
5030
|
-
polling = false;
|
|
5031
|
-
}
|
|
5032
|
-
};
|
|
5033
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5034
|
-
void poll();
|
|
5035
|
-
return {
|
|
5036
|
-
stop: () => {
|
|
5037
|
-
stopped = true;
|
|
5038
|
-
clearInterval(interval);
|
|
5039
|
-
}
|
|
5040
|
-
};
|
|
5041
|
-
}
|
|
5042
|
-
|
|
5043
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5044
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5045
|
-
import { dirname as dirname4 } from "node:path";
|
|
5046
|
-
function errorMessage(error2) {
|
|
5047
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5048
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5049
|
-
}
|
|
5050
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5051
|
-
try {
|
|
5052
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5053
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5054
|
-
if (availableBytes < requiredBytes) {
|
|
5055
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5056
|
-
}
|
|
5057
|
-
return null;
|
|
5058
|
-
} catch (err) {
|
|
5059
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5060
|
-
}
|
|
5061
|
-
}
|
|
5062
|
-
function readLogicalBytes(db) {
|
|
5063
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5064
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5065
|
-
return pageCount * pageSize;
|
|
5066
|
-
}
|
|
5067
|
-
function readCheckpointResult(db) {
|
|
5068
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5069
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5070
|
-
}
|
|
5071
|
-
async function probeReclaimAvailability(input) {
|
|
5072
|
-
const { dbPath, requiredBytes } = input;
|
|
5073
|
-
let sqlite;
|
|
5074
|
-
try {
|
|
5075
|
-
sqlite = await import("node:sqlite");
|
|
5076
|
-
} catch (err) {
|
|
5077
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5078
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5079
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5080
|
-
}
|
|
5081
|
-
let autoVacuum = null;
|
|
5082
|
-
try {
|
|
5083
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5084
|
-
try {
|
|
5085
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5086
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5087
|
-
} finally {
|
|
5088
|
-
db.close();
|
|
5089
|
-
}
|
|
5090
|
-
} catch (err) {
|
|
5091
|
-
console.warn(
|
|
5092
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5093
|
-
);
|
|
5094
|
-
}
|
|
5095
|
-
if (autoVacuum !== 0) return null;
|
|
5096
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5097
|
-
}
|
|
5098
|
-
async function reclaimSessionDbSpace(input) {
|
|
5099
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5100
|
-
let sqlite;
|
|
5101
|
-
try {
|
|
5102
|
-
sqlite = await import("node:sqlite");
|
|
5103
|
-
} catch (err) {
|
|
5104
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5105
|
-
console.warn(
|
|
5106
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5107
|
-
);
|
|
5108
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5700
|
+
reportFailure("initial stat", logPath, error2);
|
|
5701
|
+
baselineReady = false;
|
|
5702
|
+
}
|
|
5109
5703
|
}
|
|
5110
|
-
|
|
5111
|
-
let
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
);
|
|
5121
|
-
return
|
|
5122
|
-
ok: false,
|
|
5123
|
-
skipped: "full-vacuum-blocked",
|
|
5124
|
-
detail: "a session turn is live"
|
|
5125
|
-
};
|
|
5704
|
+
let polling = false;
|
|
5705
|
+
let stopped = false;
|
|
5706
|
+
const poll = async () => {
|
|
5707
|
+
if (polling || stopped) return;
|
|
5708
|
+
polling = true;
|
|
5709
|
+
try {
|
|
5710
|
+
let current;
|
|
5711
|
+
try {
|
|
5712
|
+
current = await stat(logPath);
|
|
5713
|
+
} catch (error2) {
|
|
5714
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5715
|
+
return;
|
|
5126
5716
|
}
|
|
5127
|
-
|
|
5128
|
-
|
|
5129
|
-
|
|
5130
|
-
|
|
5131
|
-
|
|
5132
|
-
);
|
|
5133
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5717
|
+
if (!baselineReady) {
|
|
5718
|
+
offset = current.size;
|
|
5719
|
+
inode = current.ino;
|
|
5720
|
+
baselineReady = true;
|
|
5721
|
+
return;
|
|
5134
5722
|
}
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5723
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5724
|
+
offset = 0;
|
|
5725
|
+
}
|
|
5726
|
+
inode = current.ino;
|
|
5727
|
+
if (current.size === offset) return;
|
|
5728
|
+
const length = current.size - offset;
|
|
5729
|
+
const fh = await open2(logPath, "r");
|
|
5730
|
+
try {
|
|
5731
|
+
const buf = Buffer.alloc(length);
|
|
5732
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5733
|
+
offset += bytesRead;
|
|
5734
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5735
|
+
} finally {
|
|
5736
|
+
await fh.close();
|
|
5737
|
+
}
|
|
5738
|
+
} catch (error2) {
|
|
5739
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5740
|
+
} finally {
|
|
5741
|
+
polling = false;
|
|
5141
5742
|
}
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5743
|
+
};
|
|
5744
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5745
|
+
void poll();
|
|
5746
|
+
return {
|
|
5747
|
+
stop: () => {
|
|
5748
|
+
stopped = true;
|
|
5749
|
+
clearInterval(interval);
|
|
5149
5750
|
}
|
|
5150
|
-
|
|
5151
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5152
|
-
);
|
|
5153
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5154
|
-
} catch (err) {
|
|
5155
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5156
|
-
return {
|
|
5157
|
-
ok: false,
|
|
5158
|
-
skipped: "reclaim-error",
|
|
5159
|
-
detail: errorMessage(err)
|
|
5160
|
-
};
|
|
5161
|
-
} finally {
|
|
5162
|
-
db?.close();
|
|
5163
|
-
}
|
|
5751
|
+
};
|
|
5164
5752
|
}
|
|
5165
5753
|
|
|
5166
5754
|
// src/lib/tunnel/connection.ts
|
|
@@ -5392,6 +5980,8 @@ var StreamForwarder = class {
|
|
|
5392
5980
|
if (done) break;
|
|
5393
5981
|
const chunk = Buffer.from(value);
|
|
5394
5982
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5983
|
+
await stream.pausePromise;
|
|
5984
|
+
if (ac.signal.aborted) break;
|
|
5395
5985
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5396
5986
|
if (this.binaryFramesSupported) {
|
|
5397
5987
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5434,6 +6024,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5434
6024
|
}
|
|
5435
6025
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5436
6026
|
var BASE_RECONNECT_DELAY = 500;
|
|
6027
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5437
6028
|
function getReconnectDelay(attempt) {
|
|
5438
6029
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5439
6030
|
const jitter = Math.random() * 1e3;
|
|
@@ -5505,10 +6096,27 @@ function connectTunnel(options) {
|
|
|
5505
6096
|
ws.close();
|
|
5506
6097
|
reject(new Error("Connection timeout"));
|
|
5507
6098
|
}, 3e4);
|
|
6099
|
+
let relaySilenceTimeout;
|
|
6100
|
+
let relaySilenceTimedOut = false;
|
|
6101
|
+
const clearRelaySilenceTimeout = () => {
|
|
6102
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6103
|
+
clearTimeout(relaySilenceTimeout);
|
|
6104
|
+
relaySilenceTimeout = void 0;
|
|
6105
|
+
};
|
|
6106
|
+
const armRelaySilenceTimeout = () => {
|
|
6107
|
+
clearRelaySilenceTimeout();
|
|
6108
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6109
|
+
relaySilenceTimeout = void 0;
|
|
6110
|
+
relaySilenceTimedOut = true;
|
|
6111
|
+
ws.terminate();
|
|
6112
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6113
|
+
relaySilenceTimeout.unref?.();
|
|
6114
|
+
};
|
|
5508
6115
|
let upgradeRejection = null;
|
|
5509
6116
|
let upgradeRejectionReason = null;
|
|
5510
6117
|
ws.on("unexpected-response", (_req, res) => {
|
|
5511
6118
|
clearTimeout(connectionTimeout);
|
|
6119
|
+
clearRelaySilenceTimeout();
|
|
5512
6120
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5513
6121
|
upgradeRejectionReason = reason;
|
|
5514
6122
|
const chunks = [];
|
|
@@ -5538,6 +6146,7 @@ function connectTunnel(options) {
|
|
|
5538
6146
|
onInfo?.("WebSocket connection established");
|
|
5539
6147
|
});
|
|
5540
6148
|
ws.on("message", (data, isBinary) => {
|
|
6149
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5541
6150
|
if (isBinary) {
|
|
5542
6151
|
try {
|
|
5543
6152
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5571,6 +6180,7 @@ function connectTunnel(options) {
|
|
|
5571
6180
|
switch (message.type) {
|
|
5572
6181
|
case "connected": {
|
|
5573
6182
|
clearTimeout(connectionTimeout);
|
|
6183
|
+
armRelaySilenceTimeout();
|
|
5574
6184
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5575
6185
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5576
6186
|
onConnected?.(connectedAgentId);
|
|
@@ -5595,6 +6205,7 @@ function connectTunnel(options) {
|
|
|
5595
6205
|
});
|
|
5596
6206
|
ws.on("error", (error2) => {
|
|
5597
6207
|
clearTimeout(connectionTimeout);
|
|
6208
|
+
clearRelaySilenceTimeout();
|
|
5598
6209
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5599
6210
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5600
6211
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5606,7 +6217,8 @@ function connectTunnel(options) {
|
|
|
5606
6217
|
);
|
|
5607
6218
|
});
|
|
5608
6219
|
ws.on("close", (code, reason) => {
|
|
5609
|
-
|
|
6220
|
+
clearRelaySilenceTimeout();
|
|
6221
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5610
6222
|
forwarder.abortAll();
|
|
5611
6223
|
onDisconnected?.(code, reasonStr);
|
|
5612
6224
|
});
|
|
@@ -5614,9 +6226,10 @@ function connectTunnel(options) {
|
|
|
5614
6226
|
}
|
|
5615
6227
|
|
|
5616
6228
|
// src/lib/tunnel/runner-connection.ts
|
|
5617
|
-
var RunnerConnection = class {
|
|
6229
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5618
6230
|
opts;
|
|
5619
6231
|
sleep;
|
|
6232
|
+
now;
|
|
5620
6233
|
connection = null;
|
|
5621
6234
|
resolvedAgentId;
|
|
5622
6235
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5625,10 +6238,17 @@ var RunnerConnection = class {
|
|
|
5625
6238
|
reconnectPromise = null;
|
|
5626
6239
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5627
6240
|
reconnectAttempt = 0;
|
|
6241
|
+
generation = 0;
|
|
6242
|
+
disconnectedAtMs = null;
|
|
6243
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6244
|
+
"No pong received",
|
|
6245
|
+
"Orphaned socket"
|
|
6246
|
+
]);
|
|
5628
6247
|
constructor(opts) {
|
|
5629
6248
|
this.opts = opts;
|
|
5630
6249
|
this.resolvedAgentId = opts.agentId;
|
|
5631
6250
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6251
|
+
this.now = opts.now ?? Date.now;
|
|
5632
6252
|
}
|
|
5633
6253
|
get agentId() {
|
|
5634
6254
|
return this.resolvedAgentId;
|
|
@@ -5664,14 +6284,24 @@ var RunnerConnection = class {
|
|
|
5664
6284
|
port: this.opts.port,
|
|
5665
6285
|
openCodePassword: this.opts.openCodePassword,
|
|
5666
6286
|
onConnected: (agentId) => {
|
|
6287
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5667
6288
|
this.reconnectAttempt = 0;
|
|
5668
6289
|
this.reconnecting = false;
|
|
5669
6290
|
this.resolvedAgentId = agentId;
|
|
5670
|
-
|
|
6291
|
+
this.generation++;
|
|
6292
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6293
|
+
this.disconnectedAtMs = null;
|
|
6294
|
+
events.onConnected(agentId, isReconnect, {
|
|
6295
|
+
generation: this.generation,
|
|
6296
|
+
attempt,
|
|
6297
|
+
downtimeMs
|
|
6298
|
+
});
|
|
5671
6299
|
},
|
|
5672
6300
|
onDisconnected: (code, reason) => {
|
|
5673
|
-
|
|
5674
|
-
|
|
6301
|
+
this.disconnectedAtMs = this.now();
|
|
6302
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6303
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6304
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5675
6305
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5676
6306
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5677
6307
|
});
|
|
@@ -6010,6 +6640,101 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6010
6640
|
};
|
|
6011
6641
|
}
|
|
6012
6642
|
|
|
6643
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6644
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6645
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6646
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6647
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6648
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6649
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6650
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6651
|
+
var BAND_RANK = {
|
|
6652
|
+
low: 1,
|
|
6653
|
+
very_low: 2,
|
|
6654
|
+
critical: 3
|
|
6655
|
+
};
|
|
6656
|
+
function bandFor(freeBytes) {
|
|
6657
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6658
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6659
|
+
return "low";
|
|
6660
|
+
}
|
|
6661
|
+
function formatBytes(bytes) {
|
|
6662
|
+
if (bytes === null) return "unavailable";
|
|
6663
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6664
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6665
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6666
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6667
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6668
|
+
}
|
|
6669
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6670
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6671
|
+
}
|
|
6672
|
+
function evaluateLowDiskWarning(input) {
|
|
6673
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6674
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6675
|
+
return { warning: null, state };
|
|
6676
|
+
}
|
|
6677
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6678
|
+
if (recovered) {
|
|
6679
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6680
|
+
}
|
|
6681
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6682
|
+
if (!low) return { warning: null, state };
|
|
6683
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6684
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6685
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6686
|
+
const worse = currentRank > previousRank;
|
|
6687
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6688
|
+
const nextState = {
|
|
6689
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6690
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6691
|
+
};
|
|
6692
|
+
return {
|
|
6693
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6694
|
+
state: nextState
|
|
6695
|
+
};
|
|
6696
|
+
}
|
|
6697
|
+
|
|
6698
|
+
// src/lib/opencode/session-store-health.ts
|
|
6699
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6700
|
+
function createSessionStoreHealth(options = {}) {
|
|
6701
|
+
const now = options.now ?? (() => Date.now());
|
|
6702
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6703
|
+
let lastUnwritableAt = null;
|
|
6704
|
+
let lastWritableAt = null;
|
|
6705
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6706
|
+
return {
|
|
6707
|
+
observeUnwritable: () => {
|
|
6708
|
+
lastUnwritableAt = now();
|
|
6709
|
+
},
|
|
6710
|
+
observeWritable: () => {
|
|
6711
|
+
lastWritableAt = now();
|
|
6712
|
+
},
|
|
6713
|
+
isUnwritable: () => {
|
|
6714
|
+
if (lastUnwritableAt === null) return false;
|
|
6715
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6716
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6717
|
+
},
|
|
6718
|
+
observeCause: (sessionId, cause) => {
|
|
6719
|
+
if (sessionId === null) return;
|
|
6720
|
+
causes.delete(sessionId);
|
|
6721
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6722
|
+
while (causes.size > 256) {
|
|
6723
|
+
const oldest = causes.keys().next().value;
|
|
6724
|
+
if (typeof oldest !== "string") break;
|
|
6725
|
+
causes.delete(oldest);
|
|
6726
|
+
}
|
|
6727
|
+
},
|
|
6728
|
+
readCause: (sessionId) => {
|
|
6729
|
+
const observed = causes.get(sessionId);
|
|
6730
|
+
if (!observed) return null;
|
|
6731
|
+
causes.delete(sessionId);
|
|
6732
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6733
|
+
return observed.cause;
|
|
6734
|
+
}
|
|
6735
|
+
};
|
|
6736
|
+
}
|
|
6737
|
+
|
|
6013
6738
|
// src/lib/channels/driver.ts
|
|
6014
6739
|
import { homedir as homedir6 } from "node:os";
|
|
6015
6740
|
|
|
@@ -6466,8 +7191,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6466
7191
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6467
7192
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6468
7193
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7194
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7195
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7196
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7197
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6469
7198
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6470
7199
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7200
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6471
7201
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6472
7202
|
var HEARTBEAT_MS = 6e4;
|
|
6473
7203
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6529,6 +7259,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6529
7259
|
maxActiveSessions;
|
|
6530
7260
|
watcherStallMs;
|
|
6531
7261
|
wedgeWarningIntervalMs;
|
|
7262
|
+
sessionStoreHealth;
|
|
6532
7263
|
openCodeClient;
|
|
6533
7264
|
/** Cache of conversationId → opencode sessionId. */
|
|
6534
7265
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6621,6 +7352,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6621
7352
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6622
7353
|
*/
|
|
6623
7354
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7355
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6624
7356
|
/**
|
|
6625
7357
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6626
7358
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6653,10 +7385,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6653
7385
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6654
7386
|
*/
|
|
6655
7387
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7388
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7389
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6656
7390
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6657
7391
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6658
7392
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6659
7393
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7394
|
+
/**
|
|
7395
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7396
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7397
|
+
* identical-text turn, so it never produces a correlation.
|
|
7398
|
+
*/
|
|
7399
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6660
7400
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6661
7401
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6662
7402
|
/**
|
|
@@ -6742,21 +7482,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6742
7482
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6743
7483
|
*/
|
|
6744
7484
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6745
|
-
/**
|
|
6746
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6747
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6748
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6749
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6750
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6751
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6752
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6753
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6754
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6755
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6756
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6757
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6758
|
-
*/
|
|
6759
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6760
7485
|
/**
|
|
6761
7486
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6762
7487
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6774,8 +7499,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6774
7499
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6775
7500
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6776
7501
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6777
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6778
|
-
*
|
|
7502
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7503
|
+
* the terminal-report fence and is never sent again).
|
|
6779
7504
|
*/
|
|
6780
7505
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6781
7506
|
/**
|
|
@@ -6898,6 +7623,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6898
7623
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6899
7624
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6900
7625
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7626
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6901
7627
|
}
|
|
6902
7628
|
get isV2() {
|
|
6903
7629
|
return this.openCodeClient.version === "v2";
|
|
@@ -6913,7 +7639,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6913
7639
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6914
7640
|
}
|
|
6915
7641
|
async listSessions() {
|
|
6916
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7642
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7643
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7644
|
+
});
|
|
6917
7645
|
}
|
|
6918
7646
|
async sessionExists(sessionId) {
|
|
6919
7647
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7090,6 +7818,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7090
7818
|
}
|
|
7091
7819
|
return ids;
|
|
7092
7820
|
}
|
|
7821
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7822
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7823
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7824
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7825
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7826
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7827
|
+
}
|
|
7828
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7829
|
+
}
|
|
7830
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7831
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7832
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7833
|
+
this.awaitingReadopt.delete(messageId);
|
|
7834
|
+
this.sessions.delete(conversationId);
|
|
7835
|
+
this.supersede(conversationId, sessionId);
|
|
7836
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7837
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7838
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7839
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7840
|
+
this.log({
|
|
7841
|
+
level: "error",
|
|
7842
|
+
message: errorMessage3,
|
|
7843
|
+
conversation_id: conversationId,
|
|
7844
|
+
message_id: messageId
|
|
7845
|
+
});
|
|
7846
|
+
const reportError = await this.reportUntrackableAck(
|
|
7847
|
+
conversationId,
|
|
7848
|
+
messageId,
|
|
7849
|
+
null,
|
|
7850
|
+
errorMessage3
|
|
7851
|
+
);
|
|
7852
|
+
if (reportError !== null) {
|
|
7853
|
+
this.log({
|
|
7854
|
+
level: "warn",
|
|
7855
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7856
|
+
conversation_id: conversationId,
|
|
7857
|
+
message_id: messageId
|
|
7858
|
+
});
|
|
7859
|
+
}
|
|
7860
|
+
if (source === "readopt") {
|
|
7861
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7862
|
+
}
|
|
7863
|
+
}
|
|
7093
7864
|
/**
|
|
7094
7865
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7095
7866
|
*
|
|
@@ -7269,6 +8040,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7269
8040
|
}
|
|
7270
8041
|
break;
|
|
7271
8042
|
}
|
|
8043
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8044
|
+
if (correlation) {
|
|
8045
|
+
correlation.attempts += 1;
|
|
8046
|
+
let correlatedId = null;
|
|
8047
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8048
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8049
|
+
conv,
|
|
8050
|
+
message,
|
|
8051
|
+
correlation.sessionId
|
|
8052
|
+
);
|
|
8053
|
+
if (polled.ok)
|
|
8054
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8055
|
+
} else if (correlation.knownUserIds === null) {
|
|
8056
|
+
this.log({
|
|
8057
|
+
level: "debug",
|
|
8058
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8059
|
+
conversation_id: conv.id,
|
|
8060
|
+
message_id: message.id
|
|
8061
|
+
});
|
|
8062
|
+
}
|
|
8063
|
+
if (correlatedId !== null) {
|
|
8064
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8065
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8066
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8067
|
+
this.dispatched.add(message.id);
|
|
8068
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8069
|
+
dispatched += 1;
|
|
8070
|
+
this.log({
|
|
8071
|
+
level: "info",
|
|
8072
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8073
|
+
conversation_id: conv.id,
|
|
8074
|
+
message_id: message.id
|
|
8075
|
+
});
|
|
8076
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8077
|
+
opencode_message_id: correlatedId
|
|
8078
|
+
});
|
|
8079
|
+
continue;
|
|
8080
|
+
}
|
|
8081
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8082
|
+
await this.handoffUntrackableCorrelation(
|
|
8083
|
+
conv.id,
|
|
8084
|
+
message.id,
|
|
8085
|
+
correlation.sessionId,
|
|
8086
|
+
correlation.attempts,
|
|
8087
|
+
"fresh"
|
|
8088
|
+
);
|
|
8089
|
+
break;
|
|
8090
|
+
}
|
|
8091
|
+
break;
|
|
8092
|
+
}
|
|
7272
8093
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7273
8094
|
if (effectiveOpencodeMessageId) {
|
|
7274
8095
|
const outcome = await this.resolveRedrive(
|
|
@@ -7290,6 +8111,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7290
8111
|
model: message.opencode_model ?? void 0
|
|
7291
8112
|
};
|
|
7292
8113
|
let opencodeMessageId;
|
|
8114
|
+
let preDispatchUserIds = null;
|
|
7293
8115
|
try {
|
|
7294
8116
|
this.log({
|
|
7295
8117
|
level: "info",
|
|
@@ -7319,7 +8141,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7319
8141
|
message.content,
|
|
7320
8142
|
options,
|
|
7321
8143
|
sendAttachments,
|
|
7322
|
-
this.openCodeClient
|
|
8144
|
+
this.openCodeClient,
|
|
8145
|
+
{
|
|
8146
|
+
onKnownUserIds: (ids) => {
|
|
8147
|
+
preDispatchUserIds = ids;
|
|
8148
|
+
},
|
|
8149
|
+
onKnownUserIdsError: (error2) => {
|
|
8150
|
+
this.log({
|
|
8151
|
+
level: "warn",
|
|
8152
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8153
|
+
conversation_id: conv.id,
|
|
8154
|
+
message_id: message.id
|
|
8155
|
+
});
|
|
8156
|
+
}
|
|
8157
|
+
}
|
|
7323
8158
|
)
|
|
7324
8159
|
);
|
|
7325
8160
|
} catch (err) {
|
|
@@ -7327,6 +8162,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7327
8162
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7328
8163
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7329
8164
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8165
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7330
8166
|
this.log({
|
|
7331
8167
|
level: "error",
|
|
7332
8168
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7402,39 +8238,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7402
8238
|
if (this.isV2) {
|
|
7403
8239
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7404
8240
|
}
|
|
7405
|
-
|
|
7406
|
-
|
|
7407
|
-
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
message_id: message.id
|
|
7412
|
-
});
|
|
7413
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7414
|
-
continue;
|
|
7415
|
-
}
|
|
7416
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7417
|
-
this.sessions.delete(conv.id);
|
|
7418
|
-
this.supersede(conv.id, sessionId);
|
|
7419
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8241
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8242
|
+
sessionId,
|
|
8243
|
+
content: message.content,
|
|
8244
|
+
attempts: 0,
|
|
8245
|
+
knownUserIds: preDispatchUserIds
|
|
8246
|
+
});
|
|
7420
8247
|
this.log({
|
|
7421
|
-
level: "
|
|
7422
|
-
message:
|
|
8248
|
+
level: "warn",
|
|
8249
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7423
8250
|
conversation_id: conv.id,
|
|
7424
8251
|
message_id: message.id
|
|
7425
8252
|
});
|
|
7426
|
-
|
|
7427
|
-
this.log({
|
|
7428
|
-
level: "warn",
|
|
7429
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7430
|
-
conversation_id: conv.id,
|
|
7431
|
-
message_id: message.id
|
|
7432
|
-
});
|
|
7433
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7434
|
-
});
|
|
8253
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7435
8254
|
break;
|
|
7436
8255
|
}
|
|
7437
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7438
8256
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7439
8257
|
this.releasedOpencodeIds.delete(message.id);
|
|
7440
8258
|
this.dispatched.add(message.id);
|
|
@@ -8039,23 +8857,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8039
8857
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8040
8858
|
return 1;
|
|
8041
8859
|
}
|
|
8042
|
-
/**
|
|
8043
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8044
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8045
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8046
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8047
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8048
|
-
* count, since a new session is a genuinely different attempt.
|
|
8049
|
-
*/
|
|
8050
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8051
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8052
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8053
|
-
existing.count += 1;
|
|
8054
|
-
return existing.count;
|
|
8055
|
-
}
|
|
8056
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8057
|
-
return 1;
|
|
8058
|
-
}
|
|
8059
8860
|
/** Record one queued give-up toward the never-started bound. */
|
|
8060
8861
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8061
8862
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8367,8 +9168,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8367
9168
|
b2LastDescendantCheckMs: 0,
|
|
8368
9169
|
b2RootOngoingHeldLogged: false,
|
|
8369
9170
|
b2AbandonedSignalled: false,
|
|
9171
|
+
b2AbandonedWithoutAnswer: false,
|
|
8370
9172
|
ambiguousPinnedSinceMs: 0,
|
|
8371
9173
|
ambiguousResolved: false,
|
|
9174
|
+
sessionStoreFailurePending: false,
|
|
8372
9175
|
neverStartedResolved: false
|
|
8373
9176
|
});
|
|
8374
9177
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8390,6 +9193,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8390
9193
|
this.bufferedSessionErrors.delete(oldest);
|
|
8391
9194
|
}
|
|
8392
9195
|
}
|
|
9196
|
+
retainSessionError(event) {
|
|
9197
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9198
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9199
|
+
reason: event.reason,
|
|
9200
|
+
observedAt: this.now()
|
|
9201
|
+
});
|
|
9202
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9203
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9204
|
+
if (typeof oldest !== "string") break;
|
|
9205
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9206
|
+
}
|
|
9207
|
+
}
|
|
9208
|
+
readRetainedSessionError(sessionId) {
|
|
9209
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9210
|
+
if (!retained) return null;
|
|
9211
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9212
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9213
|
+
return null;
|
|
9214
|
+
}
|
|
9215
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9216
|
+
return retained.reason;
|
|
9217
|
+
}
|
|
8393
9218
|
/**
|
|
8394
9219
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8395
9220
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8485,8 +9310,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8485
9310
|
b2LastDescendantCheckMs: 0,
|
|
8486
9311
|
b2RootOngoingHeldLogged: false,
|
|
8487
9312
|
b2AbandonedSignalled: false,
|
|
9313
|
+
b2AbandonedWithoutAnswer: false,
|
|
8488
9314
|
ambiguousPinnedSinceMs: 0,
|
|
8489
9315
|
ambiguousResolved: false,
|
|
9316
|
+
sessionStoreFailurePending: false,
|
|
8490
9317
|
neverStartedResolved: false
|
|
8491
9318
|
});
|
|
8492
9319
|
}
|
|
@@ -8714,9 +9541,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8714
9541
|
return;
|
|
8715
9542
|
}
|
|
8716
9543
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9544
|
+
this.retainSessionError(event);
|
|
8717
9545
|
this.log({
|
|
8718
|
-
level: "
|
|
8719
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9546
|
+
level: "warn",
|
|
9547
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8720
9548
|
conversation_id: watcher.conv.id
|
|
8721
9549
|
});
|
|
8722
9550
|
return;
|
|
@@ -8746,9 +9574,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8746
9574
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8747
9575
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8748
9576
|
if (state !== "queued") {
|
|
9577
|
+
this.retainSessionError(event);
|
|
8749
9578
|
this.log({
|
|
8750
|
-
level: "
|
|
8751
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9579
|
+
level: "warn",
|
|
9580
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8752
9581
|
conversation_id: watcher.conv.id,
|
|
8753
9582
|
message_id: inFlight.evidentMessageId
|
|
8754
9583
|
});
|
|
@@ -8900,6 +9729,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8900
9729
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8901
9730
|
this.neverStartedAttempts.delete(id);
|
|
8902
9731
|
}
|
|
9732
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9733
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9734
|
+
return;
|
|
9735
|
+
}
|
|
8903
9736
|
if (inFlight.neverStartedResolved) {
|
|
8904
9737
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8905
9738
|
inFlight.neverStartedResolved = false;
|
|
@@ -8973,7 +9806,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8973
9806
|
if (state === "failed") {
|
|
8974
9807
|
this.anchorDeliveryDeadline(inFlight);
|
|
8975
9808
|
if (!inFlight.done) {
|
|
8976
|
-
const
|
|
9809
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9810
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9811
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9812
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9813
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9814
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9815
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9816
|
+
(detail) => detail !== null
|
|
9817
|
+
);
|
|
9818
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
8977
9819
|
this.log({
|
|
8978
9820
|
level: "error",
|
|
8979
9821
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9059,10 +9901,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9059
9901
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9060
9902
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9061
9903
|
inFlight.b2AbandonedSignalled = false;
|
|
9904
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9062
9905
|
}
|
|
9063
9906
|
} else {
|
|
9064
9907
|
if (inFlight.b2AbandonedSignalled) {
|
|
9065
|
-
|
|
9908
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9909
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9910
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9911
|
+
sessionId,
|
|
9912
|
+
watcher,
|
|
9913
|
+
inFlight,
|
|
9914
|
+
messages,
|
|
9915
|
+
reply?.info?.finish ?? reply?.finish
|
|
9916
|
+
);
|
|
9917
|
+
} else {
|
|
9918
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9919
|
+
}
|
|
9066
9920
|
return;
|
|
9067
9921
|
}
|
|
9068
9922
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9079,17 +9933,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9079
9933
|
descendantOngoing,
|
|
9080
9934
|
rootOngoing
|
|
9081
9935
|
})) {
|
|
9936
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9937
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9938
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9939
|
+
watched_for_ms: pinnedForMs,
|
|
9940
|
+
run_state: "failed",
|
|
9941
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9942
|
+
});
|
|
9943
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9944
|
+
return;
|
|
9945
|
+
}
|
|
9082
9946
|
inFlight.b2AbandonedSignalled = true;
|
|
9947
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9948
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9949
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9950
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9951
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9952
|
+
watched_for_ms: pinnedForMs,
|
|
9953
|
+
run_state: "failed",
|
|
9954
|
+
finish,
|
|
9955
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9956
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9957
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9958
|
+
});
|
|
9959
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9960
|
+
sessionId,
|
|
9961
|
+
watcher,
|
|
9962
|
+
inFlight,
|
|
9963
|
+
messages,
|
|
9964
|
+
finish
|
|
9965
|
+
);
|
|
9966
|
+
return;
|
|
9967
|
+
}
|
|
9083
9968
|
this.log({
|
|
9084
9969
|
level: "warn",
|
|
9085
9970
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9086
9971
|
conversation_id: conv.id,
|
|
9087
9972
|
message_id: id
|
|
9088
9973
|
});
|
|
9089
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9090
9974
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9091
9975
|
watched_for_ms: pinnedForMs,
|
|
9092
|
-
finish
|
|
9976
|
+
finish,
|
|
9093
9977
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9094
9978
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9095
9979
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9137,6 +10021,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9137
10021
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9138
10022
|
sessionOngoing: ongoing
|
|
9139
10023
|
})) {
|
|
10024
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10025
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10026
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10027
|
+
watched_for_ms: pinnedForMs,
|
|
10028
|
+
run_state: "failed",
|
|
10029
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10030
|
+
});
|
|
10031
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10032
|
+
return;
|
|
10033
|
+
}
|
|
9140
10034
|
inFlight.ambiguousResolved = true;
|
|
9141
10035
|
this.log({
|
|
9142
10036
|
level: "warn",
|
|
@@ -9345,20 +10239,90 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9345
10239
|
}
|
|
9346
10240
|
this.log({
|
|
9347
10241
|
level: "warn",
|
|
9348
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10242
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10243
|
+
conversation_id: conv.id,
|
|
10244
|
+
message_id: inFlight.evidentMessageId
|
|
10245
|
+
});
|
|
10246
|
+
return;
|
|
10247
|
+
}
|
|
10248
|
+
inFlight.done = true;
|
|
10249
|
+
await this.reportSubagentAuthFailures(
|
|
10250
|
+
watcher.conv.id,
|
|
10251
|
+
inFlight.opencodeMessageId,
|
|
10252
|
+
inFlight.evidentMessageId,
|
|
10253
|
+
messages
|
|
10254
|
+
);
|
|
10255
|
+
}
|
|
10256
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10257
|
+
}
|
|
10258
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10259
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10260
|
+
const conv = watcher.conv;
|
|
10261
|
+
const legacyError = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
10262
|
+
const replyShape = describeAbandonedReplyShape(messages, inFlight.opencodeMessageId);
|
|
10263
|
+
const error2 = replyShape === null ? legacyError : `${legacyError} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
10264
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10265
|
+
this.log({
|
|
10266
|
+
level: "warn",
|
|
10267
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10268
|
+
conversation_id: conv.id,
|
|
10269
|
+
message_id: inFlight.evidentMessageId
|
|
10270
|
+
});
|
|
10271
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10272
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10273
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10274
|
+
messages,
|
|
10275
|
+
inFlight.opencodeMessageId,
|
|
10276
|
+
inFlight.evidentMessageId
|
|
10277
|
+
);
|
|
10278
|
+
try {
|
|
10279
|
+
await this.markFailed(
|
|
10280
|
+
conv.id,
|
|
10281
|
+
inFlight.evidentMessageId,
|
|
10282
|
+
sessionId,
|
|
10283
|
+
error2,
|
|
10284
|
+
usage,
|
|
10285
|
+
void 0,
|
|
10286
|
+
usageAgentName,
|
|
10287
|
+
subagentInvocations
|
|
10288
|
+
);
|
|
10289
|
+
} catch (err) {
|
|
10290
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10291
|
+
if (err instanceof ChannelTerminalError) {
|
|
10292
|
+
this.log({
|
|
10293
|
+
level: "warn",
|
|
10294
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9349
10295
|
conversation_id: conv.id,
|
|
9350
10296
|
message_id: inFlight.evidentMessageId
|
|
9351
10297
|
});
|
|
10298
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9352
10299
|
return;
|
|
9353
10300
|
}
|
|
9354
|
-
inFlight.
|
|
9355
|
-
|
|
9356
|
-
|
|
9357
|
-
|
|
9358
|
-
|
|
9359
|
-
|
|
9360
|
-
|
|
10301
|
+
if (this.now() >= inFlight.deadline) {
|
|
10302
|
+
this.log({
|
|
10303
|
+
level: "warn",
|
|
10304
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10305
|
+
conversation_id: conv.id,
|
|
10306
|
+
message_id: inFlight.evidentMessageId
|
|
10307
|
+
});
|
|
10308
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10309
|
+
return;
|
|
10310
|
+
}
|
|
10311
|
+
this.log({
|
|
10312
|
+
level: "warn",
|
|
10313
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10314
|
+
conversation_id: conv.id,
|
|
10315
|
+
message_id: inFlight.evidentMessageId
|
|
10316
|
+
});
|
|
10317
|
+
return;
|
|
9361
10318
|
}
|
|
10319
|
+
inFlight.done = true;
|
|
10320
|
+
await this.reportSubagentAuthFailures(
|
|
10321
|
+
watcher.conv.id,
|
|
10322
|
+
inFlight.opencodeMessageId,
|
|
10323
|
+
inFlight.evidentMessageId,
|
|
10324
|
+
messages
|
|
10325
|
+
);
|
|
9362
10326
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9363
10327
|
}
|
|
9364
10328
|
/**
|
|
@@ -9416,6 +10380,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9416
10380
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9417
10381
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9418
10382
|
}
|
|
10383
|
+
/**
|
|
10384
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10385
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10386
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10387
|
+
*/
|
|
10388
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10389
|
+
const conv = watcher.conv;
|
|
10390
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10391
|
+
this.log({
|
|
10392
|
+
level: "error",
|
|
10393
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10394
|
+
conversation_id: conv.id,
|
|
10395
|
+
message_id: inFlight.evidentMessageId
|
|
10396
|
+
});
|
|
10397
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10398
|
+
try {
|
|
10399
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10400
|
+
} catch (err) {
|
|
10401
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10402
|
+
if (err instanceof ChannelTerminalError) {
|
|
10403
|
+
this.log({
|
|
10404
|
+
level: "warn",
|
|
10405
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10406
|
+
conversation_id: conv.id,
|
|
10407
|
+
message_id: inFlight.evidentMessageId
|
|
10408
|
+
});
|
|
10409
|
+
this.recordReleasedOpencodeId(
|
|
10410
|
+
inFlight.evidentMessageId,
|
|
10411
|
+
sessionId,
|
|
10412
|
+
inFlight.opencodeMessageId
|
|
10413
|
+
);
|
|
10414
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10415
|
+
return;
|
|
10416
|
+
}
|
|
10417
|
+
if (this.now() >= inFlight.deadline) {
|
|
10418
|
+
this.log({
|
|
10419
|
+
level: "warn",
|
|
10420
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10421
|
+
conversation_id: conv.id,
|
|
10422
|
+
message_id: inFlight.evidentMessageId
|
|
10423
|
+
});
|
|
10424
|
+
this.recordReleasedOpencodeId(
|
|
10425
|
+
inFlight.evidentMessageId,
|
|
10426
|
+
sessionId,
|
|
10427
|
+
inFlight.opencodeMessageId
|
|
10428
|
+
);
|
|
10429
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10430
|
+
return;
|
|
10431
|
+
}
|
|
10432
|
+
this.log({
|
|
10433
|
+
level: "warn",
|
|
10434
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10435
|
+
conversation_id: conv.id,
|
|
10436
|
+
message_id: inFlight.evidentMessageId
|
|
10437
|
+
});
|
|
10438
|
+
return;
|
|
10439
|
+
}
|
|
10440
|
+
inFlight.done = true;
|
|
10441
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10442
|
+
}
|
|
9419
10443
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9420
10444
|
/**
|
|
9421
10445
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9454,11 +10478,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9454
10478
|
}
|
|
9455
10479
|
}
|
|
9456
10480
|
}
|
|
9457
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10481
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9458
10482
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10483
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10484
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10485
|
+
this.awaitingCorrelation.delete(id);
|
|
10486
|
+
this.log({
|
|
10487
|
+
level: "debug",
|
|
10488
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10489
|
+
message_id: id
|
|
10490
|
+
});
|
|
10491
|
+
}
|
|
10492
|
+
}
|
|
9459
10493
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9460
10494
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9461
10495
|
this.untrackableAck.delete(id);
|
|
10496
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9462
10497
|
this.untrackableAckSignalled.delete(id);
|
|
9463
10498
|
this.untrackableAckWarned.delete(id);
|
|
9464
10499
|
this.log({
|
|
@@ -9498,7 +10533,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9498
10533
|
bySession.set(row.opencode_session_id, list);
|
|
9499
10534
|
}
|
|
9500
10535
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9501
|
-
let messages;
|
|
10536
|
+
let messages = null;
|
|
9502
10537
|
try {
|
|
9503
10538
|
if (this.isV2) {
|
|
9504
10539
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9507,9 +10542,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9507
10542
|
level: "warn",
|
|
9508
10543
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9509
10544
|
});
|
|
9510
|
-
|
|
10545
|
+
} else {
|
|
10546
|
+
messages = snapshot;
|
|
9511
10547
|
}
|
|
9512
|
-
messages = snapshot;
|
|
9513
10548
|
} else {
|
|
9514
10549
|
const polled = await pollSessionMessagesForRedrive(
|
|
9515
10550
|
this.port,
|
|
@@ -9522,15 +10557,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9522
10557
|
level: "warn",
|
|
9523
10558
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9524
10559
|
});
|
|
9525
|
-
|
|
10560
|
+
} else {
|
|
10561
|
+
messages = polled.messages;
|
|
9526
10562
|
}
|
|
9527
|
-
messages = polled.messages;
|
|
9528
10563
|
}
|
|
9529
10564
|
} catch (err) {
|
|
9530
10565
|
this.log({
|
|
9531
10566
|
level: "warn",
|
|
9532
10567
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9533
10568
|
});
|
|
10569
|
+
}
|
|
10570
|
+
if (messages === null) {
|
|
10571
|
+
for (const row of sessionRows) {
|
|
10572
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10573
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10574
|
+
}
|
|
10575
|
+
}
|
|
9534
10576
|
continue;
|
|
9535
10577
|
}
|
|
9536
10578
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9540,6 +10582,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9540
10582
|
}
|
|
9541
10583
|
}
|
|
9542
10584
|
}
|
|
10585
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10586
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10587
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10588
|
+
if (!correlation) return false;
|
|
10589
|
+
correlation.attempts += 1;
|
|
10590
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10591
|
+
if (correlatedId !== null) {
|
|
10592
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10593
|
+
const conv = this.convForRow(sessionId, row);
|
|
10594
|
+
const message = this.queuedMessageForRow(row);
|
|
10595
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10596
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10597
|
+
this.dispatched.add(row.id);
|
|
10598
|
+
this.readopted.add(row.id);
|
|
10599
|
+
this.ensureWatcherRunning(sessionId);
|
|
10600
|
+
this.log({
|
|
10601
|
+
level: "info",
|
|
10602
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10603
|
+
conversation_id: row.conversation_id,
|
|
10604
|
+
message_id: row.id
|
|
10605
|
+
});
|
|
10606
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10607
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10608
|
+
opencode_message_id: correlatedId
|
|
10609
|
+
});
|
|
10610
|
+
return true;
|
|
10611
|
+
}
|
|
10612
|
+
if (correlation.knownUserIds === null) {
|
|
10613
|
+
this.log({
|
|
10614
|
+
level: "debug",
|
|
10615
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10616
|
+
conversation_id: row.conversation_id,
|
|
10617
|
+
message_id: row.id
|
|
10618
|
+
});
|
|
10619
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10620
|
+
this.log({
|
|
10621
|
+
level: "warn",
|
|
10622
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10623
|
+
conversation_id: row.conversation_id,
|
|
10624
|
+
message_id: row.id
|
|
10625
|
+
});
|
|
10626
|
+
}
|
|
10627
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10628
|
+
await this.handoffUntrackableCorrelation(
|
|
10629
|
+
row.conversation_id,
|
|
10630
|
+
row.id,
|
|
10631
|
+
correlation.sessionId,
|
|
10632
|
+
correlation.attempts,
|
|
10633
|
+
"readopt"
|
|
10634
|
+
);
|
|
10635
|
+
}
|
|
10636
|
+
return true;
|
|
10637
|
+
}
|
|
9543
10638
|
/**
|
|
9544
10639
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9545
10640
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9547,8 +10642,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9547
10642
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9548
10643
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9549
10644
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9550
|
-
* before the restart)
|
|
9551
|
-
*
|
|
10645
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10646
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10647
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10648
|
+
* `forceReadoptRun`):
|
|
9552
10649
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9553
10650
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9554
10651
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9557,7 +10654,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9557
10654
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9558
10655
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9559
10656
|
* tracking the stored id so the reply correlates by it;
|
|
9560
|
-
* - `unknown`/null id →
|
|
10657
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10658
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9561
10659
|
*
|
|
9562
10660
|
* Only `ChannelAuthError` propagates.
|
|
9563
10661
|
*/
|
|
@@ -9571,6 +10669,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9571
10669
|
});
|
|
9572
10670
|
return;
|
|
9573
10671
|
}
|
|
10672
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9574
10673
|
const ocId = row.opencode_message_id;
|
|
9575
10674
|
const state = messageRunState(messages, ocId ?? "");
|
|
9576
10675
|
if (state === "done") {
|
|
@@ -9637,12 +10736,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9637
10736
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9638
10737
|
this.dontRedispatch.delete(row.id);
|
|
9639
10738
|
this.untrackableAck.delete(row.id);
|
|
10739
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9640
10740
|
this.untrackableAckSignalled.delete(row.id);
|
|
9641
10741
|
this.untrackableAckWarned.delete(row.id);
|
|
9642
10742
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9643
10743
|
return;
|
|
9644
10744
|
}
|
|
9645
|
-
if (this.
|
|
10745
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10746
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10747
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10748
|
+
this.untrackableAckWarned.add(row.id);
|
|
10749
|
+
this.log({
|
|
10750
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10751
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10752
|
+
conversation_id: row.conversation_id,
|
|
10753
|
+
message_id: row.id
|
|
10754
|
+
});
|
|
10755
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10756
|
+
const reportError = await this.reportUntrackableAck(
|
|
10757
|
+
row.conversation_id,
|
|
10758
|
+
row.id,
|
|
10759
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10760
|
+
errorMessage3
|
|
10761
|
+
);
|
|
10762
|
+
if (reportError !== null) {
|
|
10763
|
+
this.log({
|
|
10764
|
+
level: "warn",
|
|
10765
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10766
|
+
conversation_id: row.conversation_id,
|
|
10767
|
+
message_id: row.id
|
|
10768
|
+
});
|
|
10769
|
+
}
|
|
10770
|
+
} else {
|
|
10771
|
+
this.log({
|
|
10772
|
+
level: "debug",
|
|
10773
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10774
|
+
conversation_id: row.conversation_id,
|
|
10775
|
+
message_id: row.id
|
|
10776
|
+
});
|
|
10777
|
+
}
|
|
10778
|
+
return;
|
|
10779
|
+
}
|
|
10780
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9646
10781
|
this.log({
|
|
9647
10782
|
level: "debug",
|
|
9648
10783
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9820,6 +10955,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9820
10955
|
}
|
|
9821
10956
|
this.dontRedispatch.delete(row.id);
|
|
9822
10957
|
this.untrackableAck.delete(row.id);
|
|
10958
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9823
10959
|
this.untrackableAckSignalled.delete(row.id);
|
|
9824
10960
|
this.untrackableAckWarned.delete(row.id);
|
|
9825
10961
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9839,8 +10975,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9839
10975
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9840
10976
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9841
10977
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9842
|
-
* re-entry)
|
|
9843
|
-
*
|
|
10978
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10979
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10980
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9844
10981
|
*
|
|
9845
10982
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9846
10983
|
* `processed_at` (Invariant 1).
|
|
@@ -9903,6 +11040,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9903
11040
|
);
|
|
9904
11041
|
}
|
|
9905
11042
|
let ocId;
|
|
11043
|
+
let preDispatchUserIds = null;
|
|
9906
11044
|
try {
|
|
9907
11045
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9908
11046
|
sessionId,
|
|
@@ -9912,7 +11050,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9912
11050
|
row.content,
|
|
9913
11051
|
options,
|
|
9914
11052
|
sendAttachments,
|
|
9915
|
-
this.openCodeClient
|
|
11053
|
+
this.openCodeClient,
|
|
11054
|
+
{
|
|
11055
|
+
onKnownUserIds: (ids) => {
|
|
11056
|
+
preDispatchUserIds = ids;
|
|
11057
|
+
},
|
|
11058
|
+
onKnownUserIdsError: (error2) => {
|
|
11059
|
+
this.log({
|
|
11060
|
+
level: "warn",
|
|
11061
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11062
|
+
conversation_id: row.conversation_id,
|
|
11063
|
+
message_id: row.id
|
|
11064
|
+
});
|
|
11065
|
+
}
|
|
11066
|
+
}
|
|
9916
11067
|
)
|
|
9917
11068
|
);
|
|
9918
11069
|
} catch (err) {
|
|
@@ -9945,6 +11096,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9945
11096
|
}
|
|
9946
11097
|
} else {
|
|
9947
11098
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11099
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
9948
11100
|
}
|
|
9949
11101
|
this.log({
|
|
9950
11102
|
level: "error",
|
|
@@ -9993,39 +11145,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9993
11145
|
}
|
|
9994
11146
|
if (ocId === null) {
|
|
9995
11147
|
this.awaitingReadopt.delete(row.id);
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
this.log({
|
|
10003
|
-
level: "error",
|
|
10004
|
-
message: errorMessage3,
|
|
10005
|
-
conversation_id: row.conversation_id,
|
|
10006
|
-
message_id: row.id
|
|
10007
|
-
});
|
|
10008
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10009
|
-
this.log({
|
|
10010
|
-
level: "warn",
|
|
10011
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10012
|
-
conversation_id: row.conversation_id,
|
|
10013
|
-
message_id: row.id
|
|
10014
|
-
});
|
|
10015
|
-
});
|
|
10016
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10017
|
-
return;
|
|
10018
|
-
}
|
|
11148
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11149
|
+
sessionId,
|
|
11150
|
+
content: row.content,
|
|
11151
|
+
attempts: 0,
|
|
11152
|
+
knownUserIds: preDispatchUserIds
|
|
11153
|
+
});
|
|
10019
11154
|
this.log({
|
|
10020
11155
|
level: "warn",
|
|
10021
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11156
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10022
11157
|
conversation_id: row.conversation_id,
|
|
10023
11158
|
message_id: row.id
|
|
10024
11159
|
});
|
|
10025
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10026
11160
|
return;
|
|
10027
11161
|
}
|
|
10028
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10029
11162
|
const processedAtMs = this.processedAtMs(row);
|
|
10030
11163
|
this.registerReadopted(
|
|
10031
11164
|
readoptConv,
|
|
@@ -12147,6 +13280,7 @@ function createCredentialSync({
|
|
|
12147
13280
|
|
|
12148
13281
|
// src/commands/run.ts
|
|
12149
13282
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
13283
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12150
13284
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12151
13285
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12152
13286
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12365,7 +13499,9 @@ function displayStatus(state) {
|
|
|
12365
13499
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12366
13500
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12367
13501
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12368
|
-
const detail = last ? chalk7.dim(
|
|
13502
|
+
const detail = last ? chalk7.dim(
|
|
13503
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13504
|
+
) : "";
|
|
12369
13505
|
const agent = state.agentName ?? state.agentId;
|
|
12370
13506
|
console.log(
|
|
12371
13507
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12384,7 +13520,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12384
13520
|
logActivity(state, {
|
|
12385
13521
|
type: "info",
|
|
12386
13522
|
level: "warn",
|
|
12387
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13523
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12388
13524
|
});
|
|
12389
13525
|
}
|
|
12390
13526
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12617,9 +13753,9 @@ async function driveChannels(state, driver) {
|
|
|
12617
13753
|
}
|
|
12618
13754
|
}
|
|
12619
13755
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12620
|
-
var
|
|
13756
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12621
13757
|
function shouldWarnForReclaimSkip(reason) {
|
|
12622
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13758
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12623
13759
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12624
13760
|
if (!version2) return false;
|
|
12625
13761
|
const major = Number(version2[1]);
|
|
@@ -12676,61 +13812,96 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12676
13812
|
async function runSweep(state, driver, config) {
|
|
12677
13813
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12678
13814
|
try {
|
|
12679
|
-
const
|
|
12680
|
-
|
|
12681
|
-
|
|
13815
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13816
|
+
const sessions = enumeration.sessions;
|
|
13817
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13818
|
+
if (enumerationReason !== null) {
|
|
13819
|
+
logActivity(state, {
|
|
12682
13820
|
type: "info",
|
|
12683
|
-
|
|
12684
|
-
|
|
12685
|
-
outcome: "sweep-skipped",
|
|
12686
|
-
reason: "opencode-unreachable"
|
|
12687
|
-
}
|
|
13821
|
+
level: "warn",
|
|
13822
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12688
13823
|
});
|
|
12689
|
-
return;
|
|
12690
13824
|
}
|
|
12691
|
-
const toDelete = selectSessionsToDelete(
|
|
12692
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12693
|
-
{
|
|
12694
|
-
maxAgeMs: config.maxAgeMs,
|
|
12695
|
-
maxCount: config.maxCount,
|
|
12696
|
-
nowMs: Date.now(),
|
|
12697
|
-
protectedIds: driver.protectedSessionIds()
|
|
12698
|
-
}
|
|
12699
|
-
);
|
|
12700
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12701
13825
|
let deleted = 0;
|
|
12702
13826
|
let failed = 0;
|
|
13827
|
+
let failed4xx = 0;
|
|
13828
|
+
let failed5xx = 0;
|
|
13829
|
+
let failedOtherStatus = 0;
|
|
13830
|
+
let failedThrew = 0;
|
|
12703
13831
|
let skippedNewlyActive = 0;
|
|
12704
|
-
|
|
12705
|
-
|
|
12706
|
-
|
|
12707
|
-
|
|
12708
|
-
|
|
12709
|
-
|
|
12710
|
-
|
|
12711
|
-
|
|
13832
|
+
let remaining = 0;
|
|
13833
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13834
|
+
if (sessions !== null) {
|
|
13835
|
+
const toDelete = selectSessionsToDelete(
|
|
13836
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13837
|
+
{
|
|
13838
|
+
maxAgeMs: config.maxAgeMs,
|
|
13839
|
+
maxCount: config.maxCount,
|
|
13840
|
+
nowMs: Date.now(),
|
|
13841
|
+
protectedIds: protectedNow
|
|
13842
|
+
}
|
|
13843
|
+
);
|
|
13844
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13845
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13846
|
+
protectedNow = driver.protectedSessionIds();
|
|
13847
|
+
for (const id of deleteCandidates) {
|
|
13848
|
+
if (protectedNow.has(id)) {
|
|
13849
|
+
skippedNewlyActive++;
|
|
13850
|
+
logActivity(state, {
|
|
13851
|
+
type: "info",
|
|
13852
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13853
|
+
});
|
|
13854
|
+
continue;
|
|
13855
|
+
}
|
|
13856
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13857
|
+
if (outcome.ok) {
|
|
13858
|
+
deleted++;
|
|
13859
|
+
} else if (outcome.status === "threw") {
|
|
13860
|
+
failed++;
|
|
13861
|
+
failedThrew++;
|
|
13862
|
+
} else {
|
|
13863
|
+
failed++;
|
|
13864
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13865
|
+
if (statusClass === 4) failed4xx++;
|
|
13866
|
+
else if (statusClass === 5) failed5xx++;
|
|
13867
|
+
else failedOtherStatus++;
|
|
13868
|
+
}
|
|
12712
13869
|
}
|
|
12713
|
-
|
|
12714
|
-
|
|
13870
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13871
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13872
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13873
|
+
logActivity(state, {
|
|
13874
|
+
type: "info",
|
|
13875
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13876
|
+
});
|
|
12715
13877
|
}
|
|
12716
|
-
const
|
|
12717
|
-
|
|
12718
|
-
|
|
12719
|
-
|
|
12720
|
-
|
|
12721
|
-
|
|
13878
|
+
const sessionMetadata = {
|
|
13879
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13880
|
+
sessions_deleted: deleted,
|
|
13881
|
+
sessions_failed: failed,
|
|
13882
|
+
sessions_failed_4xx: failed4xx,
|
|
13883
|
+
sessions_failed_5xx: failed5xx,
|
|
13884
|
+
sessions_failed_other: failedOtherStatus,
|
|
13885
|
+
sessions_failed_threw: failedThrew,
|
|
13886
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13887
|
+
sessions_remaining: remaining,
|
|
13888
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13889
|
+
};
|
|
12722
13890
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12723
13891
|
dbPath: sessionDbPath(),
|
|
12724
|
-
|
|
13892
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13893
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13894
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12725
13895
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12726
13896
|
});
|
|
12727
13897
|
if (reclaimResult.ok) {
|
|
13898
|
+
state.sessionStoreHealth.observeWritable();
|
|
12728
13899
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12729
13900
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12730
13901
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12731
13902
|
logSessionCleanupActivity(state, "sweep", {
|
|
12732
13903
|
type: "info",
|
|
12733
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13904
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12734
13905
|
metadata: {
|
|
12735
13906
|
outcome: "reclaimed",
|
|
12736
13907
|
mode: reclaimResult.mode,
|
|
@@ -12738,25 +13909,27 @@ async function runSweep(state, driver, config) {
|
|
|
12738
13909
|
after_bytes: reclaimResult.afterBytes,
|
|
12739
13910
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12740
13911
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12741
|
-
|
|
12742
|
-
|
|
12743
|
-
|
|
12744
|
-
|
|
13912
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13913
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13914
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13915
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13916
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13917
|
+
...sessionMetadata
|
|
12745
13918
|
}
|
|
12746
13919
|
});
|
|
12747
13920
|
} else {
|
|
13921
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13922
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13923
|
+
}
|
|
12748
13924
|
logSessionCleanupActivity(state, "sweep", {
|
|
12749
13925
|
type: "info",
|
|
12750
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13926
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12751
13927
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12752
13928
|
metadata: {
|
|
12753
13929
|
outcome: "reclaim-skipped",
|
|
12754
13930
|
reason: reclaimResult.skipped,
|
|
12755
13931
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12756
|
-
|
|
12757
|
-
sessions_deleted: deleted,
|
|
12758
|
-
sessions_failed: failed,
|
|
12759
|
-
sessions_skipped_active: skippedNewlyActive
|
|
13932
|
+
...sessionMetadata
|
|
12760
13933
|
}
|
|
12761
13934
|
});
|
|
12762
13935
|
}
|
|
@@ -13013,6 +14186,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13013
14186
|
message: `Resource usage collection: ${warning2}`
|
|
13014
14187
|
});
|
|
13015
14188
|
}
|
|
14189
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14190
|
+
...usage,
|
|
14191
|
+
nowMs: Date.now(),
|
|
14192
|
+
state: state.lowDiskWarningState
|
|
14193
|
+
});
|
|
14194
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14195
|
+
if (lowDiskDecision.warning !== null) {
|
|
14196
|
+
logActivity(state, {
|
|
14197
|
+
type: "info",
|
|
14198
|
+
level: "warn",
|
|
14199
|
+
message: lowDiskDecision.warning
|
|
14200
|
+
});
|
|
14201
|
+
}
|
|
13016
14202
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13017
14203
|
if (result.ok) {
|
|
13018
14204
|
if (consecutiveFailures > 0) {
|
|
@@ -13259,6 +14445,7 @@ async function run(options) {
|
|
|
13259
14445
|
opencodeClient: null,
|
|
13260
14446
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13261
14447
|
sessionDbProvenanceAnomaly: false,
|
|
14448
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13262
14449
|
opencodeProcess: null,
|
|
13263
14450
|
stopOpenCodeLogTail: null,
|
|
13264
14451
|
litestreamProcess: null,
|
|
@@ -13276,10 +14463,12 @@ async function run(options) {
|
|
|
13276
14463
|
openaiUsageRearm: null,
|
|
13277
14464
|
resourceUsageTimer: null,
|
|
13278
14465
|
stopResourceUsageSampling: null,
|
|
14466
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13279
14467
|
credentialSync: null,
|
|
13280
14468
|
authHeader: ""
|
|
13281
14469
|
};
|
|
13282
14470
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14471
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13283
14472
|
if (options.credentialSyncMarker) {
|
|
13284
14473
|
state.credentialSync = createCredentialSync({
|
|
13285
14474
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13448,6 +14637,34 @@ async function run(options) {
|
|
|
13448
14637
|
} else {
|
|
13449
14638
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13450
14639
|
}
|
|
14640
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14641
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14642
|
+
log2(state, message, level);
|
|
14643
|
+
if (state.interactive && !state.json) {
|
|
14644
|
+
logActivity(state, { type: "info", level, message });
|
|
14645
|
+
}
|
|
14646
|
+
};
|
|
14647
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14648
|
+
if (vaultResolution.error) {
|
|
14649
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14650
|
+
logCredentialVaultResolution(
|
|
14651
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14652
|
+
"warn"
|
|
14653
|
+
);
|
|
14654
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14655
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14656
|
+
logCredentialVaultResolution(
|
|
14657
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14658
|
+
"info"
|
|
14659
|
+
);
|
|
14660
|
+
} else {
|
|
14661
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14662
|
+
logCredentialVaultResolution(
|
|
14663
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14664
|
+
"debug"
|
|
14665
|
+
);
|
|
14666
|
+
}
|
|
14667
|
+
}
|
|
13451
14668
|
if (options.restoreRunnerCredentials) {
|
|
13452
14669
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13453
14670
|
const credentialContext = {
|
|
@@ -13465,30 +14682,48 @@ async function run(options) {
|
|
|
13465
14682
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13466
14683
|
}
|
|
13467
14684
|
state.credentialSync?.arm();
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
createOpenCodeActivityForwarder(() => ({
|
|
14685
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14686
|
+
() => ({
|
|
13471
14687
|
agentId: state.agentId,
|
|
13472
14688
|
authHeader: state.authHeader
|
|
13473
|
-
})
|
|
13474
|
-
|
|
14689
|
+
}),
|
|
14690
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14691
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14692
|
+
if (cause !== null && sessionId !== null) {
|
|
14693
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14694
|
+
}
|
|
14695
|
+
}
|
|
14696
|
+
);
|
|
14697
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14698
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
14699
|
+
openCodeActivityForwarder
|
|
14700
|
+
);
|
|
14701
|
+
state.stopOpenCodeLogTail = () => {
|
|
14702
|
+
openCodeActivityForwarder.flush();
|
|
14703
|
+
openCodeLogTail.stop();
|
|
14704
|
+
};
|
|
13475
14705
|
let sessionDbVerifyFatal = false;
|
|
14706
|
+
let sessionDbVerifyFatalReason;
|
|
13476
14707
|
if (!options.restoreSessionDb) {
|
|
13477
|
-
|
|
14708
|
+
logActivity(state, {
|
|
14709
|
+
type: "info",
|
|
14710
|
+
level: "warn",
|
|
14711
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14712
|
+
});
|
|
13478
14713
|
} else {
|
|
13479
14714
|
const health = await checkOpenCodeHealth(state.port);
|
|
13480
14715
|
if (health.healthy) {
|
|
13481
|
-
|
|
13482
|
-
|
|
13483
|
-
|
|
13484
|
-
"
|
|
13485
|
-
);
|
|
14716
|
+
logActivity(state, {
|
|
14717
|
+
type: "info",
|
|
14718
|
+
level: "warn",
|
|
14719
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14720
|
+
});
|
|
13486
14721
|
} else if (health.authFailed) {
|
|
13487
|
-
|
|
13488
|
-
|
|
13489
|
-
|
|
13490
|
-
"
|
|
13491
|
-
);
|
|
14722
|
+
logActivity(state, {
|
|
14723
|
+
type: "info",
|
|
14724
|
+
level: "warn",
|
|
14725
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14726
|
+
});
|
|
13492
14727
|
} else {
|
|
13493
14728
|
const result = await restoreAndVerifySessionDb({
|
|
13494
14729
|
dbPath: sessionDbPath(),
|
|
@@ -13505,12 +14740,13 @@ async function run(options) {
|
|
|
13505
14740
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13506
14741
|
});
|
|
13507
14742
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14743
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13508
14744
|
}
|
|
13509
14745
|
}
|
|
13510
14746
|
reportSessionDbRecovery(state);
|
|
13511
14747
|
if (sessionDbVerifyFatal) {
|
|
13512
14748
|
throw new Error(
|
|
13513
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14749
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13514
14750
|
);
|
|
13515
14751
|
}
|
|
13516
14752
|
applyRunnerOpenCodeConfig({
|
|
@@ -13755,6 +14991,7 @@ async function run(options) {
|
|
|
13755
14991
|
getAuthHeader: () => state.authHeader,
|
|
13756
14992
|
conversationFilter: state.conversationFilter,
|
|
13757
14993
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14994
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13758
14995
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13759
14996
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13760
14997
|
fileSyncDirectories,
|
|
@@ -13780,12 +15017,13 @@ async function run(options) {
|
|
|
13780
15017
|
openCodePassword: state.opencodePassword,
|
|
13781
15018
|
isRunning: () => state.running,
|
|
13782
15019
|
events: {
|
|
13783
|
-
onConnected: (agentId, isReconnect) => {
|
|
15020
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13784
15021
|
state.connected = true;
|
|
13785
15022
|
state.agentId = agentId;
|
|
15023
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13786
15024
|
logActivity(state, {
|
|
13787
15025
|
type: "info",
|
|
13788
|
-
message:
|
|
15026
|
+
message: connectionMessage
|
|
13789
15027
|
});
|
|
13790
15028
|
if (options.tunnelReadyFile) {
|
|
13791
15029
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13806,7 +15044,12 @@ async function run(options) {
|
|
|
13806
15044
|
version: state.opencodeVersion,
|
|
13807
15045
|
major: state.opencodeApiVersion,
|
|
13808
15046
|
connected: state.opencodeConnected
|
|
13809
|
-
})
|
|
15047
|
+
}),
|
|
15048
|
+
connection_generation: transition.generation,
|
|
15049
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
15050
|
+
reconnect_attempt: transition.attempt,
|
|
15051
|
+
downtime_ms: transition.downtimeMs
|
|
15052
|
+
} : {}
|
|
13810
15053
|
});
|
|
13811
15054
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13812
15055
|
if (state.interactive) displayStatus(state);
|
|
@@ -13828,13 +15071,18 @@ async function run(options) {
|
|
|
13828
15071
|
if (state.interactive) displayStatus(state);
|
|
13829
15072
|
});
|
|
13830
15073
|
},
|
|
13831
|
-
onDisconnected: (code, reason) => {
|
|
15074
|
+
onDisconnected: (code, reason, transition) => {
|
|
13832
15075
|
state.connected = false;
|
|
15076
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13833
15077
|
logActivity(state, {
|
|
13834
15078
|
type: "info",
|
|
13835
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
15079
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
15080
|
+
});
|
|
15081
|
+
emitAgentDisconnected(state.agentId, {
|
|
15082
|
+
code,
|
|
15083
|
+
reason: safeReason2,
|
|
15084
|
+
connection_generation: transition.generation
|
|
13836
15085
|
});
|
|
13837
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13838
15086
|
if (state.interactive) displayStatus(state);
|
|
13839
15087
|
},
|
|
13840
15088
|
onError: (error2) => {
|
|
@@ -13934,7 +15182,7 @@ async function run(options) {
|
|
|
13934
15182
|
},
|
|
13935
15183
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13936
15184
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13937
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
15185
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13938
15186
|
firstDelayMs: firstReportDelayMs,
|
|
13939
15187
|
nextDelayMs: usageReportDelayMs,
|
|
13940
15188
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13977,7 +15225,7 @@ async function run(options) {
|
|
|
13977
15225
|
}
|
|
13978
15226
|
|
|
13979
15227
|
// src/index.ts
|
|
13980
|
-
var
|
|
15228
|
+
var version = getCliVersion();
|
|
13981
15229
|
var program = new Command();
|
|
13982
15230
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13983
15231
|
"--endpoint <url>",
|
|
@@ -13995,6 +15243,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
13995
15243
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
13996
15244
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
13997
15245
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15246
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
13998
15247
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
13999
15248
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14000
15249
|
"-a, --agent [id]",
|