@evident-ai/cli 3.5.2-dev.e67eb8b → 3.5.2-dev.fbf25b4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1598 -501
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1070,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1158
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1323
|
|
|
1160
1324
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.fbf25b4" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1326
|
function getCliVersion() {
|
|
1163
1327
|
return CLI_VERSION;
|
|
1164
1328
|
}
|
|
@@ -1319,8 +1483,316 @@ var EventTypes = {
|
|
|
1319
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1320
1484
|
};
|
|
1321
1485
|
|
|
1322
|
-
// src/lib/
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1323
1491
|
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
function errorMessage(error2) {
|
|
1522
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1523
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1524
|
+
}
|
|
1525
|
+
function errorMessages(error2) {
|
|
1526
|
+
const messages = [];
|
|
1527
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1528
|
+
let current = error2;
|
|
1529
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1530
|
+
visited.add(current);
|
|
1531
|
+
if (typeof current === "string") {
|
|
1532
|
+
messages.push(current);
|
|
1533
|
+
break;
|
|
1534
|
+
}
|
|
1535
|
+
if (current instanceof Error) {
|
|
1536
|
+
messages.push(current.name, current.message);
|
|
1537
|
+
current = current.cause;
|
|
1538
|
+
continue;
|
|
1539
|
+
}
|
|
1540
|
+
if (typeof current === "object") {
|
|
1541
|
+
const record = current;
|
|
1542
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1543
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1544
|
+
current = record.cause;
|
|
1545
|
+
continue;
|
|
1546
|
+
}
|
|
1547
|
+
break;
|
|
1548
|
+
}
|
|
1549
|
+
return messages;
|
|
1550
|
+
}
|
|
1551
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1552
|
+
return errorMessages(error2).some(
|
|
1553
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1554
|
+
message
|
|
1555
|
+
)
|
|
1556
|
+
);
|
|
1557
|
+
}
|
|
1558
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1559
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1560
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1561
|
+
function isCauseBearingText(text) {
|
|
1562
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1563
|
+
}
|
|
1564
|
+
function structuredLogFields(line) {
|
|
1565
|
+
const fields = [];
|
|
1566
|
+
for (let index = 0; index < line.length; index++) {
|
|
1567
|
+
if (line[index] === '"') {
|
|
1568
|
+
index++;
|
|
1569
|
+
while (index < line.length) {
|
|
1570
|
+
if (line[index] === "\\") {
|
|
1571
|
+
index++;
|
|
1572
|
+
} else if (line[index] === '"') {
|
|
1573
|
+
break;
|
|
1574
|
+
}
|
|
1575
|
+
index++;
|
|
1576
|
+
}
|
|
1577
|
+
continue;
|
|
1578
|
+
}
|
|
1579
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1580
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1581
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1582
|
+
const valueStart = index + key.length;
|
|
1583
|
+
if (line[valueStart] !== '"') {
|
|
1584
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1585
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1586
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
let valueEnd = valueStart + 1;
|
|
1590
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1591
|
+
if (line[valueEnd] === "\\") {
|
|
1592
|
+
valueEnd++;
|
|
1593
|
+
} else if (line[valueEnd] === '"') {
|
|
1594
|
+
break;
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1598
|
+
index = valueEnd;
|
|
1599
|
+
}
|
|
1600
|
+
return fields;
|
|
1601
|
+
}
|
|
1602
|
+
function structuredCauseMessages(text) {
|
|
1603
|
+
const fields = structuredLogFields(text);
|
|
1604
|
+
if (fields.length > 0) return fields;
|
|
1605
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1606
|
+
}
|
|
1607
|
+
function causeTokenFromText(text) {
|
|
1608
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1609
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1610
|
+
const match = pattern.exec(text);
|
|
1611
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1612
|
+
return match[1];
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
return null;
|
|
1616
|
+
}
|
|
1617
|
+
function sessionStoreCauseToken(value) {
|
|
1618
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1619
|
+
const token = causeTokenFromText(message);
|
|
1620
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1621
|
+
}
|
|
1622
|
+
return null;
|
|
1623
|
+
}
|
|
1624
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1625
|
+
try {
|
|
1626
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1627
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1628
|
+
if (availableBytes < requiredBytes) {
|
|
1629
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1630
|
+
}
|
|
1631
|
+
return null;
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
function readLogicalBytes(db) {
|
|
1637
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1638
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1639
|
+
return pageCount * pageSize;
|
|
1640
|
+
}
|
|
1641
|
+
function readFreelistCount(db) {
|
|
1642
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1643
|
+
}
|
|
1644
|
+
function readCheckpointResult(db) {
|
|
1645
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1646
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1647
|
+
}
|
|
1648
|
+
async function probeReclaimAvailability(input) {
|
|
1649
|
+
const { dbPath, requiredBytes } = input;
|
|
1650
|
+
let sqlite;
|
|
1651
|
+
try {
|
|
1652
|
+
sqlite = await import("node:sqlite");
|
|
1653
|
+
} catch (err) {
|
|
1654
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1655
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1656
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1657
|
+
}
|
|
1658
|
+
let autoVacuum = null;
|
|
1659
|
+
try {
|
|
1660
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1661
|
+
try {
|
|
1662
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1663
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1664
|
+
} finally {
|
|
1665
|
+
db.close();
|
|
1666
|
+
}
|
|
1667
|
+
} catch (err) {
|
|
1668
|
+
console.warn(
|
|
1669
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
if (autoVacuum !== 0) return null;
|
|
1673
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1674
|
+
}
|
|
1675
|
+
async function reclaimSessionDbSpace(input) {
|
|
1676
|
+
const {
|
|
1677
|
+
dbPath,
|
|
1678
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1679
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1680
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1681
|
+
now = () => performance.now(),
|
|
1682
|
+
allowFullVacuum = true
|
|
1683
|
+
} = input;
|
|
1684
|
+
let sqlite;
|
|
1685
|
+
try {
|
|
1686
|
+
sqlite = await import("node:sqlite");
|
|
1687
|
+
} catch (err) {
|
|
1688
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1689
|
+
console.warn(
|
|
1690
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1691
|
+
);
|
|
1692
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1693
|
+
}
|
|
1694
|
+
const { DatabaseSync } = sqlite;
|
|
1695
|
+
let db;
|
|
1696
|
+
try {
|
|
1697
|
+
db = new DatabaseSync(dbPath);
|
|
1698
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1699
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1700
|
+
if (autoVacuum === 0) {
|
|
1701
|
+
if (!allowFullVacuum) {
|
|
1702
|
+
console.warn(
|
|
1703
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1704
|
+
);
|
|
1705
|
+
return {
|
|
1706
|
+
ok: false,
|
|
1707
|
+
skipped: "full-vacuum-blocked",
|
|
1708
|
+
detail: "a session turn is live"
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1712
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1713
|
+
if (skipReason !== null) {
|
|
1714
|
+
console.warn(
|
|
1715
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1716
|
+
);
|
|
1717
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1718
|
+
}
|
|
1719
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1720
|
+
const freelistBefore = readFreelistCount(db);
|
|
1721
|
+
const startedAt = now();
|
|
1722
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1723
|
+
db.exec("VACUUM");
|
|
1724
|
+
const afterBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistAfter = readFreelistCount(db);
|
|
1726
|
+
const checkpoint = readCheckpointResult(db);
|
|
1727
|
+
return {
|
|
1728
|
+
ok: true,
|
|
1729
|
+
mode: "convert",
|
|
1730
|
+
beforeBytes,
|
|
1731
|
+
afterBytes,
|
|
1732
|
+
freelistBefore,
|
|
1733
|
+
freelistAfter,
|
|
1734
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1735
|
+
stoppedBy: "freelist-drained",
|
|
1736
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1737
|
+
checkpoint
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
if (autoVacuum === 2) {
|
|
1741
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1742
|
+
const freelistBefore = readFreelistCount(db);
|
|
1743
|
+
const startedAt = now();
|
|
1744
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1745
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1746
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1747
|
+
let pagesAttempted = 0;
|
|
1748
|
+
let freelistAfter = freelistBefore;
|
|
1749
|
+
let stoppedBy = "freelist-drained";
|
|
1750
|
+
while (freelistAfter > 0) {
|
|
1751
|
+
if (pagesAttempted >= pageLimit) {
|
|
1752
|
+
stoppedBy = "page-ceiling";
|
|
1753
|
+
break;
|
|
1754
|
+
}
|
|
1755
|
+
if (now() - startedAt >= budget) {
|
|
1756
|
+
stoppedBy = "time-budget";
|
|
1757
|
+
break;
|
|
1758
|
+
}
|
|
1759
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1760
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1761
|
+
pagesAttempted += pages;
|
|
1762
|
+
freelistAfter = readFreelistCount(db);
|
|
1763
|
+
}
|
|
1764
|
+
const afterBytes = readLogicalBytes(db);
|
|
1765
|
+
const checkpoint = readCheckpointResult(db);
|
|
1766
|
+
return {
|
|
1767
|
+
ok: true,
|
|
1768
|
+
mode: "incremental",
|
|
1769
|
+
beforeBytes,
|
|
1770
|
+
afterBytes,
|
|
1771
|
+
freelistBefore,
|
|
1772
|
+
freelistAfter,
|
|
1773
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1774
|
+
stoppedBy,
|
|
1775
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1776
|
+
checkpoint
|
|
1777
|
+
};
|
|
1778
|
+
}
|
|
1779
|
+
console.warn(
|
|
1780
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1781
|
+
);
|
|
1782
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1783
|
+
} catch (err) {
|
|
1784
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1785
|
+
return {
|
|
1786
|
+
ok: false,
|
|
1787
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1788
|
+
detail: errorMessage(err)
|
|
1789
|
+
};
|
|
1790
|
+
} finally {
|
|
1791
|
+
db?.close();
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1324
1796
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1325
1797
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1326
1798
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1337,11 +1809,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1337
1809
|
if (!levelMatch) return null;
|
|
1338
1810
|
const level = levelMatch[1].toUpperCase();
|
|
1339
1811
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1340
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1812
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1813
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1341
1814
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1342
1815
|
const parsed = {
|
|
1343
1816
|
level: level === "WARN" ? "warn" : "error",
|
|
1344
|
-
sessionID: sessionMatch?.[1]
|
|
1817
|
+
sessionID: sessionMatch?.[1],
|
|
1818
|
+
messageID: messageIDMatch?.[1]
|
|
1345
1819
|
};
|
|
1346
1820
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1347
1821
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1349,12 +1823,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1349
1823
|
}
|
|
1350
1824
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1351
1825
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1352
|
-
function createOpenCodeActivityForwarder(getContext) {
|
|
1826
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1353
1827
|
let buffer = Buffer.alloc(0);
|
|
1354
1828
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1355
1829
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1356
1830
|
const metadata = {
|
|
1357
1831
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1832
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1358
1833
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1359
1834
|
};
|
|
1360
1835
|
forwardRunnerActivity(
|
|
@@ -1380,9 +1855,24 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1380
1855
|
};
|
|
1381
1856
|
const flushLine = (line) => {
|
|
1382
1857
|
const parsed = parseOpenCodeLogLine(line);
|
|
1858
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1859
|
+
const cause = sessionStoreCauseToken(line);
|
|
1860
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1861
|
+
try {
|
|
1862
|
+
onSessionStoreWriteFailure({
|
|
1863
|
+
isDiskFailure,
|
|
1864
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1865
|
+
cause
|
|
1866
|
+
});
|
|
1867
|
+
} catch (err) {
|
|
1868
|
+
reportTelemetryDiagnostic(
|
|
1869
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1383
1873
|
if (!parsed) return;
|
|
1384
1874
|
const now = Date.now();
|
|
1385
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1875
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1386
1876
|
expireRepeatStates(now, key);
|
|
1387
1877
|
if (key === void 0) {
|
|
1388
1878
|
forwardLine(line, parsed);
|
|
@@ -1428,34 +1918,21 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1428
1918
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1429
1919
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1430
1920
|
var MAX_METADATA_ENTRIES = 20;
|
|
1431
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1432
|
-
function redactHomeDirectory(message) {
|
|
1433
|
-
const home = homedir2();
|
|
1434
|
-
if (!home || home === "/") {
|
|
1435
|
-
return message;
|
|
1436
|
-
}
|
|
1437
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1438
|
-
let result = "";
|
|
1439
|
-
let searchFrom = 0;
|
|
1440
|
-
let homeIndex;
|
|
1441
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1442
|
-
const homeEnd = homeIndex + home.length;
|
|
1443
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1444
|
-
const nextCharacter = message[homeEnd];
|
|
1445
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1446
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1447
|
-
result += atPathBoundary ? "~" : home;
|
|
1448
|
-
searchFrom = homeEnd;
|
|
1449
|
-
}
|
|
1450
|
-
return result + message.slice(searchFrom);
|
|
1451
|
-
}
|
|
1452
|
-
function redactDiagnosticText(message) {
|
|
1453
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1454
|
-
return redactHomeDirectory(redacted);
|
|
1455
|
-
}
|
|
1456
1921
|
function truncate(message) {
|
|
1457
1922
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1458
|
-
|
|
1923
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1924
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1925
|
+
let previousDropped = -1;
|
|
1926
|
+
while (dropped !== previousDropped) {
|
|
1927
|
+
previousDropped = dropped;
|
|
1928
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1929
|
+
dropped = nextDropped;
|
|
1930
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1931
|
+
}
|
|
1932
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1933
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1934
|
+
const tailLength = remaining - headLength;
|
|
1935
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1459
1936
|
}
|
|
1460
1937
|
function sanitiseMetadata(metadata) {
|
|
1461
1938
|
if (!metadata) return {};
|
|
@@ -1665,12 +2142,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1665
2142
|
metadata: withoutContractFields(record),
|
|
1666
2143
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1667
2144
|
};
|
|
1668
|
-
case "session_db_boot_refused":
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
2145
|
+
case "session_db_boot_refused": {
|
|
2146
|
+
const message = (() => {
|
|
2147
|
+
switch (record.reason) {
|
|
2148
|
+
case "local_discard_failed":
|
|
2149
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2150
|
+
case "replica_separation_unproven":
|
|
2151
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2152
|
+
case "verification_timeout_before_separation":
|
|
2153
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2154
|
+
case "verification_timeout_during_separation":
|
|
2155
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2156
|
+
default:
|
|
2157
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2158
|
+
}
|
|
2159
|
+
})();
|
|
2160
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2161
|
+
}
|
|
1674
2162
|
case "schema_provenance_mismatch":
|
|
1675
2163
|
return {
|
|
1676
2164
|
level,
|
|
@@ -1920,7 +2408,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1920
2408
|
return null;
|
|
1921
2409
|
}
|
|
1922
2410
|
}
|
|
1923
|
-
function
|
|
2411
|
+
function roleOf2(m) {
|
|
1924
2412
|
if (!m || typeof m !== "object") return void 0;
|
|
1925
2413
|
if (typeof m.role === "string") return m.role;
|
|
1926
2414
|
const infoRole = m.info?.role;
|
|
@@ -1934,13 +2422,13 @@ function createdOf(m) {
|
|
|
1934
2422
|
if (!m || typeof m !== "object") return void 0;
|
|
1935
2423
|
return m.info?.time?.created ?? m.time?.created;
|
|
1936
2424
|
}
|
|
1937
|
-
function
|
|
2425
|
+
function idOf2(m) {
|
|
1938
2426
|
if (!m || typeof m !== "object") return void 0;
|
|
1939
2427
|
if (typeof m.id === "string") return m.id;
|
|
1940
2428
|
const infoId = m.info?.id;
|
|
1941
2429
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1942
2430
|
}
|
|
1943
|
-
function
|
|
2431
|
+
function parentIdOf2(m) {
|
|
1944
2432
|
if (!m || typeof m !== "object") return void 0;
|
|
1945
2433
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1946
2434
|
const infoParent = m.info?.parentID;
|
|
@@ -1956,8 +2444,12 @@ function errorOf(m) {
|
|
|
1956
2444
|
if (!m || typeof m !== "object") return void 0;
|
|
1957
2445
|
return m.info?.error ?? m.error;
|
|
1958
2446
|
}
|
|
2447
|
+
function isTerminalStopAnswer(m) {
|
|
2448
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2449
|
+
}
|
|
1959
2450
|
function isAssistantInFlight(m) {
|
|
1960
2451
|
if (completedOf(m) == null) return true;
|
|
2452
|
+
if (errorOf(m) != null) return false;
|
|
1961
2453
|
return finishOf(m) === "tool-calls";
|
|
1962
2454
|
}
|
|
1963
2455
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2005,9 +2497,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2005
2497
|
function isSessionActivelyGenerating(messages) {
|
|
2006
2498
|
if (!messages || messages.length === 0) return false;
|
|
2007
2499
|
const last = messages[messages.length - 1];
|
|
2008
|
-
if (
|
|
2500
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2009
2501
|
return completedOf(last) == null;
|
|
2010
2502
|
}
|
|
2503
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2011
2504
|
function sessionLastActivityMs(session) {
|
|
2012
2505
|
const candidates = [
|
|
2013
2506
|
session.time?.updated,
|
|
@@ -2022,30 +2515,60 @@ function sessionLastActivityMs(session) {
|
|
|
2022
2515
|
}
|
|
2023
2516
|
return null;
|
|
2024
2517
|
}
|
|
2025
|
-
|
|
2026
|
-
|
|
2518
|
+
function sessionListPath(options) {
|
|
2519
|
+
const params = new URLSearchParams();
|
|
2520
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2521
|
+
const query = params.toString();
|
|
2522
|
+
return query ? `/session?${query}` : "/session";
|
|
2523
|
+
}
|
|
2524
|
+
async function listSessionPage(port, client, options) {
|
|
2525
|
+
const path = sessionListPath(options);
|
|
2027
2526
|
try {
|
|
2028
|
-
const res = await requestWithClient(port, client,
|
|
2527
|
+
const res = await requestWithClient(port, client, path);
|
|
2029
2528
|
if (!res.ok) return null;
|
|
2030
2529
|
const body = await res.json();
|
|
2031
2530
|
return Array.isArray(body) ? body : null;
|
|
2032
2531
|
} catch (error2) {
|
|
2033
2532
|
console.error(
|
|
2034
|
-
`[
|
|
2533
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2035
2534
|
);
|
|
2036
2535
|
return null;
|
|
2037
2536
|
}
|
|
2038
2537
|
}
|
|
2538
|
+
async function listSessions(port, client, options) {
|
|
2539
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2540
|
+
return listSessionPage(port, client, options);
|
|
2541
|
+
}
|
|
2542
|
+
async function listAllSessions(port, client) {
|
|
2543
|
+
if (client?.version === "v2") {
|
|
2544
|
+
const sessions2 = await listV2Sessions(client);
|
|
2545
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2546
|
+
}
|
|
2547
|
+
const sessions = await listSessionPage(port, client, {
|
|
2548
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2549
|
+
});
|
|
2550
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2551
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2552
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2553
|
+
}
|
|
2554
|
+
return { sessions };
|
|
2555
|
+
}
|
|
2039
2556
|
async function deleteSession(port, id, client) {
|
|
2040
2557
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2041
2558
|
try {
|
|
2042
|
-
const res = await requestWithClient(
|
|
2043
|
-
|
|
2559
|
+
const res = await requestWithClient(
|
|
2560
|
+
port,
|
|
2561
|
+
client,
|
|
2562
|
+
`/session/${id}`,
|
|
2563
|
+
{ method: "DELETE" },
|
|
2564
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2565
|
+
);
|
|
2566
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
2044
2567
|
} catch (error2) {
|
|
2045
2568
|
console.error(
|
|
2046
2569
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2047
2570
|
);
|
|
2048
|
-
return false;
|
|
2571
|
+
return { ok: false, status: "threw" };
|
|
2049
2572
|
}
|
|
2050
2573
|
}
|
|
2051
2574
|
async function sessionExists(port, id, client) {
|
|
@@ -2283,11 +2806,31 @@ function messageText(m) {
|
|
|
2283
2806
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2284
2807
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2285
2808
|
}
|
|
2286
|
-
|
|
2809
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2810
|
+
if (!messages || messages.length === 0) return null;
|
|
2811
|
+
let best = null;
|
|
2812
|
+
for (const message of messages) {
|
|
2813
|
+
if (roleOf2(message) !== "user") continue;
|
|
2814
|
+
const id = idOf2(message);
|
|
2815
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2816
|
+
if (messageText(message) !== content) continue;
|
|
2817
|
+
const created = createdOf(message) ?? 0;
|
|
2818
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2819
|
+
}
|
|
2820
|
+
return best?.id ?? null;
|
|
2821
|
+
}
|
|
2822
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2287
2823
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2288
2824
|
const knownUserIds = new Set(
|
|
2289
|
-
(before ?? []).filter((m) =>
|
|
2825
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2290
2826
|
);
|
|
2827
|
+
if (callbacks) {
|
|
2828
|
+
try {
|
|
2829
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2830
|
+
} catch (error2) {
|
|
2831
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2291
2834
|
const parts = [{ type: "text", text: content }];
|
|
2292
2835
|
let pendingOutcomes = null;
|
|
2293
2836
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2326,22 +2869,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2326
2869
|
const READ_BACK_DELAY_MS = 150;
|
|
2327
2870
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2328
2871
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
const id = idOf(m);
|
|
2334
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2335
|
-
if (messageText(m) !== content) continue;
|
|
2336
|
-
const created = createdOf(m) ?? 0;
|
|
2337
|
-
if (best === null || created > best.created) {
|
|
2338
|
-
best = { id, created };
|
|
2339
|
-
}
|
|
2340
|
-
}
|
|
2341
|
-
if (best) {
|
|
2342
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2343
|
-
return best.id;
|
|
2344
|
-
}
|
|
2872
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2873
|
+
if (dispatchedUserMessageId !== null) {
|
|
2874
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2875
|
+
return dispatchedUserMessageId;
|
|
2345
2876
|
}
|
|
2346
2877
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2347
2878
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2352,57 +2883,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2352
2883
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2353
2884
|
if (!messages || messages.length === 0) return null;
|
|
2354
2885
|
const byParent = messages.find(
|
|
2355
|
-
(m) =>
|
|
2886
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2356
2887
|
);
|
|
2357
2888
|
if (byParent) return byParent;
|
|
2358
|
-
const userIndex = messages.findIndex((m) =>
|
|
2889
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2359
2890
|
if (userIndex === -1) return null;
|
|
2360
2891
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2361
|
-
if (
|
|
2892
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2362
2893
|
}
|
|
2363
2894
|
return null;
|
|
2364
2895
|
}
|
|
2365
2896
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2366
2897
|
if (!messages || messages.length === 0) return null;
|
|
2367
2898
|
let lastCorrelated = null;
|
|
2368
|
-
let lastNonErrored = null;
|
|
2369
2899
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2370
2900
|
const m = messages[i];
|
|
2371
|
-
if (
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2901
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2902
|
+
lastCorrelated = m;
|
|
2903
|
+
break;
|
|
2904
|
+
}
|
|
2905
|
+
if (lastCorrelated) {
|
|
2906
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2907
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2908
|
+
const m = messages[i];
|
|
2909
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2910
|
+
return m;
|
|
2911
|
+
}
|
|
2376
2912
|
}
|
|
2913
|
+
return lastCorrelated;
|
|
2377
2914
|
}
|
|
2378
|
-
|
|
2379
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2915
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2380
2916
|
if (userIndex === -1) return null;
|
|
2381
2917
|
let last = null;
|
|
2382
|
-
let
|
|
2918
|
+
let lastAssistantIndex = -1;
|
|
2383
2919
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2384
|
-
const role =
|
|
2920
|
+
const role = roleOf2(messages[i]);
|
|
2385
2921
|
if (role === "user") break;
|
|
2386
2922
|
if (role === "assistant") {
|
|
2387
2923
|
last = messages[i];
|
|
2388
|
-
|
|
2924
|
+
lastAssistantIndex = i;
|
|
2389
2925
|
}
|
|
2390
2926
|
}
|
|
2391
|
-
|
|
2927
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2928
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2929
|
+
const m = messages[i];
|
|
2930
|
+
if (roleOf2(m) === "user") break;
|
|
2931
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2932
|
+
}
|
|
2933
|
+
return last;
|
|
2392
2934
|
}
|
|
2393
2935
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2394
2936
|
if (!messages || messages.length === 0) return [];
|
|
2395
2937
|
const byParent = messages.filter(
|
|
2396
|
-
(message) =>
|
|
2938
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2397
2939
|
);
|
|
2398
2940
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2399
2941
|
if (assistants.length === 0) {
|
|
2400
|
-
const userIndex = messages.findIndex((message) =>
|
|
2942
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2401
2943
|
if (userIndex === -1) return [];
|
|
2402
2944
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2403
2945
|
const message = messages[i];
|
|
2404
|
-
if (
|
|
2405
|
-
if (
|
|
2946
|
+
if (roleOf2(message) === "user") break;
|
|
2947
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2406
2948
|
}
|
|
2407
2949
|
}
|
|
2408
2950
|
const refs = [];
|
|
@@ -2442,7 +2984,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2442
2984
|
if (!messages || messages.length === 0) return [];
|
|
2443
2985
|
const calls = [];
|
|
2444
2986
|
for (const message of messages) {
|
|
2445
|
-
if (
|
|
2987
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2446
2988
|
for (const part of message.parts ?? []) {
|
|
2447
2989
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2448
2990
|
continue;
|
|
@@ -2472,7 +3014,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2472
3014
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2473
3015
|
const unattributed = [];
|
|
2474
3016
|
for (const message of messages ?? []) {
|
|
2475
|
-
if (
|
|
3017
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2476
3018
|
const created = finiteNumber(createdOf(message));
|
|
2477
3019
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2478
3020
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2494,8 +3036,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2494
3036
|
}
|
|
2495
3037
|
function sumAssistantUsage(messages) {
|
|
2496
3038
|
if (!messages || messages.length === 0) return null;
|
|
2497
|
-
const
|
|
2498
|
-
const
|
|
3039
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3040
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3041
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2499
3042
|
let sawAnyUsage = false;
|
|
2500
3043
|
let inputSum = 0;
|
|
2501
3044
|
let outputSum = 0;
|
|
@@ -2549,13 +3092,11 @@ function sumAssistantUsage(messages) {
|
|
|
2549
3092
|
function messageUsage(messages, userMessageId) {
|
|
2550
3093
|
if (!messages || messages.length === 0) return null;
|
|
2551
3094
|
const byParentAll = messages.filter(
|
|
2552
|
-
(m) =>
|
|
3095
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2553
3096
|
);
|
|
2554
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2555
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2556
3097
|
let correlated;
|
|
2557
|
-
if (
|
|
2558
|
-
correlated =
|
|
3098
|
+
if (byParentAll.length > 0) {
|
|
3099
|
+
correlated = byParentAll;
|
|
2559
3100
|
} else {
|
|
2560
3101
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2561
3102
|
correlated = reply ? [reply] : [];
|
|
@@ -2565,7 +3106,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2565
3106
|
}
|
|
2566
3107
|
function messageRunState(messages, userMessageId) {
|
|
2567
3108
|
if (!messages || messages.length === 0) return "unknown";
|
|
2568
|
-
const hasUser = messages.some((m) =>
|
|
3109
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2569
3110
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2570
3111
|
if (!hasUser) {
|
|
2571
3112
|
if (!reply) return "unknown";
|
|
@@ -2601,14 +3142,38 @@ function messageError(messages, userMessageId) {
|
|
|
2601
3142
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2602
3143
|
const error2 = errorOf(reply);
|
|
2603
3144
|
if (error2 == null) return null;
|
|
2604
|
-
|
|
2605
|
-
if (typeof error2 === "
|
|
3145
|
+
let rendered;
|
|
3146
|
+
if (typeof error2 === "string") {
|
|
3147
|
+
rendered = error2;
|
|
3148
|
+
} else if (typeof error2 === "object") {
|
|
2606
3149
|
const e = error2;
|
|
2607
3150
|
const dataMessage = e.data?.message;
|
|
2608
|
-
if (typeof dataMessage === "string")
|
|
2609
|
-
|
|
3151
|
+
if (typeof dataMessage === "string") {
|
|
3152
|
+
rendered = dataMessage;
|
|
3153
|
+
} else if (typeof e.message === "string") {
|
|
3154
|
+
rendered = e.message;
|
|
3155
|
+
} else {
|
|
3156
|
+
rendered = "The agent run failed.";
|
|
3157
|
+
}
|
|
3158
|
+
} else {
|
|
3159
|
+
rendered = "The agent run failed.";
|
|
2610
3160
|
}
|
|
2611
|
-
|
|
3161
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3162
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3163
|
+
) : void 0;
|
|
3164
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3165
|
+
}
|
|
3166
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3167
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3168
|
+
if (!reply) return null;
|
|
3169
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3170
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3171
|
+
return {
|
|
3172
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3173
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3174
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3175
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3176
|
+
};
|
|
2612
3177
|
}
|
|
2613
3178
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2614
3179
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2657,7 +3222,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2657
3222
|
if (!messages || messages.length === 0) return null;
|
|
2658
3223
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2659
3224
|
const message = messages[i];
|
|
2660
|
-
if (
|
|
3225
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2661
3226
|
const created = createdOf(message);
|
|
2662
3227
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2663
3228
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2689,25 +3254,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2689
3254
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2690
3255
|
if (!messages || messages.length === 0) return false;
|
|
2691
3256
|
return messages.some(
|
|
2692
|
-
(m) =>
|
|
3257
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2693
3258
|
);
|
|
2694
3259
|
}
|
|
2695
3260
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2696
3261
|
if (!messages || messages.length === 0) return false;
|
|
2697
|
-
const userIndex = messages.findIndex((message) =>
|
|
3262
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2698
3263
|
if (userIndex === -1) return false;
|
|
2699
3264
|
let hasLaterUser = false;
|
|
2700
3265
|
let hasStartedLaterUser = false;
|
|
2701
3266
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2702
3267
|
const message = messages[i];
|
|
2703
|
-
if (
|
|
3268
|
+
if (roleOf2(message) !== "user") continue;
|
|
2704
3269
|
hasLaterUser = true;
|
|
2705
|
-
const laterUserMessageId =
|
|
3270
|
+
const laterUserMessageId = idOf2(message);
|
|
2706
3271
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2707
3272
|
return false;
|
|
2708
3273
|
}
|
|
2709
3274
|
if (messages.some(
|
|
2710
|
-
(candidate) =>
|
|
3275
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2711
3276
|
)) {
|
|
2712
3277
|
hasStartedLaterUser = true;
|
|
2713
3278
|
}
|
|
@@ -3178,13 +3743,17 @@ async function listV2Sessions(client) {
|
|
|
3178
3743
|
}
|
|
3179
3744
|
async function deleteV2Session(client, sessionId) {
|
|
3180
3745
|
try {
|
|
3181
|
-
await client.request(
|
|
3182
|
-
|
|
3746
|
+
const response = await client.request(
|
|
3747
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3748
|
+
{ method: "DELETE" },
|
|
3749
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3750
|
+
);
|
|
3751
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3183
3752
|
} catch (error2) {
|
|
3184
3753
|
console.error(
|
|
3185
3754
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3186
3755
|
);
|
|
3187
|
-
return false;
|
|
3756
|
+
return { ok: false, status: "threw" };
|
|
3188
3757
|
}
|
|
3189
3758
|
}
|
|
3190
3759
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3806,9 +4375,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3806
4375
|
|
|
3807
4376
|
// src/lib/opencode/session-db-boot.ts
|
|
3808
4377
|
import { spawn as spawn2 } from "node:child_process";
|
|
3809
|
-
import { mkdirSync, statSync as
|
|
4378
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3810
4379
|
import { homedir as homedir4 } from "node:os";
|
|
3811
|
-
import { dirname as
|
|
4380
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3812
4381
|
|
|
3813
4382
|
// src/lib/runner-synchroniser.ts
|
|
3814
4383
|
import { spawn } from "node:child_process";
|
|
@@ -3888,9 +4457,12 @@ function runSynchroniser(args, opts) {
|
|
|
3888
4457
|
|
|
3889
4458
|
// src/lib/opencode/session-db-boot.ts
|
|
3890
4459
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3891
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4460
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3892
4461
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3893
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4462
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4463
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4464
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4465
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3894
4466
|
function commandError(result) {
|
|
3895
4467
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3896
4468
|
}
|
|
@@ -3933,7 +4505,7 @@ function clearMarker(options) {
|
|
|
3933
4505
|
function markNoReplicate(options, message) {
|
|
3934
4506
|
if (options.noReplicateMarker) {
|
|
3935
4507
|
try {
|
|
3936
|
-
mkdirSync(
|
|
4508
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3937
4509
|
writeFileSync(options.noReplicateMarker, "");
|
|
3938
4510
|
} catch (error2) {
|
|
3939
4511
|
options.log(
|
|
@@ -4058,7 +4630,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4058
4630
|
return null;
|
|
4059
4631
|
}
|
|
4060
4632
|
try {
|
|
4061
|
-
if (
|
|
4633
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4062
4634
|
} catch (error2) {
|
|
4063
4635
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4064
4636
|
options.log(
|
|
@@ -4088,7 +4660,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4088
4660
|
return null;
|
|
4089
4661
|
}
|
|
4090
4662
|
try {
|
|
4091
|
-
mkdirSync(
|
|
4663
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
4092
4664
|
writeFileSync(configPath, rendered.stdout);
|
|
4093
4665
|
} catch (error2) {
|
|
4094
4666
|
options.log(
|
|
@@ -4201,20 +4773,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4201
4773
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4202
4774
|
env: {
|
|
4203
4775
|
...env,
|
|
4204
|
-
// The synchroniser reads
|
|
4205
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4776
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4206
4777
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4207
4778
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4779
|
+
),
|
|
4780
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4781
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4782
|
+
),
|
|
4783
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4784
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4785
|
+
),
|
|
4786
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4787
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4208
4788
|
)
|
|
4209
4789
|
}
|
|
4210
4790
|
});
|
|
4211
4791
|
logSynchroniserDiagnostics(result, options);
|
|
4212
4792
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4793
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4794
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4795
|
+
options.log(
|
|
4796
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4797
|
+
"error"
|
|
4798
|
+
);
|
|
4799
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4800
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4801
|
+
}
|
|
4213
4802
|
options.log(
|
|
4214
4803
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4215
4804
|
"warn"
|
|
4216
4805
|
);
|
|
4217
|
-
return false;
|
|
4806
|
+
return { verifyFatal: false };
|
|
4218
4807
|
}
|
|
4219
4808
|
if (result.code === 34) {
|
|
4220
4809
|
reportRecord(
|
|
@@ -4224,14 +4813,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4224
4813
|
null,
|
|
4225
4814
|
options
|
|
4226
4815
|
);
|
|
4227
|
-
return true;
|
|
4816
|
+
return { verifyFatal: true };
|
|
4228
4817
|
}
|
|
4229
4818
|
if (result.code === 33) {
|
|
4230
4819
|
options.log(
|
|
4231
4820
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4232
4821
|
"warn"
|
|
4233
4822
|
);
|
|
4234
|
-
return false;
|
|
4823
|
+
return { verifyFatal: false };
|
|
4235
4824
|
}
|
|
4236
4825
|
if (result.code !== 0) {
|
|
4237
4826
|
options.log(
|
|
@@ -4239,17 +4828,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4239
4828
|
"warn"
|
|
4240
4829
|
);
|
|
4241
4830
|
}
|
|
4242
|
-
return false;
|
|
4831
|
+
return { verifyFatal: false };
|
|
4243
4832
|
}
|
|
4244
4833
|
options.log(
|
|
4245
4834
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4246
4835
|
"debug"
|
|
4247
4836
|
);
|
|
4248
|
-
return false;
|
|
4837
|
+
return { verifyFatal: false };
|
|
4249
4838
|
}
|
|
4250
4839
|
function fileExists(path) {
|
|
4251
4840
|
try {
|
|
4252
|
-
|
|
4841
|
+
statSync3(path);
|
|
4253
4842
|
return true;
|
|
4254
4843
|
} catch (error2) {
|
|
4255
4844
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4332,13 +4921,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4332
4921
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4333
4922
|
return { verifyFatal: false };
|
|
4334
4923
|
}
|
|
4335
|
-
return
|
|
4924
|
+
return verifySessionDb(options, configPath, env);
|
|
4336
4925
|
}
|
|
4337
4926
|
|
|
4338
4927
|
// src/lib/opencode/session-db-provenance.ts
|
|
4339
4928
|
import { createRequire } from "node:module";
|
|
4340
4929
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4341
|
-
import { dirname as
|
|
4930
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4342
4931
|
var require2 = createRequire(import.meta.url);
|
|
4343
4932
|
function readSessionDbMigrationIds(dbPath) {
|
|
4344
4933
|
let db;
|
|
@@ -4404,7 +4993,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4404
4993
|
}
|
|
4405
4994
|
function saveSessionDbProvenanceState(path, state) {
|
|
4406
4995
|
try {
|
|
4407
|
-
mkdirSync2(
|
|
4996
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4408
4997
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4409
4998
|
`, "utf8");
|
|
4410
4999
|
} catch (error2) {
|
|
@@ -4985,7 +5574,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4985
5574
|
toDelete.push(s.id);
|
|
4986
5575
|
}
|
|
4987
5576
|
}
|
|
4988
|
-
|
|
5577
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5578
|
+
return toDelete.sort(
|
|
5579
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5580
|
+
);
|
|
4989
5581
|
}
|
|
4990
5582
|
var DEFAULT_INTERVAL = "1h";
|
|
4991
5583
|
function resolve(flag, envValue, fallback) {
|
|
@@ -5045,13 +5637,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5045
5637
|
}
|
|
5046
5638
|
|
|
5047
5639
|
// src/lib/opencode/session-db-size.ts
|
|
5048
|
-
import { statSync as
|
|
5640
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5049
5641
|
import { join as join6 } from "node:path";
|
|
5050
5642
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5051
5643
|
function statSessionDbBytes(homeDir) {
|
|
5052
5644
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5053
5645
|
try {
|
|
5054
|
-
return
|
|
5646
|
+
return statSync4(dbPath).size;
|
|
5055
5647
|
} catch (err) {
|
|
5056
5648
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5057
5649
|
if (!isMissingFile) {
|
|
@@ -5077,7 +5669,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5077
5669
|
}
|
|
5078
5670
|
|
|
5079
5671
|
// src/lib/opencode/log-tail.ts
|
|
5080
|
-
import { statSync as
|
|
5672
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5081
5673
|
import { homedir as homedir5 } from "node:os";
|
|
5082
5674
|
import { join as join7 } from "node:path";
|
|
5083
5675
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -5096,190 +5688,67 @@ function reportFailure(operation, logPath, error2) {
|
|
|
5096
5688
|
}
|
|
5097
5689
|
function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
5098
5690
|
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
5099
|
-
let offset = 0;
|
|
5100
|
-
let inode = null;
|
|
5101
|
-
let baselineReady = true;
|
|
5102
|
-
try {
|
|
5103
|
-
const initial = statSync4(logPath);
|
|
5104
|
-
offset = initial.size;
|
|
5105
|
-
inode = initial.ino;
|
|
5106
|
-
} catch (error2) {
|
|
5107
|
-
if (!isEnoent(error2)) {
|
|
5108
|
-
reportFailure("initial stat", logPath, error2);
|
|
5109
|
-
baselineReady = false;
|
|
5110
|
-
}
|
|
5111
|
-
}
|
|
5112
|
-
let polling = false;
|
|
5113
|
-
let stopped = false;
|
|
5114
|
-
const poll = async () => {
|
|
5115
|
-
if (polling || stopped) return;
|
|
5116
|
-
polling = true;
|
|
5117
|
-
try {
|
|
5118
|
-
let current;
|
|
5119
|
-
try {
|
|
5120
|
-
current = await stat(logPath);
|
|
5121
|
-
} catch (error2) {
|
|
5122
|
-
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5123
|
-
return;
|
|
5124
|
-
}
|
|
5125
|
-
if (!baselineReady) {
|
|
5126
|
-
offset = current.size;
|
|
5127
|
-
inode = current.ino;
|
|
5128
|
-
baselineReady = true;
|
|
5129
|
-
return;
|
|
5130
|
-
}
|
|
5131
|
-
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5132
|
-
offset = 0;
|
|
5133
|
-
}
|
|
5134
|
-
inode = current.ino;
|
|
5135
|
-
if (current.size === offset) return;
|
|
5136
|
-
const length = current.size - offset;
|
|
5137
|
-
const fh = await open2(logPath, "r");
|
|
5138
|
-
try {
|
|
5139
|
-
const buf = Buffer.alloc(length);
|
|
5140
|
-
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5141
|
-
offset += bytesRead;
|
|
5142
|
-
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5143
|
-
} finally {
|
|
5144
|
-
await fh.close();
|
|
5145
|
-
}
|
|
5146
|
-
} catch (error2) {
|
|
5147
|
-
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5148
|
-
} finally {
|
|
5149
|
-
polling = false;
|
|
5150
|
-
}
|
|
5151
|
-
};
|
|
5152
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5153
|
-
void poll();
|
|
5154
|
-
return {
|
|
5155
|
-
stop: () => {
|
|
5156
|
-
stopped = true;
|
|
5157
|
-
clearInterval(interval);
|
|
5158
|
-
}
|
|
5159
|
-
};
|
|
5160
|
-
}
|
|
5161
|
-
|
|
5162
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5163
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5164
|
-
import { dirname as dirname4 } from "node:path";
|
|
5165
|
-
function errorMessage(error2) {
|
|
5166
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5167
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5168
|
-
}
|
|
5169
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5170
|
-
try {
|
|
5171
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5172
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5173
|
-
if (availableBytes < requiredBytes) {
|
|
5174
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5175
|
-
}
|
|
5176
|
-
return null;
|
|
5177
|
-
} catch (err) {
|
|
5178
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5179
|
-
}
|
|
5180
|
-
}
|
|
5181
|
-
function readLogicalBytes(db) {
|
|
5182
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5183
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5184
|
-
return pageCount * pageSize;
|
|
5185
|
-
}
|
|
5186
|
-
function readCheckpointResult(db) {
|
|
5187
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5188
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5189
|
-
}
|
|
5190
|
-
async function probeReclaimAvailability(input) {
|
|
5191
|
-
const { dbPath, requiredBytes } = input;
|
|
5192
|
-
let sqlite;
|
|
5193
|
-
try {
|
|
5194
|
-
sqlite = await import("node:sqlite");
|
|
5195
|
-
} catch (err) {
|
|
5196
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5197
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5198
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5199
|
-
}
|
|
5200
|
-
let autoVacuum = null;
|
|
5201
|
-
try {
|
|
5202
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5203
|
-
try {
|
|
5204
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5205
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5206
|
-
} finally {
|
|
5207
|
-
db.close();
|
|
5208
|
-
}
|
|
5209
|
-
} catch (err) {
|
|
5210
|
-
console.warn(
|
|
5211
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5212
|
-
);
|
|
5213
|
-
}
|
|
5214
|
-
if (autoVacuum !== 0) return null;
|
|
5215
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5216
|
-
}
|
|
5217
|
-
async function reclaimSessionDbSpace(input) {
|
|
5218
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5219
|
-
let sqlite;
|
|
5220
|
-
try {
|
|
5221
|
-
sqlite = await import("node:sqlite");
|
|
5222
|
-
} catch (err) {
|
|
5223
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5224
|
-
console.warn(
|
|
5225
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5226
|
-
);
|
|
5227
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5228
|
-
}
|
|
5229
|
-
const { DatabaseSync } = sqlite;
|
|
5230
|
-
let db;
|
|
5691
|
+
let offset = 0;
|
|
5692
|
+
let inode = null;
|
|
5693
|
+
let baselineReady = true;
|
|
5231
5694
|
try {
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
|
|
5239
|
-
|
|
5240
|
-
|
|
5241
|
-
|
|
5242
|
-
|
|
5243
|
-
|
|
5244
|
-
|
|
5695
|
+
const initial = statSync5(logPath);
|
|
5696
|
+
offset = initial.size;
|
|
5697
|
+
inode = initial.ino;
|
|
5698
|
+
} catch (error2) {
|
|
5699
|
+
if (!isEnoent(error2)) {
|
|
5700
|
+
reportFailure("initial stat", logPath, error2);
|
|
5701
|
+
baselineReady = false;
|
|
5702
|
+
}
|
|
5703
|
+
}
|
|
5704
|
+
let polling = false;
|
|
5705
|
+
let stopped = false;
|
|
5706
|
+
const poll = async () => {
|
|
5707
|
+
if (polling || stopped) return;
|
|
5708
|
+
polling = true;
|
|
5709
|
+
try {
|
|
5710
|
+
let current;
|
|
5711
|
+
try {
|
|
5712
|
+
current = await stat(logPath);
|
|
5713
|
+
} catch (error2) {
|
|
5714
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5715
|
+
return;
|
|
5245
5716
|
}
|
|
5246
|
-
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
|
|
5251
|
-
);
|
|
5252
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5717
|
+
if (!baselineReady) {
|
|
5718
|
+
offset = current.size;
|
|
5719
|
+
inode = current.ino;
|
|
5720
|
+
baselineReady = true;
|
|
5721
|
+
return;
|
|
5253
5722
|
}
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5723
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5724
|
+
offset = 0;
|
|
5725
|
+
}
|
|
5726
|
+
inode = current.ino;
|
|
5727
|
+
if (current.size === offset) return;
|
|
5728
|
+
const length = current.size - offset;
|
|
5729
|
+
const fh = await open2(logPath, "r");
|
|
5730
|
+
try {
|
|
5731
|
+
const buf = Buffer.alloc(length);
|
|
5732
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5733
|
+
offset += bytesRead;
|
|
5734
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5735
|
+
} finally {
|
|
5736
|
+
await fh.close();
|
|
5737
|
+
}
|
|
5738
|
+
} catch (error2) {
|
|
5739
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5740
|
+
} finally {
|
|
5741
|
+
polling = false;
|
|
5260
5742
|
}
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5743
|
+
};
|
|
5744
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5745
|
+
void poll();
|
|
5746
|
+
return {
|
|
5747
|
+
stop: () => {
|
|
5748
|
+
stopped = true;
|
|
5749
|
+
clearInterval(interval);
|
|
5268
5750
|
}
|
|
5269
|
-
|
|
5270
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5271
|
-
);
|
|
5272
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5273
|
-
} catch (err) {
|
|
5274
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5275
|
-
return {
|
|
5276
|
-
ok: false,
|
|
5277
|
-
skipped: "reclaim-error",
|
|
5278
|
-
detail: errorMessage(err)
|
|
5279
|
-
};
|
|
5280
|
-
} finally {
|
|
5281
|
-
db?.close();
|
|
5282
|
-
}
|
|
5751
|
+
};
|
|
5283
5752
|
}
|
|
5284
5753
|
|
|
5285
5754
|
// src/lib/tunnel/connection.ts
|
|
@@ -5511,6 +5980,8 @@ var StreamForwarder = class {
|
|
|
5511
5980
|
if (done) break;
|
|
5512
5981
|
const chunk = Buffer.from(value);
|
|
5513
5982
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5983
|
+
await stream.pausePromise;
|
|
5984
|
+
if (ac.signal.aborted) break;
|
|
5514
5985
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5515
5986
|
if (this.binaryFramesSupported) {
|
|
5516
5987
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5553,6 +6024,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5553
6024
|
}
|
|
5554
6025
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5555
6026
|
var BASE_RECONNECT_DELAY = 500;
|
|
6027
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5556
6028
|
function getReconnectDelay(attempt) {
|
|
5557
6029
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5558
6030
|
const jitter = Math.random() * 1e3;
|
|
@@ -5624,10 +6096,27 @@ function connectTunnel(options) {
|
|
|
5624
6096
|
ws.close();
|
|
5625
6097
|
reject(new Error("Connection timeout"));
|
|
5626
6098
|
}, 3e4);
|
|
6099
|
+
let relaySilenceTimeout;
|
|
6100
|
+
let relaySilenceTimedOut = false;
|
|
6101
|
+
const clearRelaySilenceTimeout = () => {
|
|
6102
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6103
|
+
clearTimeout(relaySilenceTimeout);
|
|
6104
|
+
relaySilenceTimeout = void 0;
|
|
6105
|
+
};
|
|
6106
|
+
const armRelaySilenceTimeout = () => {
|
|
6107
|
+
clearRelaySilenceTimeout();
|
|
6108
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6109
|
+
relaySilenceTimeout = void 0;
|
|
6110
|
+
relaySilenceTimedOut = true;
|
|
6111
|
+
ws.terminate();
|
|
6112
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6113
|
+
relaySilenceTimeout.unref?.();
|
|
6114
|
+
};
|
|
5627
6115
|
let upgradeRejection = null;
|
|
5628
6116
|
let upgradeRejectionReason = null;
|
|
5629
6117
|
ws.on("unexpected-response", (_req, res) => {
|
|
5630
6118
|
clearTimeout(connectionTimeout);
|
|
6119
|
+
clearRelaySilenceTimeout();
|
|
5631
6120
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5632
6121
|
upgradeRejectionReason = reason;
|
|
5633
6122
|
const chunks = [];
|
|
@@ -5657,6 +6146,7 @@ function connectTunnel(options) {
|
|
|
5657
6146
|
onInfo?.("WebSocket connection established");
|
|
5658
6147
|
});
|
|
5659
6148
|
ws.on("message", (data, isBinary) => {
|
|
6149
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5660
6150
|
if (isBinary) {
|
|
5661
6151
|
try {
|
|
5662
6152
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5690,6 +6180,7 @@ function connectTunnel(options) {
|
|
|
5690
6180
|
switch (message.type) {
|
|
5691
6181
|
case "connected": {
|
|
5692
6182
|
clearTimeout(connectionTimeout);
|
|
6183
|
+
armRelaySilenceTimeout();
|
|
5693
6184
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5694
6185
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5695
6186
|
onConnected?.(connectedAgentId);
|
|
@@ -5714,6 +6205,7 @@ function connectTunnel(options) {
|
|
|
5714
6205
|
});
|
|
5715
6206
|
ws.on("error", (error2) => {
|
|
5716
6207
|
clearTimeout(connectionTimeout);
|
|
6208
|
+
clearRelaySilenceTimeout();
|
|
5717
6209
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5718
6210
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5719
6211
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5725,7 +6217,8 @@ function connectTunnel(options) {
|
|
|
5725
6217
|
);
|
|
5726
6218
|
});
|
|
5727
6219
|
ws.on("close", (code, reason) => {
|
|
5728
|
-
|
|
6220
|
+
clearRelaySilenceTimeout();
|
|
6221
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5729
6222
|
forwarder.abortAll();
|
|
5730
6223
|
onDisconnected?.(code, reasonStr);
|
|
5731
6224
|
});
|
|
@@ -5733,7 +6226,7 @@ function connectTunnel(options) {
|
|
|
5733
6226
|
}
|
|
5734
6227
|
|
|
5735
6228
|
// src/lib/tunnel/runner-connection.ts
|
|
5736
|
-
var RunnerConnection = class {
|
|
6229
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5737
6230
|
opts;
|
|
5738
6231
|
sleep;
|
|
5739
6232
|
now;
|
|
@@ -5747,6 +6240,10 @@ var RunnerConnection = class {
|
|
|
5747
6240
|
reconnectAttempt = 0;
|
|
5748
6241
|
generation = 0;
|
|
5749
6242
|
disconnectedAtMs = null;
|
|
6243
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6244
|
+
"No pong received",
|
|
6245
|
+
"Orphaned socket"
|
|
6246
|
+
]);
|
|
5750
6247
|
constructor(opts) {
|
|
5751
6248
|
this.opts = opts;
|
|
5752
6249
|
this.resolvedAgentId = opts.agentId;
|
|
@@ -5803,7 +6300,8 @@ var RunnerConnection = class {
|
|
|
5803
6300
|
onDisconnected: (code, reason) => {
|
|
5804
6301
|
this.disconnectedAtMs = this.now();
|
|
5805
6302
|
events.onDisconnected(code, reason, { generation: this.generation });
|
|
5806
|
-
|
|
6303
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6304
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5807
6305
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5808
6306
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5809
6307
|
});
|
|
@@ -6142,6 +6640,101 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6142
6640
|
};
|
|
6143
6641
|
}
|
|
6144
6642
|
|
|
6643
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6644
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6645
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6646
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6647
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6648
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6649
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6650
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6651
|
+
var BAND_RANK = {
|
|
6652
|
+
low: 1,
|
|
6653
|
+
very_low: 2,
|
|
6654
|
+
critical: 3
|
|
6655
|
+
};
|
|
6656
|
+
function bandFor(freeBytes) {
|
|
6657
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6658
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6659
|
+
return "low";
|
|
6660
|
+
}
|
|
6661
|
+
function formatBytes(bytes) {
|
|
6662
|
+
if (bytes === null) return "unavailable";
|
|
6663
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6664
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6665
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6666
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6667
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6668
|
+
}
|
|
6669
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6670
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6671
|
+
}
|
|
6672
|
+
function evaluateLowDiskWarning(input) {
|
|
6673
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6674
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6675
|
+
return { warning: null, state };
|
|
6676
|
+
}
|
|
6677
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6678
|
+
if (recovered) {
|
|
6679
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6680
|
+
}
|
|
6681
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6682
|
+
if (!low) return { warning: null, state };
|
|
6683
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6684
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6685
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6686
|
+
const worse = currentRank > previousRank;
|
|
6687
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6688
|
+
const nextState = {
|
|
6689
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6690
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6691
|
+
};
|
|
6692
|
+
return {
|
|
6693
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6694
|
+
state: nextState
|
|
6695
|
+
};
|
|
6696
|
+
}
|
|
6697
|
+
|
|
6698
|
+
// src/lib/opencode/session-store-health.ts
|
|
6699
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6700
|
+
function createSessionStoreHealth(options = {}) {
|
|
6701
|
+
const now = options.now ?? (() => Date.now());
|
|
6702
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6703
|
+
let lastUnwritableAt = null;
|
|
6704
|
+
let lastWritableAt = null;
|
|
6705
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6706
|
+
return {
|
|
6707
|
+
observeUnwritable: () => {
|
|
6708
|
+
lastUnwritableAt = now();
|
|
6709
|
+
},
|
|
6710
|
+
observeWritable: () => {
|
|
6711
|
+
lastWritableAt = now();
|
|
6712
|
+
},
|
|
6713
|
+
isUnwritable: () => {
|
|
6714
|
+
if (lastUnwritableAt === null) return false;
|
|
6715
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6716
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6717
|
+
},
|
|
6718
|
+
observeCause: (sessionId, cause) => {
|
|
6719
|
+
if (sessionId === null) return;
|
|
6720
|
+
causes.delete(sessionId);
|
|
6721
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6722
|
+
while (causes.size > 256) {
|
|
6723
|
+
const oldest = causes.keys().next().value;
|
|
6724
|
+
if (typeof oldest !== "string") break;
|
|
6725
|
+
causes.delete(oldest);
|
|
6726
|
+
}
|
|
6727
|
+
},
|
|
6728
|
+
readCause: (sessionId) => {
|
|
6729
|
+
const observed = causes.get(sessionId);
|
|
6730
|
+
if (!observed) return null;
|
|
6731
|
+
causes.delete(sessionId);
|
|
6732
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6733
|
+
return observed.cause;
|
|
6734
|
+
}
|
|
6735
|
+
};
|
|
6736
|
+
}
|
|
6737
|
+
|
|
6145
6738
|
// src/lib/channels/driver.ts
|
|
6146
6739
|
import { homedir as homedir6 } from "node:os";
|
|
6147
6740
|
|
|
@@ -6598,8 +7191,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6598
7191
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6599
7192
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6600
7193
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7194
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7195
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7196
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7197
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6601
7198
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6602
7199
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7200
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6603
7201
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6604
7202
|
var HEARTBEAT_MS = 6e4;
|
|
6605
7203
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6661,6 +7259,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6661
7259
|
maxActiveSessions;
|
|
6662
7260
|
watcherStallMs;
|
|
6663
7261
|
wedgeWarningIntervalMs;
|
|
7262
|
+
sessionStoreHealth;
|
|
6664
7263
|
openCodeClient;
|
|
6665
7264
|
/** Cache of conversationId → opencode sessionId. */
|
|
6666
7265
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6753,6 +7352,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6753
7352
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6754
7353
|
*/
|
|
6755
7354
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7355
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6756
7356
|
/**
|
|
6757
7357
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6758
7358
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6785,10 +7385,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6785
7385
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6786
7386
|
*/
|
|
6787
7387
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7388
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7389
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6788
7390
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6789
7391
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6790
7392
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6791
7393
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7394
|
+
/**
|
|
7395
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7396
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7397
|
+
* identical-text turn, so it never produces a correlation.
|
|
7398
|
+
*/
|
|
7399
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6792
7400
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6793
7401
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6794
7402
|
/**
|
|
@@ -6874,21 +7482,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6874
7482
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6875
7483
|
*/
|
|
6876
7484
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6877
|
-
/**
|
|
6878
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6879
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6880
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6881
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6882
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6883
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6884
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6885
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6886
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6887
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6888
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6889
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6890
|
-
*/
|
|
6891
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6892
7485
|
/**
|
|
6893
7486
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6894
7487
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6906,8 +7499,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6906
7499
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6907
7500
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6908
7501
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6909
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6910
|
-
*
|
|
7502
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7503
|
+
* the terminal-report fence and is never sent again).
|
|
6911
7504
|
*/
|
|
6912
7505
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6913
7506
|
/**
|
|
@@ -7030,6 +7623,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7030
7623
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7031
7624
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7032
7625
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7626
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7033
7627
|
}
|
|
7034
7628
|
get isV2() {
|
|
7035
7629
|
return this.openCodeClient.version === "v2";
|
|
@@ -7045,7 +7639,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7045
7639
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7046
7640
|
}
|
|
7047
7641
|
async listSessions() {
|
|
7048
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7642
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7643
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7644
|
+
});
|
|
7049
7645
|
}
|
|
7050
7646
|
async sessionExists(sessionId) {
|
|
7051
7647
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7222,6 +7818,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7222
7818
|
}
|
|
7223
7819
|
return ids;
|
|
7224
7820
|
}
|
|
7821
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7822
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7823
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7824
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7825
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7826
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7827
|
+
}
|
|
7828
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7829
|
+
}
|
|
7830
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7831
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7832
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7833
|
+
this.awaitingReadopt.delete(messageId);
|
|
7834
|
+
this.sessions.delete(conversationId);
|
|
7835
|
+
this.supersede(conversationId, sessionId);
|
|
7836
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7837
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7838
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7839
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7840
|
+
this.log({
|
|
7841
|
+
level: "error",
|
|
7842
|
+
message: errorMessage3,
|
|
7843
|
+
conversation_id: conversationId,
|
|
7844
|
+
message_id: messageId
|
|
7845
|
+
});
|
|
7846
|
+
const reportError = await this.reportUntrackableAck(
|
|
7847
|
+
conversationId,
|
|
7848
|
+
messageId,
|
|
7849
|
+
null,
|
|
7850
|
+
errorMessage3
|
|
7851
|
+
);
|
|
7852
|
+
if (reportError !== null) {
|
|
7853
|
+
this.log({
|
|
7854
|
+
level: "warn",
|
|
7855
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7856
|
+
conversation_id: conversationId,
|
|
7857
|
+
message_id: messageId
|
|
7858
|
+
});
|
|
7859
|
+
}
|
|
7860
|
+
if (source === "readopt") {
|
|
7861
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7862
|
+
}
|
|
7863
|
+
}
|
|
7225
7864
|
/**
|
|
7226
7865
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7227
7866
|
*
|
|
@@ -7401,6 +8040,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7401
8040
|
}
|
|
7402
8041
|
break;
|
|
7403
8042
|
}
|
|
8043
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8044
|
+
if (correlation) {
|
|
8045
|
+
correlation.attempts += 1;
|
|
8046
|
+
let correlatedId = null;
|
|
8047
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8048
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8049
|
+
conv,
|
|
8050
|
+
message,
|
|
8051
|
+
correlation.sessionId
|
|
8052
|
+
);
|
|
8053
|
+
if (polled.ok)
|
|
8054
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8055
|
+
} else if (correlation.knownUserIds === null) {
|
|
8056
|
+
this.log({
|
|
8057
|
+
level: "debug",
|
|
8058
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8059
|
+
conversation_id: conv.id,
|
|
8060
|
+
message_id: message.id
|
|
8061
|
+
});
|
|
8062
|
+
}
|
|
8063
|
+
if (correlatedId !== null) {
|
|
8064
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8065
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8066
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8067
|
+
this.dispatched.add(message.id);
|
|
8068
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8069
|
+
dispatched += 1;
|
|
8070
|
+
this.log({
|
|
8071
|
+
level: "info",
|
|
8072
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8073
|
+
conversation_id: conv.id,
|
|
8074
|
+
message_id: message.id
|
|
8075
|
+
});
|
|
8076
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8077
|
+
opencode_message_id: correlatedId
|
|
8078
|
+
});
|
|
8079
|
+
continue;
|
|
8080
|
+
}
|
|
8081
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8082
|
+
await this.handoffUntrackableCorrelation(
|
|
8083
|
+
conv.id,
|
|
8084
|
+
message.id,
|
|
8085
|
+
correlation.sessionId,
|
|
8086
|
+
correlation.attempts,
|
|
8087
|
+
"fresh"
|
|
8088
|
+
);
|
|
8089
|
+
break;
|
|
8090
|
+
}
|
|
8091
|
+
break;
|
|
8092
|
+
}
|
|
7404
8093
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7405
8094
|
if (effectiveOpencodeMessageId) {
|
|
7406
8095
|
const outcome = await this.resolveRedrive(
|
|
@@ -7422,6 +8111,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7422
8111
|
model: message.opencode_model ?? void 0
|
|
7423
8112
|
};
|
|
7424
8113
|
let opencodeMessageId;
|
|
8114
|
+
let preDispatchUserIds = null;
|
|
7425
8115
|
try {
|
|
7426
8116
|
this.log({
|
|
7427
8117
|
level: "info",
|
|
@@ -7451,7 +8141,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7451
8141
|
message.content,
|
|
7452
8142
|
options,
|
|
7453
8143
|
sendAttachments,
|
|
7454
|
-
this.openCodeClient
|
|
8144
|
+
this.openCodeClient,
|
|
8145
|
+
{
|
|
8146
|
+
onKnownUserIds: (ids) => {
|
|
8147
|
+
preDispatchUserIds = ids;
|
|
8148
|
+
},
|
|
8149
|
+
onKnownUserIdsError: (error2) => {
|
|
8150
|
+
this.log({
|
|
8151
|
+
level: "warn",
|
|
8152
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8153
|
+
conversation_id: conv.id,
|
|
8154
|
+
message_id: message.id
|
|
8155
|
+
});
|
|
8156
|
+
}
|
|
8157
|
+
}
|
|
7455
8158
|
)
|
|
7456
8159
|
);
|
|
7457
8160
|
} catch (err) {
|
|
@@ -7459,6 +8162,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7459
8162
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7460
8163
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7461
8164
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8165
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7462
8166
|
this.log({
|
|
7463
8167
|
level: "error",
|
|
7464
8168
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7534,39 +8238,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7534
8238
|
if (this.isV2) {
|
|
7535
8239
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7536
8240
|
}
|
|
7537
|
-
|
|
7538
|
-
|
|
7539
|
-
|
|
7540
|
-
|
|
7541
|
-
|
|
7542
|
-
|
|
7543
|
-
message_id: message.id
|
|
7544
|
-
});
|
|
7545
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7546
|
-
continue;
|
|
7547
|
-
}
|
|
7548
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7549
|
-
this.sessions.delete(conv.id);
|
|
7550
|
-
this.supersede(conv.id, sessionId);
|
|
7551
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8241
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8242
|
+
sessionId,
|
|
8243
|
+
content: message.content,
|
|
8244
|
+
attempts: 0,
|
|
8245
|
+
knownUserIds: preDispatchUserIds
|
|
8246
|
+
});
|
|
7552
8247
|
this.log({
|
|
7553
|
-
level: "
|
|
7554
|
-
message:
|
|
8248
|
+
level: "warn",
|
|
8249
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7555
8250
|
conversation_id: conv.id,
|
|
7556
8251
|
message_id: message.id
|
|
7557
8252
|
});
|
|
7558
|
-
|
|
7559
|
-
this.log({
|
|
7560
|
-
level: "warn",
|
|
7561
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7562
|
-
conversation_id: conv.id,
|
|
7563
|
-
message_id: message.id
|
|
7564
|
-
});
|
|
7565
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7566
|
-
});
|
|
8253
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7567
8254
|
break;
|
|
7568
8255
|
}
|
|
7569
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7570
8256
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7571
8257
|
this.releasedOpencodeIds.delete(message.id);
|
|
7572
8258
|
this.dispatched.add(message.id);
|
|
@@ -8171,23 +8857,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8171
8857
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8172
8858
|
return 1;
|
|
8173
8859
|
}
|
|
8174
|
-
/**
|
|
8175
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8176
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8177
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8178
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8179
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8180
|
-
* count, since a new session is a genuinely different attempt.
|
|
8181
|
-
*/
|
|
8182
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8183
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8184
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8185
|
-
existing.count += 1;
|
|
8186
|
-
return existing.count;
|
|
8187
|
-
}
|
|
8188
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8189
|
-
return 1;
|
|
8190
|
-
}
|
|
8191
8860
|
/** Record one queued give-up toward the never-started bound. */
|
|
8192
8861
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8193
8862
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8499,8 +9168,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8499
9168
|
b2LastDescendantCheckMs: 0,
|
|
8500
9169
|
b2RootOngoingHeldLogged: false,
|
|
8501
9170
|
b2AbandonedSignalled: false,
|
|
9171
|
+
b2AbandonedWithoutAnswer: false,
|
|
8502
9172
|
ambiguousPinnedSinceMs: 0,
|
|
8503
9173
|
ambiguousResolved: false,
|
|
9174
|
+
sessionStoreFailurePending: false,
|
|
8504
9175
|
neverStartedResolved: false
|
|
8505
9176
|
});
|
|
8506
9177
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8522,6 +9193,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8522
9193
|
this.bufferedSessionErrors.delete(oldest);
|
|
8523
9194
|
}
|
|
8524
9195
|
}
|
|
9196
|
+
retainSessionError(event) {
|
|
9197
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9198
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9199
|
+
reason: event.reason,
|
|
9200
|
+
observedAt: this.now()
|
|
9201
|
+
});
|
|
9202
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9203
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9204
|
+
if (typeof oldest !== "string") break;
|
|
9205
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9206
|
+
}
|
|
9207
|
+
}
|
|
9208
|
+
readRetainedSessionError(sessionId) {
|
|
9209
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9210
|
+
if (!retained) return null;
|
|
9211
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9212
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9213
|
+
return null;
|
|
9214
|
+
}
|
|
9215
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9216
|
+
return retained.reason;
|
|
9217
|
+
}
|
|
8525
9218
|
/**
|
|
8526
9219
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8527
9220
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8617,8 +9310,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8617
9310
|
b2LastDescendantCheckMs: 0,
|
|
8618
9311
|
b2RootOngoingHeldLogged: false,
|
|
8619
9312
|
b2AbandonedSignalled: false,
|
|
9313
|
+
b2AbandonedWithoutAnswer: false,
|
|
8620
9314
|
ambiguousPinnedSinceMs: 0,
|
|
8621
9315
|
ambiguousResolved: false,
|
|
9316
|
+
sessionStoreFailurePending: false,
|
|
8622
9317
|
neverStartedResolved: false
|
|
8623
9318
|
});
|
|
8624
9319
|
}
|
|
@@ -8846,9 +9541,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8846
9541
|
return;
|
|
8847
9542
|
}
|
|
8848
9543
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9544
|
+
this.retainSessionError(event);
|
|
8849
9545
|
this.log({
|
|
8850
|
-
level: "
|
|
8851
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9546
|
+
level: "warn",
|
|
9547
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8852
9548
|
conversation_id: watcher.conv.id
|
|
8853
9549
|
});
|
|
8854
9550
|
return;
|
|
@@ -8878,9 +9574,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8878
9574
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8879
9575
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8880
9576
|
if (state !== "queued") {
|
|
9577
|
+
this.retainSessionError(event);
|
|
8881
9578
|
this.log({
|
|
8882
|
-
level: "
|
|
8883
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9579
|
+
level: "warn",
|
|
9580
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8884
9581
|
conversation_id: watcher.conv.id,
|
|
8885
9582
|
message_id: inFlight.evidentMessageId
|
|
8886
9583
|
});
|
|
@@ -9032,6 +9729,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9032
9729
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9033
9730
|
this.neverStartedAttempts.delete(id);
|
|
9034
9731
|
}
|
|
9732
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9733
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9734
|
+
return;
|
|
9735
|
+
}
|
|
9035
9736
|
if (inFlight.neverStartedResolved) {
|
|
9036
9737
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9037
9738
|
inFlight.neverStartedResolved = false;
|
|
@@ -9105,7 +9806,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9105
9806
|
if (state === "failed") {
|
|
9106
9807
|
this.anchorDeliveryDeadline(inFlight);
|
|
9107
9808
|
if (!inFlight.done) {
|
|
9108
|
-
const
|
|
9809
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9810
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9811
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9812
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9813
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9814
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9815
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9816
|
+
(detail) => detail !== null
|
|
9817
|
+
);
|
|
9818
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9109
9819
|
this.log({
|
|
9110
9820
|
level: "error",
|
|
9111
9821
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9191,10 +9901,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9191
9901
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9192
9902
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9193
9903
|
inFlight.b2AbandonedSignalled = false;
|
|
9904
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9194
9905
|
}
|
|
9195
9906
|
} else {
|
|
9196
9907
|
if (inFlight.b2AbandonedSignalled) {
|
|
9197
|
-
|
|
9908
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9909
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9910
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9911
|
+
sessionId,
|
|
9912
|
+
watcher,
|
|
9913
|
+
inFlight,
|
|
9914
|
+
messages,
|
|
9915
|
+
reply?.info?.finish ?? reply?.finish
|
|
9916
|
+
);
|
|
9917
|
+
} else {
|
|
9918
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9919
|
+
}
|
|
9198
9920
|
return;
|
|
9199
9921
|
}
|
|
9200
9922
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9211,17 +9933,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9211
9933
|
descendantOngoing,
|
|
9212
9934
|
rootOngoing
|
|
9213
9935
|
})) {
|
|
9936
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9937
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9938
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9939
|
+
watched_for_ms: pinnedForMs,
|
|
9940
|
+
run_state: "failed",
|
|
9941
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9942
|
+
});
|
|
9943
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9944
|
+
return;
|
|
9945
|
+
}
|
|
9214
9946
|
inFlight.b2AbandonedSignalled = true;
|
|
9947
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9948
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9949
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9950
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9951
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9952
|
+
watched_for_ms: pinnedForMs,
|
|
9953
|
+
run_state: "failed",
|
|
9954
|
+
finish,
|
|
9955
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9956
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9957
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9958
|
+
});
|
|
9959
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9960
|
+
sessionId,
|
|
9961
|
+
watcher,
|
|
9962
|
+
inFlight,
|
|
9963
|
+
messages,
|
|
9964
|
+
finish
|
|
9965
|
+
);
|
|
9966
|
+
return;
|
|
9967
|
+
}
|
|
9215
9968
|
this.log({
|
|
9216
9969
|
level: "warn",
|
|
9217
9970
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9218
9971
|
conversation_id: conv.id,
|
|
9219
9972
|
message_id: id
|
|
9220
9973
|
});
|
|
9221
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9222
9974
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9223
9975
|
watched_for_ms: pinnedForMs,
|
|
9224
|
-
finish
|
|
9976
|
+
finish,
|
|
9225
9977
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9226
9978
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9227
9979
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9269,6 +10021,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9269
10021
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9270
10022
|
sessionOngoing: ongoing
|
|
9271
10023
|
})) {
|
|
10024
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10025
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10026
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10027
|
+
watched_for_ms: pinnedForMs,
|
|
10028
|
+
run_state: "failed",
|
|
10029
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10030
|
+
});
|
|
10031
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10032
|
+
return;
|
|
10033
|
+
}
|
|
9272
10034
|
inFlight.ambiguousResolved = true;
|
|
9273
10035
|
this.log({
|
|
9274
10036
|
level: "warn",
|
|
@@ -9477,20 +10239,90 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9477
10239
|
}
|
|
9478
10240
|
this.log({
|
|
9479
10241
|
level: "warn",
|
|
9480
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10242
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10243
|
+
conversation_id: conv.id,
|
|
10244
|
+
message_id: inFlight.evidentMessageId
|
|
10245
|
+
});
|
|
10246
|
+
return;
|
|
10247
|
+
}
|
|
10248
|
+
inFlight.done = true;
|
|
10249
|
+
await this.reportSubagentAuthFailures(
|
|
10250
|
+
watcher.conv.id,
|
|
10251
|
+
inFlight.opencodeMessageId,
|
|
10252
|
+
inFlight.evidentMessageId,
|
|
10253
|
+
messages
|
|
10254
|
+
);
|
|
10255
|
+
}
|
|
10256
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10257
|
+
}
|
|
10258
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10259
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10260
|
+
const conv = watcher.conv;
|
|
10261
|
+
const legacyError = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
10262
|
+
const replyShape = describeAbandonedReplyShape(messages, inFlight.opencodeMessageId);
|
|
10263
|
+
const error2 = replyShape === null ? legacyError : `${legacyError} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
10264
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10265
|
+
this.log({
|
|
10266
|
+
level: "warn",
|
|
10267
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10268
|
+
conversation_id: conv.id,
|
|
10269
|
+
message_id: inFlight.evidentMessageId
|
|
10270
|
+
});
|
|
10271
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10272
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10273
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10274
|
+
messages,
|
|
10275
|
+
inFlight.opencodeMessageId,
|
|
10276
|
+
inFlight.evidentMessageId
|
|
10277
|
+
);
|
|
10278
|
+
try {
|
|
10279
|
+
await this.markFailed(
|
|
10280
|
+
conv.id,
|
|
10281
|
+
inFlight.evidentMessageId,
|
|
10282
|
+
sessionId,
|
|
10283
|
+
error2,
|
|
10284
|
+
usage,
|
|
10285
|
+
void 0,
|
|
10286
|
+
usageAgentName,
|
|
10287
|
+
subagentInvocations
|
|
10288
|
+
);
|
|
10289
|
+
} catch (err) {
|
|
10290
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10291
|
+
if (err instanceof ChannelTerminalError) {
|
|
10292
|
+
this.log({
|
|
10293
|
+
level: "warn",
|
|
10294
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9481
10295
|
conversation_id: conv.id,
|
|
9482
10296
|
message_id: inFlight.evidentMessageId
|
|
9483
10297
|
});
|
|
10298
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9484
10299
|
return;
|
|
9485
10300
|
}
|
|
9486
|
-
inFlight.
|
|
9487
|
-
|
|
9488
|
-
|
|
9489
|
-
|
|
9490
|
-
|
|
9491
|
-
|
|
9492
|
-
|
|
10301
|
+
if (this.now() >= inFlight.deadline) {
|
|
10302
|
+
this.log({
|
|
10303
|
+
level: "warn",
|
|
10304
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10305
|
+
conversation_id: conv.id,
|
|
10306
|
+
message_id: inFlight.evidentMessageId
|
|
10307
|
+
});
|
|
10308
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10309
|
+
return;
|
|
10310
|
+
}
|
|
10311
|
+
this.log({
|
|
10312
|
+
level: "warn",
|
|
10313
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10314
|
+
conversation_id: conv.id,
|
|
10315
|
+
message_id: inFlight.evidentMessageId
|
|
10316
|
+
});
|
|
10317
|
+
return;
|
|
9493
10318
|
}
|
|
10319
|
+
inFlight.done = true;
|
|
10320
|
+
await this.reportSubagentAuthFailures(
|
|
10321
|
+
watcher.conv.id,
|
|
10322
|
+
inFlight.opencodeMessageId,
|
|
10323
|
+
inFlight.evidentMessageId,
|
|
10324
|
+
messages
|
|
10325
|
+
);
|
|
9494
10326
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9495
10327
|
}
|
|
9496
10328
|
/**
|
|
@@ -9548,6 +10380,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9548
10380
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9549
10381
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9550
10382
|
}
|
|
10383
|
+
/**
|
|
10384
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10385
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10386
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10387
|
+
*/
|
|
10388
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10389
|
+
const conv = watcher.conv;
|
|
10390
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10391
|
+
this.log({
|
|
10392
|
+
level: "error",
|
|
10393
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10394
|
+
conversation_id: conv.id,
|
|
10395
|
+
message_id: inFlight.evidentMessageId
|
|
10396
|
+
});
|
|
10397
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10398
|
+
try {
|
|
10399
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10400
|
+
} catch (err) {
|
|
10401
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10402
|
+
if (err instanceof ChannelTerminalError) {
|
|
10403
|
+
this.log({
|
|
10404
|
+
level: "warn",
|
|
10405
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10406
|
+
conversation_id: conv.id,
|
|
10407
|
+
message_id: inFlight.evidentMessageId
|
|
10408
|
+
});
|
|
10409
|
+
this.recordReleasedOpencodeId(
|
|
10410
|
+
inFlight.evidentMessageId,
|
|
10411
|
+
sessionId,
|
|
10412
|
+
inFlight.opencodeMessageId
|
|
10413
|
+
);
|
|
10414
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10415
|
+
return;
|
|
10416
|
+
}
|
|
10417
|
+
if (this.now() >= inFlight.deadline) {
|
|
10418
|
+
this.log({
|
|
10419
|
+
level: "warn",
|
|
10420
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10421
|
+
conversation_id: conv.id,
|
|
10422
|
+
message_id: inFlight.evidentMessageId
|
|
10423
|
+
});
|
|
10424
|
+
this.recordReleasedOpencodeId(
|
|
10425
|
+
inFlight.evidentMessageId,
|
|
10426
|
+
sessionId,
|
|
10427
|
+
inFlight.opencodeMessageId
|
|
10428
|
+
);
|
|
10429
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10430
|
+
return;
|
|
10431
|
+
}
|
|
10432
|
+
this.log({
|
|
10433
|
+
level: "warn",
|
|
10434
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10435
|
+
conversation_id: conv.id,
|
|
10436
|
+
message_id: inFlight.evidentMessageId
|
|
10437
|
+
});
|
|
10438
|
+
return;
|
|
10439
|
+
}
|
|
10440
|
+
inFlight.done = true;
|
|
10441
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10442
|
+
}
|
|
9551
10443
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9552
10444
|
/**
|
|
9553
10445
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9586,11 +10478,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9586
10478
|
}
|
|
9587
10479
|
}
|
|
9588
10480
|
}
|
|
9589
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10481
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9590
10482
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10483
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10484
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10485
|
+
this.awaitingCorrelation.delete(id);
|
|
10486
|
+
this.log({
|
|
10487
|
+
level: "debug",
|
|
10488
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10489
|
+
message_id: id
|
|
10490
|
+
});
|
|
10491
|
+
}
|
|
10492
|
+
}
|
|
9591
10493
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9592
10494
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9593
10495
|
this.untrackableAck.delete(id);
|
|
10496
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9594
10497
|
this.untrackableAckSignalled.delete(id);
|
|
9595
10498
|
this.untrackableAckWarned.delete(id);
|
|
9596
10499
|
this.log({
|
|
@@ -9630,7 +10533,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9630
10533
|
bySession.set(row.opencode_session_id, list);
|
|
9631
10534
|
}
|
|
9632
10535
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9633
|
-
let messages;
|
|
10536
|
+
let messages = null;
|
|
9634
10537
|
try {
|
|
9635
10538
|
if (this.isV2) {
|
|
9636
10539
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9639,9 +10542,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9639
10542
|
level: "warn",
|
|
9640
10543
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9641
10544
|
});
|
|
9642
|
-
|
|
10545
|
+
} else {
|
|
10546
|
+
messages = snapshot;
|
|
9643
10547
|
}
|
|
9644
|
-
messages = snapshot;
|
|
9645
10548
|
} else {
|
|
9646
10549
|
const polled = await pollSessionMessagesForRedrive(
|
|
9647
10550
|
this.port,
|
|
@@ -9654,15 +10557,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9654
10557
|
level: "warn",
|
|
9655
10558
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9656
10559
|
});
|
|
9657
|
-
|
|
10560
|
+
} else {
|
|
10561
|
+
messages = polled.messages;
|
|
9658
10562
|
}
|
|
9659
|
-
messages = polled.messages;
|
|
9660
10563
|
}
|
|
9661
10564
|
} catch (err) {
|
|
9662
10565
|
this.log({
|
|
9663
10566
|
level: "warn",
|
|
9664
10567
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9665
10568
|
});
|
|
10569
|
+
}
|
|
10570
|
+
if (messages === null) {
|
|
10571
|
+
for (const row of sessionRows) {
|
|
10572
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10573
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10574
|
+
}
|
|
10575
|
+
}
|
|
9666
10576
|
continue;
|
|
9667
10577
|
}
|
|
9668
10578
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9672,6 +10582,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9672
10582
|
}
|
|
9673
10583
|
}
|
|
9674
10584
|
}
|
|
10585
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10586
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10587
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10588
|
+
if (!correlation) return false;
|
|
10589
|
+
correlation.attempts += 1;
|
|
10590
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10591
|
+
if (correlatedId !== null) {
|
|
10592
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10593
|
+
const conv = this.convForRow(sessionId, row);
|
|
10594
|
+
const message = this.queuedMessageForRow(row);
|
|
10595
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10596
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10597
|
+
this.dispatched.add(row.id);
|
|
10598
|
+
this.readopted.add(row.id);
|
|
10599
|
+
this.ensureWatcherRunning(sessionId);
|
|
10600
|
+
this.log({
|
|
10601
|
+
level: "info",
|
|
10602
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10603
|
+
conversation_id: row.conversation_id,
|
|
10604
|
+
message_id: row.id
|
|
10605
|
+
});
|
|
10606
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10607
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10608
|
+
opencode_message_id: correlatedId
|
|
10609
|
+
});
|
|
10610
|
+
return true;
|
|
10611
|
+
}
|
|
10612
|
+
if (correlation.knownUserIds === null) {
|
|
10613
|
+
this.log({
|
|
10614
|
+
level: "debug",
|
|
10615
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10616
|
+
conversation_id: row.conversation_id,
|
|
10617
|
+
message_id: row.id
|
|
10618
|
+
});
|
|
10619
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10620
|
+
this.log({
|
|
10621
|
+
level: "warn",
|
|
10622
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10623
|
+
conversation_id: row.conversation_id,
|
|
10624
|
+
message_id: row.id
|
|
10625
|
+
});
|
|
10626
|
+
}
|
|
10627
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10628
|
+
await this.handoffUntrackableCorrelation(
|
|
10629
|
+
row.conversation_id,
|
|
10630
|
+
row.id,
|
|
10631
|
+
correlation.sessionId,
|
|
10632
|
+
correlation.attempts,
|
|
10633
|
+
"readopt"
|
|
10634
|
+
);
|
|
10635
|
+
}
|
|
10636
|
+
return true;
|
|
10637
|
+
}
|
|
9675
10638
|
/**
|
|
9676
10639
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9677
10640
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9679,8 +10642,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9679
10642
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9680
10643
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9681
10644
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9682
|
-
* before the restart)
|
|
9683
|
-
*
|
|
10645
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10646
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10647
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10648
|
+
* `forceReadoptRun`):
|
|
9684
10649
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9685
10650
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9686
10651
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9689,7 +10654,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9689
10654
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9690
10655
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9691
10656
|
* tracking the stored id so the reply correlates by it;
|
|
9692
|
-
* - `unknown`/null id →
|
|
10657
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10658
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9693
10659
|
*
|
|
9694
10660
|
* Only `ChannelAuthError` propagates.
|
|
9695
10661
|
*/
|
|
@@ -9703,6 +10669,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9703
10669
|
});
|
|
9704
10670
|
return;
|
|
9705
10671
|
}
|
|
10672
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9706
10673
|
const ocId = row.opencode_message_id;
|
|
9707
10674
|
const state = messageRunState(messages, ocId ?? "");
|
|
9708
10675
|
if (state === "done") {
|
|
@@ -9769,12 +10736,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9769
10736
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9770
10737
|
this.dontRedispatch.delete(row.id);
|
|
9771
10738
|
this.untrackableAck.delete(row.id);
|
|
10739
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9772
10740
|
this.untrackableAckSignalled.delete(row.id);
|
|
9773
10741
|
this.untrackableAckWarned.delete(row.id);
|
|
9774
10742
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9775
10743
|
return;
|
|
9776
10744
|
}
|
|
9777
|
-
if (this.
|
|
10745
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10746
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10747
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10748
|
+
this.untrackableAckWarned.add(row.id);
|
|
10749
|
+
this.log({
|
|
10750
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10751
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10752
|
+
conversation_id: row.conversation_id,
|
|
10753
|
+
message_id: row.id
|
|
10754
|
+
});
|
|
10755
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10756
|
+
const reportError = await this.reportUntrackableAck(
|
|
10757
|
+
row.conversation_id,
|
|
10758
|
+
row.id,
|
|
10759
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10760
|
+
errorMessage3
|
|
10761
|
+
);
|
|
10762
|
+
if (reportError !== null) {
|
|
10763
|
+
this.log({
|
|
10764
|
+
level: "warn",
|
|
10765
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10766
|
+
conversation_id: row.conversation_id,
|
|
10767
|
+
message_id: row.id
|
|
10768
|
+
});
|
|
10769
|
+
}
|
|
10770
|
+
} else {
|
|
10771
|
+
this.log({
|
|
10772
|
+
level: "debug",
|
|
10773
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10774
|
+
conversation_id: row.conversation_id,
|
|
10775
|
+
message_id: row.id
|
|
10776
|
+
});
|
|
10777
|
+
}
|
|
10778
|
+
return;
|
|
10779
|
+
}
|
|
10780
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9778
10781
|
this.log({
|
|
9779
10782
|
level: "debug",
|
|
9780
10783
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9952,6 +10955,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9952
10955
|
}
|
|
9953
10956
|
this.dontRedispatch.delete(row.id);
|
|
9954
10957
|
this.untrackableAck.delete(row.id);
|
|
10958
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9955
10959
|
this.untrackableAckSignalled.delete(row.id);
|
|
9956
10960
|
this.untrackableAckWarned.delete(row.id);
|
|
9957
10961
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9971,8 +10975,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9971
10975
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9972
10976
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9973
10977
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9974
|
-
* re-entry)
|
|
9975
|
-
*
|
|
10978
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10979
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10980
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9976
10981
|
*
|
|
9977
10982
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9978
10983
|
* `processed_at` (Invariant 1).
|
|
@@ -10035,6 +11040,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10035
11040
|
);
|
|
10036
11041
|
}
|
|
10037
11042
|
let ocId;
|
|
11043
|
+
let preDispatchUserIds = null;
|
|
10038
11044
|
try {
|
|
10039
11045
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10040
11046
|
sessionId,
|
|
@@ -10044,7 +11050,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10044
11050
|
row.content,
|
|
10045
11051
|
options,
|
|
10046
11052
|
sendAttachments,
|
|
10047
|
-
this.openCodeClient
|
|
11053
|
+
this.openCodeClient,
|
|
11054
|
+
{
|
|
11055
|
+
onKnownUserIds: (ids) => {
|
|
11056
|
+
preDispatchUserIds = ids;
|
|
11057
|
+
},
|
|
11058
|
+
onKnownUserIdsError: (error2) => {
|
|
11059
|
+
this.log({
|
|
11060
|
+
level: "warn",
|
|
11061
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11062
|
+
conversation_id: row.conversation_id,
|
|
11063
|
+
message_id: row.id
|
|
11064
|
+
});
|
|
11065
|
+
}
|
|
11066
|
+
}
|
|
10048
11067
|
)
|
|
10049
11068
|
);
|
|
10050
11069
|
} catch (err) {
|
|
@@ -10077,6 +11096,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10077
11096
|
}
|
|
10078
11097
|
} else {
|
|
10079
11098
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11099
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10080
11100
|
}
|
|
10081
11101
|
this.log({
|
|
10082
11102
|
level: "error",
|
|
@@ -10125,39 +11145,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10125
11145
|
}
|
|
10126
11146
|
if (ocId === null) {
|
|
10127
11147
|
this.awaitingReadopt.delete(row.id);
|
|
10128
|
-
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
|
|
10132
|
-
|
|
10133
|
-
|
|
10134
|
-
this.log({
|
|
10135
|
-
level: "error",
|
|
10136
|
-
message: errorMessage3,
|
|
10137
|
-
conversation_id: row.conversation_id,
|
|
10138
|
-
message_id: row.id
|
|
10139
|
-
});
|
|
10140
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10141
|
-
this.log({
|
|
10142
|
-
level: "warn",
|
|
10143
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10144
|
-
conversation_id: row.conversation_id,
|
|
10145
|
-
message_id: row.id
|
|
10146
|
-
});
|
|
10147
|
-
});
|
|
10148
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10149
|
-
return;
|
|
10150
|
-
}
|
|
11148
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11149
|
+
sessionId,
|
|
11150
|
+
content: row.content,
|
|
11151
|
+
attempts: 0,
|
|
11152
|
+
knownUserIds: preDispatchUserIds
|
|
11153
|
+
});
|
|
10151
11154
|
this.log({
|
|
10152
11155
|
level: "warn",
|
|
10153
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11156
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10154
11157
|
conversation_id: row.conversation_id,
|
|
10155
11158
|
message_id: row.id
|
|
10156
11159
|
});
|
|
10157
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10158
11160
|
return;
|
|
10159
11161
|
}
|
|
10160
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10161
11162
|
const processedAtMs = this.processedAtMs(row);
|
|
10162
11163
|
this.registerReadopted(
|
|
10163
11164
|
readoptConv,
|
|
@@ -12752,9 +13753,9 @@ async function driveChannels(state, driver) {
|
|
|
12752
13753
|
}
|
|
12753
13754
|
}
|
|
12754
13755
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12755
|
-
var
|
|
13756
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12756
13757
|
function shouldWarnForReclaimSkip(reason) {
|
|
12757
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13758
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12758
13759
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12759
13760
|
if (!version2) return false;
|
|
12760
13761
|
const major = Number(version2[1]);
|
|
@@ -12811,61 +13812,96 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12811
13812
|
async function runSweep(state, driver, config) {
|
|
12812
13813
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12813
13814
|
try {
|
|
12814
|
-
const
|
|
12815
|
-
|
|
12816
|
-
|
|
13815
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13816
|
+
const sessions = enumeration.sessions;
|
|
13817
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13818
|
+
if (enumerationReason !== null) {
|
|
13819
|
+
logActivity(state, {
|
|
12817
13820
|
type: "info",
|
|
12818
|
-
|
|
12819
|
-
|
|
12820
|
-
outcome: "sweep-skipped",
|
|
12821
|
-
reason: "opencode-unreachable"
|
|
12822
|
-
}
|
|
13821
|
+
level: "warn",
|
|
13822
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12823
13823
|
});
|
|
12824
|
-
return;
|
|
12825
13824
|
}
|
|
12826
|
-
const toDelete = selectSessionsToDelete(
|
|
12827
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12828
|
-
{
|
|
12829
|
-
maxAgeMs: config.maxAgeMs,
|
|
12830
|
-
maxCount: config.maxCount,
|
|
12831
|
-
nowMs: Date.now(),
|
|
12832
|
-
protectedIds: driver.protectedSessionIds()
|
|
12833
|
-
}
|
|
12834
|
-
);
|
|
12835
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12836
13825
|
let deleted = 0;
|
|
12837
13826
|
let failed = 0;
|
|
13827
|
+
let failed4xx = 0;
|
|
13828
|
+
let failed5xx = 0;
|
|
13829
|
+
let failedOtherStatus = 0;
|
|
13830
|
+
let failedThrew = 0;
|
|
12838
13831
|
let skippedNewlyActive = 0;
|
|
12839
|
-
|
|
12840
|
-
|
|
12841
|
-
|
|
12842
|
-
|
|
12843
|
-
|
|
12844
|
-
|
|
12845
|
-
|
|
12846
|
-
|
|
13832
|
+
let remaining = 0;
|
|
13833
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13834
|
+
if (sessions !== null) {
|
|
13835
|
+
const toDelete = selectSessionsToDelete(
|
|
13836
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13837
|
+
{
|
|
13838
|
+
maxAgeMs: config.maxAgeMs,
|
|
13839
|
+
maxCount: config.maxCount,
|
|
13840
|
+
nowMs: Date.now(),
|
|
13841
|
+
protectedIds: protectedNow
|
|
13842
|
+
}
|
|
13843
|
+
);
|
|
13844
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13845
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13846
|
+
protectedNow = driver.protectedSessionIds();
|
|
13847
|
+
for (const id of deleteCandidates) {
|
|
13848
|
+
if (protectedNow.has(id)) {
|
|
13849
|
+
skippedNewlyActive++;
|
|
13850
|
+
logActivity(state, {
|
|
13851
|
+
type: "info",
|
|
13852
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13853
|
+
});
|
|
13854
|
+
continue;
|
|
13855
|
+
}
|
|
13856
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13857
|
+
if (outcome.ok) {
|
|
13858
|
+
deleted++;
|
|
13859
|
+
} else if (outcome.status === "threw") {
|
|
13860
|
+
failed++;
|
|
13861
|
+
failedThrew++;
|
|
13862
|
+
} else {
|
|
13863
|
+
failed++;
|
|
13864
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13865
|
+
if (statusClass === 4) failed4xx++;
|
|
13866
|
+
else if (statusClass === 5) failed5xx++;
|
|
13867
|
+
else failedOtherStatus++;
|
|
13868
|
+
}
|
|
12847
13869
|
}
|
|
12848
|
-
|
|
12849
|
-
|
|
13870
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13871
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13872
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13873
|
+
logActivity(state, {
|
|
13874
|
+
type: "info",
|
|
13875
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13876
|
+
});
|
|
12850
13877
|
}
|
|
12851
|
-
const
|
|
12852
|
-
|
|
12853
|
-
|
|
12854
|
-
|
|
12855
|
-
|
|
12856
|
-
|
|
13878
|
+
const sessionMetadata = {
|
|
13879
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13880
|
+
sessions_deleted: deleted,
|
|
13881
|
+
sessions_failed: failed,
|
|
13882
|
+
sessions_failed_4xx: failed4xx,
|
|
13883
|
+
sessions_failed_5xx: failed5xx,
|
|
13884
|
+
sessions_failed_other: failedOtherStatus,
|
|
13885
|
+
sessions_failed_threw: failedThrew,
|
|
13886
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13887
|
+
sessions_remaining: remaining,
|
|
13888
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13889
|
+
};
|
|
12857
13890
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12858
13891
|
dbPath: sessionDbPath(),
|
|
12859
|
-
|
|
13892
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13893
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13894
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12860
13895
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12861
13896
|
});
|
|
12862
13897
|
if (reclaimResult.ok) {
|
|
13898
|
+
state.sessionStoreHealth.observeWritable();
|
|
12863
13899
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12864
13900
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12865
13901
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12866
13902
|
logSessionCleanupActivity(state, "sweep", {
|
|
12867
13903
|
type: "info",
|
|
12868
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13904
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12869
13905
|
metadata: {
|
|
12870
13906
|
outcome: "reclaimed",
|
|
12871
13907
|
mode: reclaimResult.mode,
|
|
@@ -12873,25 +13909,27 @@ async function runSweep(state, driver, config) {
|
|
|
12873
13909
|
after_bytes: reclaimResult.afterBytes,
|
|
12874
13910
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12875
13911
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
12879
|
-
|
|
13912
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13913
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13914
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13915
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13916
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13917
|
+
...sessionMetadata
|
|
12880
13918
|
}
|
|
12881
13919
|
});
|
|
12882
13920
|
} else {
|
|
13921
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13922
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13923
|
+
}
|
|
12883
13924
|
logSessionCleanupActivity(state, "sweep", {
|
|
12884
13925
|
type: "info",
|
|
12885
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13926
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12886
13927
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12887
13928
|
metadata: {
|
|
12888
13929
|
outcome: "reclaim-skipped",
|
|
12889
13930
|
reason: reclaimResult.skipped,
|
|
12890
13931
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12891
|
-
|
|
12892
|
-
sessions_deleted: deleted,
|
|
12893
|
-
sessions_failed: failed,
|
|
12894
|
-
sessions_skipped_active: skippedNewlyActive
|
|
13932
|
+
...sessionMetadata
|
|
12895
13933
|
}
|
|
12896
13934
|
});
|
|
12897
13935
|
}
|
|
@@ -13148,6 +14186,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13148
14186
|
message: `Resource usage collection: ${warning2}`
|
|
13149
14187
|
});
|
|
13150
14188
|
}
|
|
14189
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14190
|
+
...usage,
|
|
14191
|
+
nowMs: Date.now(),
|
|
14192
|
+
state: state.lowDiskWarningState
|
|
14193
|
+
});
|
|
14194
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14195
|
+
if (lowDiskDecision.warning !== null) {
|
|
14196
|
+
logActivity(state, {
|
|
14197
|
+
type: "info",
|
|
14198
|
+
level: "warn",
|
|
14199
|
+
message: lowDiskDecision.warning
|
|
14200
|
+
});
|
|
14201
|
+
}
|
|
13151
14202
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13152
14203
|
if (result.ok) {
|
|
13153
14204
|
if (consecutiveFailures > 0) {
|
|
@@ -13394,6 +14445,7 @@ async function run(options) {
|
|
|
13394
14445
|
opencodeClient: null,
|
|
13395
14446
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13396
14447
|
sessionDbProvenanceAnomaly: false,
|
|
14448
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13397
14449
|
opencodeProcess: null,
|
|
13398
14450
|
stopOpenCodeLogTail: null,
|
|
13399
14451
|
litestreamProcess: null,
|
|
@@ -13411,6 +14463,7 @@ async function run(options) {
|
|
|
13411
14463
|
openaiUsageRearm: null,
|
|
13412
14464
|
resourceUsageTimer: null,
|
|
13413
14465
|
stopResourceUsageSampling: null,
|
|
14466
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13414
14467
|
credentialSync: null,
|
|
13415
14468
|
authHeader: ""
|
|
13416
14469
|
};
|
|
@@ -13584,6 +14637,34 @@ async function run(options) {
|
|
|
13584
14637
|
} else {
|
|
13585
14638
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13586
14639
|
}
|
|
14640
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14641
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14642
|
+
log2(state, message, level);
|
|
14643
|
+
if (state.interactive && !state.json) {
|
|
14644
|
+
logActivity(state, { type: "info", level, message });
|
|
14645
|
+
}
|
|
14646
|
+
};
|
|
14647
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14648
|
+
if (vaultResolution.error) {
|
|
14649
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14650
|
+
logCredentialVaultResolution(
|
|
14651
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14652
|
+
"warn"
|
|
14653
|
+
);
|
|
14654
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14655
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14656
|
+
logCredentialVaultResolution(
|
|
14657
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14658
|
+
"info"
|
|
14659
|
+
);
|
|
14660
|
+
} else {
|
|
14661
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14662
|
+
logCredentialVaultResolution(
|
|
14663
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14664
|
+
"debug"
|
|
14665
|
+
);
|
|
14666
|
+
}
|
|
14667
|
+
}
|
|
13587
14668
|
if (options.restoreRunnerCredentials) {
|
|
13588
14669
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13589
14670
|
const credentialContext = {
|
|
@@ -13601,10 +14682,18 @@ async function run(options) {
|
|
|
13601
14682
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13602
14683
|
}
|
|
13603
14684
|
state.credentialSync?.arm();
|
|
13604
|
-
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13605
|
-
|
|
13606
|
-
|
|
13607
|
-
|
|
14685
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14686
|
+
() => ({
|
|
14687
|
+
agentId: state.agentId,
|
|
14688
|
+
authHeader: state.authHeader
|
|
14689
|
+
}),
|
|
14690
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14691
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14692
|
+
if (cause !== null && sessionId !== null) {
|
|
14693
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14694
|
+
}
|
|
14695
|
+
}
|
|
14696
|
+
);
|
|
13608
14697
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13609
14698
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13610
14699
|
openCodeActivityForwarder
|
|
@@ -13614,22 +14703,27 @@ async function run(options) {
|
|
|
13614
14703
|
openCodeLogTail.stop();
|
|
13615
14704
|
};
|
|
13616
14705
|
let sessionDbVerifyFatal = false;
|
|
14706
|
+
let sessionDbVerifyFatalReason;
|
|
13617
14707
|
if (!options.restoreSessionDb) {
|
|
13618
|
-
|
|
14708
|
+
logActivity(state, {
|
|
14709
|
+
type: "info",
|
|
14710
|
+
level: "warn",
|
|
14711
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14712
|
+
});
|
|
13619
14713
|
} else {
|
|
13620
14714
|
const health = await checkOpenCodeHealth(state.port);
|
|
13621
14715
|
if (health.healthy) {
|
|
13622
|
-
|
|
13623
|
-
|
|
13624
|
-
|
|
13625
|
-
"
|
|
13626
|
-
);
|
|
14716
|
+
logActivity(state, {
|
|
14717
|
+
type: "info",
|
|
14718
|
+
level: "warn",
|
|
14719
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14720
|
+
});
|
|
13627
14721
|
} else if (health.authFailed) {
|
|
13628
|
-
|
|
13629
|
-
|
|
13630
|
-
|
|
13631
|
-
"
|
|
13632
|
-
);
|
|
14722
|
+
logActivity(state, {
|
|
14723
|
+
type: "info",
|
|
14724
|
+
level: "warn",
|
|
14725
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14726
|
+
});
|
|
13633
14727
|
} else {
|
|
13634
14728
|
const result = await restoreAndVerifySessionDb({
|
|
13635
14729
|
dbPath: sessionDbPath(),
|
|
@@ -13646,12 +14740,13 @@ async function run(options) {
|
|
|
13646
14740
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13647
14741
|
});
|
|
13648
14742
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14743
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13649
14744
|
}
|
|
13650
14745
|
}
|
|
13651
14746
|
reportSessionDbRecovery(state);
|
|
13652
14747
|
if (sessionDbVerifyFatal) {
|
|
13653
14748
|
throw new Error(
|
|
13654
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14749
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13655
14750
|
);
|
|
13656
14751
|
}
|
|
13657
14752
|
applyRunnerOpenCodeConfig({
|
|
@@ -13896,6 +14991,7 @@ async function run(options) {
|
|
|
13896
14991
|
getAuthHeader: () => state.authHeader,
|
|
13897
14992
|
conversationFilter: state.conversationFilter,
|
|
13898
14993
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14994
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13899
14995
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13900
14996
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13901
14997
|
fileSyncDirectories,
|
|
@@ -13977,14 +15073,14 @@ async function run(options) {
|
|
|
13977
15073
|
},
|
|
13978
15074
|
onDisconnected: (code, reason, transition) => {
|
|
13979
15075
|
state.connected = false;
|
|
13980
|
-
const
|
|
15076
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13981
15077
|
logActivity(state, {
|
|
13982
15078
|
type: "info",
|
|
13983
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15079
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
13984
15080
|
});
|
|
13985
15081
|
emitAgentDisconnected(state.agentId, {
|
|
13986
15082
|
code,
|
|
13987
|
-
reason:
|
|
15083
|
+
reason: safeReason2,
|
|
13988
15084
|
connection_generation: transition.generation
|
|
13989
15085
|
});
|
|
13990
15086
|
if (state.interactive) displayStatus(state);
|
|
@@ -14129,7 +15225,7 @@ async function run(options) {
|
|
|
14129
15225
|
}
|
|
14130
15226
|
|
|
14131
15227
|
// src/index.ts
|
|
14132
|
-
var
|
|
15228
|
+
var version = getCliVersion();
|
|
14133
15229
|
var program = new Command();
|
|
14134
15230
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
14135
15231
|
"--endpoint <url>",
|
|
@@ -14147,6 +15243,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14147
15243
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14148
15244
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14149
15245
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15246
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14150
15247
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14151
15248
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14152
15249
|
"-a, --agent [id]",
|