@evident-ai/cli 3.5.2-dev.817a271 → 3.5.2-dev.8b315dd
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1688 -498
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -703,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
703
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
704
704
|
});
|
|
705
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
706
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
707
732
|
const apiUrl = getApiUrlConfig();
|
|
708
733
|
try {
|
|
@@ -877,6 +902,76 @@ async function status(options = {}) {
|
|
|
877
902
|
process.exit(result.exitCode);
|
|
878
903
|
}
|
|
879
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// src/lib/claude-usage.ts
|
|
881
976
|
import { execFileSync } from "node:child_process";
|
|
882
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1069,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1069
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1070
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1071
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1072
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1073
1238
|
var TelemetryEventTypes = {
|
|
1074
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1157,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1157
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1158
1323
|
|
|
1159
1324
|
// src/lib/telemetry.ts
|
|
1160
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.8b315dd" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1161
1326
|
function getCliVersion() {
|
|
1162
1327
|
return CLI_VERSION;
|
|
1163
1328
|
}
|
|
@@ -1318,8 +1483,316 @@ var EventTypes = {
|
|
|
1318
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1319
1484
|
};
|
|
1320
1485
|
|
|
1321
|
-
// src/lib/
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1322
1491
|
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
function errorMessage(error2) {
|
|
1522
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1523
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1524
|
+
}
|
|
1525
|
+
function errorMessages(error2) {
|
|
1526
|
+
const messages = [];
|
|
1527
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1528
|
+
let current = error2;
|
|
1529
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1530
|
+
visited.add(current);
|
|
1531
|
+
if (typeof current === "string") {
|
|
1532
|
+
messages.push(current);
|
|
1533
|
+
break;
|
|
1534
|
+
}
|
|
1535
|
+
if (current instanceof Error) {
|
|
1536
|
+
messages.push(current.name, current.message);
|
|
1537
|
+
current = current.cause;
|
|
1538
|
+
continue;
|
|
1539
|
+
}
|
|
1540
|
+
if (typeof current === "object") {
|
|
1541
|
+
const record = current;
|
|
1542
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1543
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1544
|
+
current = record.cause;
|
|
1545
|
+
continue;
|
|
1546
|
+
}
|
|
1547
|
+
break;
|
|
1548
|
+
}
|
|
1549
|
+
return messages;
|
|
1550
|
+
}
|
|
1551
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1552
|
+
return errorMessages(error2).some(
|
|
1553
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1554
|
+
message
|
|
1555
|
+
)
|
|
1556
|
+
);
|
|
1557
|
+
}
|
|
1558
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1559
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1560
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1561
|
+
function isCauseBearingText(text) {
|
|
1562
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1563
|
+
}
|
|
1564
|
+
function structuredLogFields(line) {
|
|
1565
|
+
const fields = [];
|
|
1566
|
+
for (let index = 0; index < line.length; index++) {
|
|
1567
|
+
if (line[index] === '"') {
|
|
1568
|
+
index++;
|
|
1569
|
+
while (index < line.length) {
|
|
1570
|
+
if (line[index] === "\\") {
|
|
1571
|
+
index++;
|
|
1572
|
+
} else if (line[index] === '"') {
|
|
1573
|
+
break;
|
|
1574
|
+
}
|
|
1575
|
+
index++;
|
|
1576
|
+
}
|
|
1577
|
+
continue;
|
|
1578
|
+
}
|
|
1579
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1580
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1581
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1582
|
+
const valueStart = index + key.length;
|
|
1583
|
+
if (line[valueStart] !== '"') {
|
|
1584
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1585
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1586
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
let valueEnd = valueStart + 1;
|
|
1590
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1591
|
+
if (line[valueEnd] === "\\") {
|
|
1592
|
+
valueEnd++;
|
|
1593
|
+
} else if (line[valueEnd] === '"') {
|
|
1594
|
+
break;
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1598
|
+
index = valueEnd;
|
|
1599
|
+
}
|
|
1600
|
+
return fields;
|
|
1601
|
+
}
|
|
1602
|
+
function structuredCauseMessages(text) {
|
|
1603
|
+
const fields = structuredLogFields(text);
|
|
1604
|
+
if (fields.length > 0) return fields;
|
|
1605
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1606
|
+
}
|
|
1607
|
+
function causeTokenFromText(text) {
|
|
1608
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1609
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1610
|
+
const match = pattern.exec(text);
|
|
1611
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1612
|
+
return match[1];
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
return null;
|
|
1616
|
+
}
|
|
1617
|
+
function sessionStoreCauseToken(value) {
|
|
1618
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1619
|
+
const token = causeTokenFromText(message);
|
|
1620
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1621
|
+
}
|
|
1622
|
+
return null;
|
|
1623
|
+
}
|
|
1624
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1625
|
+
try {
|
|
1626
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1627
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1628
|
+
if (availableBytes < requiredBytes) {
|
|
1629
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1630
|
+
}
|
|
1631
|
+
return null;
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
function readLogicalBytes(db) {
|
|
1637
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1638
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1639
|
+
return pageCount * pageSize;
|
|
1640
|
+
}
|
|
1641
|
+
function readFreelistCount(db) {
|
|
1642
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1643
|
+
}
|
|
1644
|
+
function readCheckpointResult(db) {
|
|
1645
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1646
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1647
|
+
}
|
|
1648
|
+
async function probeReclaimAvailability(input) {
|
|
1649
|
+
const { dbPath, requiredBytes } = input;
|
|
1650
|
+
let sqlite;
|
|
1651
|
+
try {
|
|
1652
|
+
sqlite = await import("node:sqlite");
|
|
1653
|
+
} catch (err) {
|
|
1654
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1655
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1656
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1657
|
+
}
|
|
1658
|
+
let autoVacuum = null;
|
|
1659
|
+
try {
|
|
1660
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1661
|
+
try {
|
|
1662
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1663
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1664
|
+
} finally {
|
|
1665
|
+
db.close();
|
|
1666
|
+
}
|
|
1667
|
+
} catch (err) {
|
|
1668
|
+
console.warn(
|
|
1669
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
if (autoVacuum !== 0) return null;
|
|
1673
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1674
|
+
}
|
|
1675
|
+
async function reclaimSessionDbSpace(input) {
|
|
1676
|
+
const {
|
|
1677
|
+
dbPath,
|
|
1678
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1679
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1680
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1681
|
+
now = () => performance.now(),
|
|
1682
|
+
allowFullVacuum = true
|
|
1683
|
+
} = input;
|
|
1684
|
+
let sqlite;
|
|
1685
|
+
try {
|
|
1686
|
+
sqlite = await import("node:sqlite");
|
|
1687
|
+
} catch (err) {
|
|
1688
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1689
|
+
console.warn(
|
|
1690
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1691
|
+
);
|
|
1692
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1693
|
+
}
|
|
1694
|
+
const { DatabaseSync } = sqlite;
|
|
1695
|
+
let db;
|
|
1696
|
+
try {
|
|
1697
|
+
db = new DatabaseSync(dbPath);
|
|
1698
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1699
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1700
|
+
if (autoVacuum === 0) {
|
|
1701
|
+
if (!allowFullVacuum) {
|
|
1702
|
+
console.warn(
|
|
1703
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1704
|
+
);
|
|
1705
|
+
return {
|
|
1706
|
+
ok: false,
|
|
1707
|
+
skipped: "full-vacuum-blocked",
|
|
1708
|
+
detail: "a session turn is live"
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1712
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1713
|
+
if (skipReason !== null) {
|
|
1714
|
+
console.warn(
|
|
1715
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1716
|
+
);
|
|
1717
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1718
|
+
}
|
|
1719
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1720
|
+
const freelistBefore = readFreelistCount(db);
|
|
1721
|
+
const startedAt = now();
|
|
1722
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1723
|
+
db.exec("VACUUM");
|
|
1724
|
+
const afterBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistAfter = readFreelistCount(db);
|
|
1726
|
+
const checkpoint = readCheckpointResult(db);
|
|
1727
|
+
return {
|
|
1728
|
+
ok: true,
|
|
1729
|
+
mode: "convert",
|
|
1730
|
+
beforeBytes,
|
|
1731
|
+
afterBytes,
|
|
1732
|
+
freelistBefore,
|
|
1733
|
+
freelistAfter,
|
|
1734
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1735
|
+
stoppedBy: "freelist-drained",
|
|
1736
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1737
|
+
checkpoint
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
if (autoVacuum === 2) {
|
|
1741
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1742
|
+
const freelistBefore = readFreelistCount(db);
|
|
1743
|
+
const startedAt = now();
|
|
1744
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1745
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1746
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1747
|
+
let pagesAttempted = 0;
|
|
1748
|
+
let freelistAfter = freelistBefore;
|
|
1749
|
+
let stoppedBy = "freelist-drained";
|
|
1750
|
+
while (freelistAfter > 0) {
|
|
1751
|
+
if (pagesAttempted >= pageLimit) {
|
|
1752
|
+
stoppedBy = "page-ceiling";
|
|
1753
|
+
break;
|
|
1754
|
+
}
|
|
1755
|
+
if (now() - startedAt >= budget) {
|
|
1756
|
+
stoppedBy = "time-budget";
|
|
1757
|
+
break;
|
|
1758
|
+
}
|
|
1759
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1760
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1761
|
+
pagesAttempted += pages;
|
|
1762
|
+
freelistAfter = readFreelistCount(db);
|
|
1763
|
+
}
|
|
1764
|
+
const afterBytes = readLogicalBytes(db);
|
|
1765
|
+
const checkpoint = readCheckpointResult(db);
|
|
1766
|
+
return {
|
|
1767
|
+
ok: true,
|
|
1768
|
+
mode: "incremental",
|
|
1769
|
+
beforeBytes,
|
|
1770
|
+
afterBytes,
|
|
1771
|
+
freelistBefore,
|
|
1772
|
+
freelistAfter,
|
|
1773
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1774
|
+
stoppedBy,
|
|
1775
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1776
|
+
checkpoint
|
|
1777
|
+
};
|
|
1778
|
+
}
|
|
1779
|
+
console.warn(
|
|
1780
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1781
|
+
);
|
|
1782
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1783
|
+
} catch (err) {
|
|
1784
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1785
|
+
return {
|
|
1786
|
+
ok: false,
|
|
1787
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1788
|
+
detail: errorMessage(err)
|
|
1789
|
+
};
|
|
1790
|
+
} finally {
|
|
1791
|
+
db?.close();
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1323
1796
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1324
1797
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1325
1798
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1336,11 +1809,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1336
1809
|
if (!levelMatch) return null;
|
|
1337
1810
|
const level = levelMatch[1].toUpperCase();
|
|
1338
1811
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1339
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1812
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1813
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1340
1814
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1341
1815
|
const parsed = {
|
|
1342
1816
|
level: level === "WARN" ? "warn" : "error",
|
|
1343
|
-
sessionID: sessionMatch?.[1]
|
|
1817
|
+
sessionID: sessionMatch?.[1],
|
|
1818
|
+
messageID: messageIDMatch?.[1]
|
|
1344
1819
|
};
|
|
1345
1820
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1346
1821
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1348,12 +1823,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1348
1823
|
}
|
|
1349
1824
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1350
1825
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1351
|
-
function createOpenCodeActivityForwarder(getContext) {
|
|
1826
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1352
1827
|
let buffer = Buffer.alloc(0);
|
|
1353
1828
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1354
1829
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1355
1830
|
const metadata = {
|
|
1356
1831
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1832
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1357
1833
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1358
1834
|
};
|
|
1359
1835
|
forwardRunnerActivity(
|
|
@@ -1379,9 +1855,24 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1379
1855
|
};
|
|
1380
1856
|
const flushLine = (line) => {
|
|
1381
1857
|
const parsed = parseOpenCodeLogLine(line);
|
|
1858
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1859
|
+
const cause = sessionStoreCauseToken(line);
|
|
1860
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1861
|
+
try {
|
|
1862
|
+
onSessionStoreWriteFailure({
|
|
1863
|
+
isDiskFailure,
|
|
1864
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1865
|
+
cause
|
|
1866
|
+
});
|
|
1867
|
+
} catch (err) {
|
|
1868
|
+
reportTelemetryDiagnostic(
|
|
1869
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1382
1873
|
if (!parsed) return;
|
|
1383
1874
|
const now = Date.now();
|
|
1384
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1875
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1385
1876
|
expireRepeatStates(now, key);
|
|
1386
1877
|
if (key === void 0) {
|
|
1387
1878
|
forwardLine(line, parsed);
|
|
@@ -1427,34 +1918,21 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1427
1918
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1428
1919
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1429
1920
|
var MAX_METADATA_ENTRIES = 20;
|
|
1430
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1431
|
-
function redactHomeDirectory(message) {
|
|
1432
|
-
const home = homedir2();
|
|
1433
|
-
if (!home || home === "/") {
|
|
1434
|
-
return message;
|
|
1435
|
-
}
|
|
1436
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1437
|
-
let result = "";
|
|
1438
|
-
let searchFrom = 0;
|
|
1439
|
-
let homeIndex;
|
|
1440
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1441
|
-
const homeEnd = homeIndex + home.length;
|
|
1442
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1443
|
-
const nextCharacter = message[homeEnd];
|
|
1444
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1445
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1446
|
-
result += atPathBoundary ? "~" : home;
|
|
1447
|
-
searchFrom = homeEnd;
|
|
1448
|
-
}
|
|
1449
|
-
return result + message.slice(searchFrom);
|
|
1450
|
-
}
|
|
1451
|
-
function redactDiagnosticText(message) {
|
|
1452
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1453
|
-
return redactHomeDirectory(redacted);
|
|
1454
|
-
}
|
|
1455
1921
|
function truncate(message) {
|
|
1456
1922
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1457
|
-
|
|
1923
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1924
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1925
|
+
let previousDropped = -1;
|
|
1926
|
+
while (dropped !== previousDropped) {
|
|
1927
|
+
previousDropped = dropped;
|
|
1928
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1929
|
+
dropped = nextDropped;
|
|
1930
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1931
|
+
}
|
|
1932
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1933
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1934
|
+
const tailLength = remaining - headLength;
|
|
1935
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1458
1936
|
}
|
|
1459
1937
|
function sanitiseMetadata(metadata) {
|
|
1460
1938
|
if (!metadata) return {};
|
|
@@ -1664,12 +2142,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1664
2142
|
metadata: withoutContractFields(record),
|
|
1665
2143
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1666
2144
|
};
|
|
1667
|
-
case "session_db_boot_refused":
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
2145
|
+
case "session_db_boot_refused": {
|
|
2146
|
+
const message = (() => {
|
|
2147
|
+
switch (record.reason) {
|
|
2148
|
+
case "local_discard_failed":
|
|
2149
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2150
|
+
case "replica_separation_unproven":
|
|
2151
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2152
|
+
case "verification_timeout_before_separation":
|
|
2153
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2154
|
+
case "verification_timeout_during_separation":
|
|
2155
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2156
|
+
default:
|
|
2157
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2158
|
+
}
|
|
2159
|
+
})();
|
|
2160
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2161
|
+
}
|
|
1673
2162
|
case "schema_provenance_mismatch":
|
|
1674
2163
|
return {
|
|
1675
2164
|
level,
|
|
@@ -1919,7 +2408,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1919
2408
|
return null;
|
|
1920
2409
|
}
|
|
1921
2410
|
}
|
|
1922
|
-
function
|
|
2411
|
+
function roleOf2(m) {
|
|
1923
2412
|
if (!m || typeof m !== "object") return void 0;
|
|
1924
2413
|
if (typeof m.role === "string") return m.role;
|
|
1925
2414
|
const infoRole = m.info?.role;
|
|
@@ -1933,13 +2422,13 @@ function createdOf(m) {
|
|
|
1933
2422
|
if (!m || typeof m !== "object") return void 0;
|
|
1934
2423
|
return m.info?.time?.created ?? m.time?.created;
|
|
1935
2424
|
}
|
|
1936
|
-
function
|
|
2425
|
+
function idOf2(m) {
|
|
1937
2426
|
if (!m || typeof m !== "object") return void 0;
|
|
1938
2427
|
if (typeof m.id === "string") return m.id;
|
|
1939
2428
|
const infoId = m.info?.id;
|
|
1940
2429
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1941
2430
|
}
|
|
1942
|
-
function
|
|
2431
|
+
function parentIdOf2(m) {
|
|
1943
2432
|
if (!m || typeof m !== "object") return void 0;
|
|
1944
2433
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1945
2434
|
const infoParent = m.info?.parentID;
|
|
@@ -1955,8 +2444,12 @@ function errorOf(m) {
|
|
|
1955
2444
|
if (!m || typeof m !== "object") return void 0;
|
|
1956
2445
|
return m.info?.error ?? m.error;
|
|
1957
2446
|
}
|
|
2447
|
+
function isTerminalStopAnswer(m) {
|
|
2448
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2449
|
+
}
|
|
1958
2450
|
function isAssistantInFlight(m) {
|
|
1959
2451
|
if (completedOf(m) == null) return true;
|
|
2452
|
+
if (errorOf(m) != null) return false;
|
|
1960
2453
|
return finishOf(m) === "tool-calls";
|
|
1961
2454
|
}
|
|
1962
2455
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2004,9 +2497,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2004
2497
|
function isSessionActivelyGenerating(messages) {
|
|
2005
2498
|
if (!messages || messages.length === 0) return false;
|
|
2006
2499
|
const last = messages[messages.length - 1];
|
|
2007
|
-
if (
|
|
2500
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2008
2501
|
return completedOf(last) == null;
|
|
2009
2502
|
}
|
|
2503
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2010
2504
|
function sessionLastActivityMs(session) {
|
|
2011
2505
|
const candidates = [
|
|
2012
2506
|
session.time?.updated,
|
|
@@ -2021,30 +2515,71 @@ function sessionLastActivityMs(session) {
|
|
|
2021
2515
|
}
|
|
2022
2516
|
return null;
|
|
2023
2517
|
}
|
|
2024
|
-
|
|
2025
|
-
|
|
2518
|
+
function sessionListPath(options) {
|
|
2519
|
+
const params = new URLSearchParams();
|
|
2520
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2521
|
+
const query = params.toString();
|
|
2522
|
+
return query ? `/session?${query}` : "/session";
|
|
2523
|
+
}
|
|
2524
|
+
async function listSessionPage(port, client, options) {
|
|
2525
|
+
const path = sessionListPath(options);
|
|
2026
2526
|
try {
|
|
2027
|
-
const res = await requestWithClient(port, client,
|
|
2527
|
+
const res = await requestWithClient(port, client, path);
|
|
2028
2528
|
if (!res.ok) return null;
|
|
2029
2529
|
const body = await res.json();
|
|
2030
2530
|
return Array.isArray(body) ? body : null;
|
|
2031
2531
|
} catch (error2) {
|
|
2032
2532
|
console.error(
|
|
2033
|
-
`[
|
|
2533
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2034
2534
|
);
|
|
2035
2535
|
return null;
|
|
2036
2536
|
}
|
|
2037
2537
|
}
|
|
2538
|
+
async function listSessions(port, client, options) {
|
|
2539
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2540
|
+
return listSessionPage(port, client, options);
|
|
2541
|
+
}
|
|
2542
|
+
async function listAllSessions(port, client) {
|
|
2543
|
+
if (client?.version === "v2") {
|
|
2544
|
+
const sessions2 = await listV2Sessions(client);
|
|
2545
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2546
|
+
}
|
|
2547
|
+
const sessions = await listSessionPage(port, client, {
|
|
2548
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2549
|
+
});
|
|
2550
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2551
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2552
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2553
|
+
}
|
|
2554
|
+
return { sessions };
|
|
2555
|
+
}
|
|
2038
2556
|
async function deleteSession(port, id, client) {
|
|
2039
2557
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2040
2558
|
try {
|
|
2041
|
-
const res = await requestWithClient(
|
|
2042
|
-
|
|
2559
|
+
const res = await requestWithClient(
|
|
2560
|
+
port,
|
|
2561
|
+
client,
|
|
2562
|
+
`/session/${id}`,
|
|
2563
|
+
{ method: "DELETE" },
|
|
2564
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2565
|
+
);
|
|
2566
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2567
|
+
return { ok: true, status: res.status };
|
|
2568
|
+
}
|
|
2569
|
+
try {
|
|
2570
|
+
const body = await res.text();
|
|
2571
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2572
|
+
} catch (error2) {
|
|
2573
|
+
console.error(
|
|
2574
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2575
|
+
);
|
|
2576
|
+
return { ok: false, status: res.status };
|
|
2577
|
+
}
|
|
2043
2578
|
} catch (error2) {
|
|
2044
2579
|
console.error(
|
|
2045
2580
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2046
2581
|
);
|
|
2047
|
-
return false;
|
|
2582
|
+
return { ok: false, status: "threw" };
|
|
2048
2583
|
}
|
|
2049
2584
|
}
|
|
2050
2585
|
async function sessionExists(port, id, client) {
|
|
@@ -2282,11 +2817,31 @@ function messageText(m) {
|
|
|
2282
2817
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2283
2818
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2284
2819
|
}
|
|
2285
|
-
|
|
2820
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2821
|
+
if (!messages || messages.length === 0) return null;
|
|
2822
|
+
let best = null;
|
|
2823
|
+
for (const message of messages) {
|
|
2824
|
+
if (roleOf2(message) !== "user") continue;
|
|
2825
|
+
const id = idOf2(message);
|
|
2826
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2827
|
+
if (messageText(message) !== content) continue;
|
|
2828
|
+
const created = createdOf(message) ?? 0;
|
|
2829
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2830
|
+
}
|
|
2831
|
+
return best?.id ?? null;
|
|
2832
|
+
}
|
|
2833
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2286
2834
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2287
2835
|
const knownUserIds = new Set(
|
|
2288
|
-
(before ?? []).filter((m) =>
|
|
2836
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2289
2837
|
);
|
|
2838
|
+
if (callbacks) {
|
|
2839
|
+
try {
|
|
2840
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2841
|
+
} catch (error2) {
|
|
2842
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2290
2845
|
const parts = [{ type: "text", text: content }];
|
|
2291
2846
|
let pendingOutcomes = null;
|
|
2292
2847
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2325,22 +2880,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2325
2880
|
const READ_BACK_DELAY_MS = 150;
|
|
2326
2881
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2327
2882
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
const id = idOf(m);
|
|
2333
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2334
|
-
if (messageText(m) !== content) continue;
|
|
2335
|
-
const created = createdOf(m) ?? 0;
|
|
2336
|
-
if (best === null || created > best.created) {
|
|
2337
|
-
best = { id, created };
|
|
2338
|
-
}
|
|
2339
|
-
}
|
|
2340
|
-
if (best) {
|
|
2341
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2342
|
-
return best.id;
|
|
2343
|
-
}
|
|
2883
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2884
|
+
if (dispatchedUserMessageId !== null) {
|
|
2885
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2886
|
+
return dispatchedUserMessageId;
|
|
2344
2887
|
}
|
|
2345
2888
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2346
2889
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2351,57 +2894,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2351
2894
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2352
2895
|
if (!messages || messages.length === 0) return null;
|
|
2353
2896
|
const byParent = messages.find(
|
|
2354
|
-
(m) =>
|
|
2897
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2355
2898
|
);
|
|
2356
2899
|
if (byParent) return byParent;
|
|
2357
|
-
const userIndex = messages.findIndex((m) =>
|
|
2900
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2358
2901
|
if (userIndex === -1) return null;
|
|
2359
2902
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2360
|
-
if (
|
|
2903
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2361
2904
|
}
|
|
2362
2905
|
return null;
|
|
2363
2906
|
}
|
|
2364
2907
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2365
2908
|
if (!messages || messages.length === 0) return null;
|
|
2366
2909
|
let lastCorrelated = null;
|
|
2367
|
-
let lastNonErrored = null;
|
|
2368
2910
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2369
2911
|
const m = messages[i];
|
|
2370
|
-
if (
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2912
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2913
|
+
lastCorrelated = m;
|
|
2914
|
+
break;
|
|
2915
|
+
}
|
|
2916
|
+
if (lastCorrelated) {
|
|
2917
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2918
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2919
|
+
const m = messages[i];
|
|
2920
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2921
|
+
return m;
|
|
2922
|
+
}
|
|
2375
2923
|
}
|
|
2924
|
+
return lastCorrelated;
|
|
2376
2925
|
}
|
|
2377
|
-
|
|
2378
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2926
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2379
2927
|
if (userIndex === -1) return null;
|
|
2380
2928
|
let last = null;
|
|
2381
|
-
let
|
|
2929
|
+
let lastAssistantIndex = -1;
|
|
2382
2930
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2383
|
-
const role =
|
|
2931
|
+
const role = roleOf2(messages[i]);
|
|
2384
2932
|
if (role === "user") break;
|
|
2385
2933
|
if (role === "assistant") {
|
|
2386
2934
|
last = messages[i];
|
|
2387
|
-
|
|
2935
|
+
lastAssistantIndex = i;
|
|
2388
2936
|
}
|
|
2389
2937
|
}
|
|
2390
|
-
|
|
2938
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2939
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2940
|
+
const m = messages[i];
|
|
2941
|
+
if (roleOf2(m) === "user") break;
|
|
2942
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2943
|
+
}
|
|
2944
|
+
return last;
|
|
2391
2945
|
}
|
|
2392
2946
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2393
2947
|
if (!messages || messages.length === 0) return [];
|
|
2394
2948
|
const byParent = messages.filter(
|
|
2395
|
-
(message) =>
|
|
2949
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2396
2950
|
);
|
|
2397
2951
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2398
2952
|
if (assistants.length === 0) {
|
|
2399
|
-
const userIndex = messages.findIndex((message) =>
|
|
2953
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2400
2954
|
if (userIndex === -1) return [];
|
|
2401
2955
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2402
2956
|
const message = messages[i];
|
|
2403
|
-
if (
|
|
2404
|
-
if (
|
|
2957
|
+
if (roleOf2(message) === "user") break;
|
|
2958
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2405
2959
|
}
|
|
2406
2960
|
}
|
|
2407
2961
|
const refs = [];
|
|
@@ -2441,7 +2995,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2441
2995
|
if (!messages || messages.length === 0) return [];
|
|
2442
2996
|
const calls = [];
|
|
2443
2997
|
for (const message of messages) {
|
|
2444
|
-
if (
|
|
2998
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2445
2999
|
for (const part of message.parts ?? []) {
|
|
2446
3000
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2447
3001
|
continue;
|
|
@@ -2471,7 +3025,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2471
3025
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2472
3026
|
const unattributed = [];
|
|
2473
3027
|
for (const message of messages ?? []) {
|
|
2474
|
-
if (
|
|
3028
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2475
3029
|
const created = finiteNumber(createdOf(message));
|
|
2476
3030
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2477
3031
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2493,8 +3047,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2493
3047
|
}
|
|
2494
3048
|
function sumAssistantUsage(messages) {
|
|
2495
3049
|
if (!messages || messages.length === 0) return null;
|
|
2496
|
-
const
|
|
2497
|
-
const
|
|
3050
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3051
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3052
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2498
3053
|
let sawAnyUsage = false;
|
|
2499
3054
|
let inputSum = 0;
|
|
2500
3055
|
let outputSum = 0;
|
|
@@ -2548,13 +3103,11 @@ function sumAssistantUsage(messages) {
|
|
|
2548
3103
|
function messageUsage(messages, userMessageId) {
|
|
2549
3104
|
if (!messages || messages.length === 0) return null;
|
|
2550
3105
|
const byParentAll = messages.filter(
|
|
2551
|
-
(m) =>
|
|
3106
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2552
3107
|
);
|
|
2553
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2554
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2555
3108
|
let correlated;
|
|
2556
|
-
if (
|
|
2557
|
-
correlated =
|
|
3109
|
+
if (byParentAll.length > 0) {
|
|
3110
|
+
correlated = byParentAll;
|
|
2558
3111
|
} else {
|
|
2559
3112
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2560
3113
|
correlated = reply ? [reply] : [];
|
|
@@ -2564,7 +3117,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2564
3117
|
}
|
|
2565
3118
|
function messageRunState(messages, userMessageId) {
|
|
2566
3119
|
if (!messages || messages.length === 0) return "unknown";
|
|
2567
|
-
const hasUser = messages.some((m) =>
|
|
3120
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2568
3121
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2569
3122
|
if (!hasUser) {
|
|
2570
3123
|
if (!reply) return "unknown";
|
|
@@ -2600,14 +3153,38 @@ function messageError(messages, userMessageId) {
|
|
|
2600
3153
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2601
3154
|
const error2 = errorOf(reply);
|
|
2602
3155
|
if (error2 == null) return null;
|
|
2603
|
-
|
|
2604
|
-
if (typeof error2 === "
|
|
3156
|
+
let rendered;
|
|
3157
|
+
if (typeof error2 === "string") {
|
|
3158
|
+
rendered = error2;
|
|
3159
|
+
} else if (typeof error2 === "object") {
|
|
2605
3160
|
const e = error2;
|
|
2606
3161
|
const dataMessage = e.data?.message;
|
|
2607
|
-
if (typeof dataMessage === "string")
|
|
2608
|
-
|
|
3162
|
+
if (typeof dataMessage === "string") {
|
|
3163
|
+
rendered = dataMessage;
|
|
3164
|
+
} else if (typeof e.message === "string") {
|
|
3165
|
+
rendered = e.message;
|
|
3166
|
+
} else {
|
|
3167
|
+
rendered = "The agent run failed.";
|
|
3168
|
+
}
|
|
3169
|
+
} else {
|
|
3170
|
+
rendered = "The agent run failed.";
|
|
2609
3171
|
}
|
|
2610
|
-
|
|
3172
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3173
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3174
|
+
) : void 0;
|
|
3175
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3176
|
+
}
|
|
3177
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3178
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3179
|
+
if (!reply) return null;
|
|
3180
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3181
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3182
|
+
return {
|
|
3183
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3184
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3185
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3186
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3187
|
+
};
|
|
2611
3188
|
}
|
|
2612
3189
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2613
3190
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2656,7 +3233,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2656
3233
|
if (!messages || messages.length === 0) return null;
|
|
2657
3234
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2658
3235
|
const message = messages[i];
|
|
2659
|
-
if (
|
|
3236
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2660
3237
|
const created = createdOf(message);
|
|
2661
3238
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2662
3239
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2688,25 +3265,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2688
3265
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2689
3266
|
if (!messages || messages.length === 0) return false;
|
|
2690
3267
|
return messages.some(
|
|
2691
|
-
(m) =>
|
|
3268
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2692
3269
|
);
|
|
2693
3270
|
}
|
|
2694
3271
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2695
3272
|
if (!messages || messages.length === 0) return false;
|
|
2696
|
-
const userIndex = messages.findIndex((message) =>
|
|
3273
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2697
3274
|
if (userIndex === -1) return false;
|
|
2698
3275
|
let hasLaterUser = false;
|
|
2699
3276
|
let hasStartedLaterUser = false;
|
|
2700
3277
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2701
3278
|
const message = messages[i];
|
|
2702
|
-
if (
|
|
3279
|
+
if (roleOf2(message) !== "user") continue;
|
|
2703
3280
|
hasLaterUser = true;
|
|
2704
|
-
const laterUserMessageId =
|
|
3281
|
+
const laterUserMessageId = idOf2(message);
|
|
2705
3282
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2706
3283
|
return false;
|
|
2707
3284
|
}
|
|
2708
3285
|
if (messages.some(
|
|
2709
|
-
(candidate) =>
|
|
3286
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2710
3287
|
)) {
|
|
2711
3288
|
hasStartedLaterUser = true;
|
|
2712
3289
|
}
|
|
@@ -3177,13 +3754,28 @@ async function listV2Sessions(client) {
|
|
|
3177
3754
|
}
|
|
3178
3755
|
async function deleteV2Session(client, sessionId) {
|
|
3179
3756
|
try {
|
|
3180
|
-
await client.request(
|
|
3181
|
-
|
|
3757
|
+
const response = await client.request(
|
|
3758
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3759
|
+
{ method: "DELETE" },
|
|
3760
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3761
|
+
);
|
|
3762
|
+
if (response.status >= 200 && response.status < 300) {
|
|
3763
|
+
return { ok: true, status: response.status };
|
|
3764
|
+
}
|
|
3765
|
+
try {
|
|
3766
|
+
const body = await response.text();
|
|
3767
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
3768
|
+
} catch (error2) {
|
|
3769
|
+
console.error(
|
|
3770
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3771
|
+
);
|
|
3772
|
+
return { ok: false, status: response.status };
|
|
3773
|
+
}
|
|
3182
3774
|
} catch (error2) {
|
|
3183
3775
|
console.error(
|
|
3184
3776
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3185
3777
|
);
|
|
3186
|
-
return false;
|
|
3778
|
+
return { ok: false, status: "threw" };
|
|
3187
3779
|
}
|
|
3188
3780
|
}
|
|
3189
3781
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3805,9 +4397,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3805
4397
|
|
|
3806
4398
|
// src/lib/opencode/session-db-boot.ts
|
|
3807
4399
|
import { spawn as spawn2 } from "node:child_process";
|
|
3808
|
-
import { mkdirSync, statSync as
|
|
4400
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3809
4401
|
import { homedir as homedir4 } from "node:os";
|
|
3810
|
-
import { dirname as
|
|
4402
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3811
4403
|
|
|
3812
4404
|
// src/lib/runner-synchroniser.ts
|
|
3813
4405
|
import { spawn } from "node:child_process";
|
|
@@ -3887,9 +4479,12 @@ function runSynchroniser(args, opts) {
|
|
|
3887
4479
|
|
|
3888
4480
|
// src/lib/opencode/session-db-boot.ts
|
|
3889
4481
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3890
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4482
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3891
4483
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3892
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4484
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4485
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4486
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4487
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3893
4488
|
function commandError(result) {
|
|
3894
4489
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3895
4490
|
}
|
|
@@ -3932,7 +4527,7 @@ function clearMarker(options) {
|
|
|
3932
4527
|
function markNoReplicate(options, message) {
|
|
3933
4528
|
if (options.noReplicateMarker) {
|
|
3934
4529
|
try {
|
|
3935
|
-
mkdirSync(
|
|
4530
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3936
4531
|
writeFileSync(options.noReplicateMarker, "");
|
|
3937
4532
|
} catch (error2) {
|
|
3938
4533
|
options.log(
|
|
@@ -4057,7 +4652,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4057
4652
|
return null;
|
|
4058
4653
|
}
|
|
4059
4654
|
try {
|
|
4060
|
-
if (
|
|
4655
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4061
4656
|
} catch (error2) {
|
|
4062
4657
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4063
4658
|
options.log(
|
|
@@ -4087,7 +4682,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4087
4682
|
return null;
|
|
4088
4683
|
}
|
|
4089
4684
|
try {
|
|
4090
|
-
mkdirSync(
|
|
4685
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
4091
4686
|
writeFileSync(configPath, rendered.stdout);
|
|
4092
4687
|
} catch (error2) {
|
|
4093
4688
|
options.log(
|
|
@@ -4200,20 +4795,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4200
4795
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4201
4796
|
env: {
|
|
4202
4797
|
...env,
|
|
4203
|
-
// The synchroniser reads
|
|
4204
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4798
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4205
4799
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4206
4800
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4801
|
+
),
|
|
4802
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4803
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4804
|
+
),
|
|
4805
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4806
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4807
|
+
),
|
|
4808
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4809
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4207
4810
|
)
|
|
4208
4811
|
}
|
|
4209
4812
|
});
|
|
4210
4813
|
logSynchroniserDiagnostics(result, options);
|
|
4211
4814
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4815
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4816
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4817
|
+
options.log(
|
|
4818
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4819
|
+
"error"
|
|
4820
|
+
);
|
|
4821
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4822
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4823
|
+
}
|
|
4212
4824
|
options.log(
|
|
4213
4825
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4214
4826
|
"warn"
|
|
4215
4827
|
);
|
|
4216
|
-
return false;
|
|
4828
|
+
return { verifyFatal: false };
|
|
4217
4829
|
}
|
|
4218
4830
|
if (result.code === 34) {
|
|
4219
4831
|
reportRecord(
|
|
@@ -4223,14 +4835,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4223
4835
|
null,
|
|
4224
4836
|
options
|
|
4225
4837
|
);
|
|
4226
|
-
return true;
|
|
4838
|
+
return { verifyFatal: true };
|
|
4227
4839
|
}
|
|
4228
4840
|
if (result.code === 33) {
|
|
4229
4841
|
options.log(
|
|
4230
4842
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4231
4843
|
"warn"
|
|
4232
4844
|
);
|
|
4233
|
-
return false;
|
|
4845
|
+
return { verifyFatal: false };
|
|
4234
4846
|
}
|
|
4235
4847
|
if (result.code !== 0) {
|
|
4236
4848
|
options.log(
|
|
@@ -4238,17 +4850,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4238
4850
|
"warn"
|
|
4239
4851
|
);
|
|
4240
4852
|
}
|
|
4241
|
-
return false;
|
|
4853
|
+
return { verifyFatal: false };
|
|
4242
4854
|
}
|
|
4243
4855
|
options.log(
|
|
4244
4856
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4245
4857
|
"debug"
|
|
4246
4858
|
);
|
|
4247
|
-
return false;
|
|
4859
|
+
return { verifyFatal: false };
|
|
4248
4860
|
}
|
|
4249
4861
|
function fileExists(path) {
|
|
4250
4862
|
try {
|
|
4251
|
-
|
|
4863
|
+
statSync3(path);
|
|
4252
4864
|
return true;
|
|
4253
4865
|
} catch (error2) {
|
|
4254
4866
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4331,13 +4943,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4331
4943
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4332
4944
|
return { verifyFatal: false };
|
|
4333
4945
|
}
|
|
4334
|
-
return
|
|
4946
|
+
return verifySessionDb(options, configPath, env);
|
|
4335
4947
|
}
|
|
4336
4948
|
|
|
4337
4949
|
// src/lib/opencode/session-db-provenance.ts
|
|
4338
4950
|
import { createRequire } from "node:module";
|
|
4339
4951
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4340
|
-
import { dirname as
|
|
4952
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4341
4953
|
var require2 = createRequire(import.meta.url);
|
|
4342
4954
|
function readSessionDbMigrationIds(dbPath) {
|
|
4343
4955
|
let db;
|
|
@@ -4403,7 +5015,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4403
5015
|
}
|
|
4404
5016
|
function saveSessionDbProvenanceState(path, state) {
|
|
4405
5017
|
try {
|
|
4406
|
-
mkdirSync2(
|
|
5018
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4407
5019
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4408
5020
|
`, "utf8");
|
|
4409
5021
|
} catch (error2) {
|
|
@@ -4984,7 +5596,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4984
5596
|
toDelete.push(s.id);
|
|
4985
5597
|
}
|
|
4986
5598
|
}
|
|
4987
|
-
|
|
5599
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5600
|
+
return toDelete.sort(
|
|
5601
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5602
|
+
);
|
|
4988
5603
|
}
|
|
4989
5604
|
var DEFAULT_INTERVAL = "1h";
|
|
4990
5605
|
function resolve(flag, envValue, fallback) {
|
|
@@ -5044,13 +5659,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5044
5659
|
}
|
|
5045
5660
|
|
|
5046
5661
|
// src/lib/opencode/session-db-size.ts
|
|
5047
|
-
import { statSync as
|
|
5662
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5048
5663
|
import { join as join6 } from "node:path";
|
|
5049
5664
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5050
5665
|
function statSessionDbBytes(homeDir) {
|
|
5051
5666
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5052
5667
|
try {
|
|
5053
|
-
return
|
|
5668
|
+
return statSync4(dbPath).size;
|
|
5054
5669
|
} catch (err) {
|
|
5055
5670
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5056
5671
|
if (!isMissingFile) {
|
|
@@ -5076,7 +5691,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5076
5691
|
}
|
|
5077
5692
|
|
|
5078
5693
|
// src/lib/opencode/log-tail.ts
|
|
5079
|
-
import { statSync as
|
|
5694
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5080
5695
|
import { homedir as homedir5 } from "node:os";
|
|
5081
5696
|
import { join as join7 } from "node:path";
|
|
5082
5697
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -5099,7 +5714,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5099
5714
|
let inode = null;
|
|
5100
5715
|
let baselineReady = true;
|
|
5101
5716
|
try {
|
|
5102
|
-
const initial =
|
|
5717
|
+
const initial = statSync5(logPath);
|
|
5103
5718
|
offset = initial.size;
|
|
5104
5719
|
inode = initial.ino;
|
|
5105
5720
|
} catch (error2) {
|
|
@@ -5108,177 +5723,54 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5108
5723
|
baselineReady = false;
|
|
5109
5724
|
}
|
|
5110
5725
|
}
|
|
5111
|
-
let polling = false;
|
|
5112
|
-
let stopped = false;
|
|
5113
|
-
const poll = async () => {
|
|
5114
|
-
if (polling || stopped) return;
|
|
5115
|
-
polling = true;
|
|
5116
|
-
try {
|
|
5117
|
-
let current;
|
|
5118
|
-
try {
|
|
5119
|
-
current = await stat(logPath);
|
|
5120
|
-
} catch (error2) {
|
|
5121
|
-
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5122
|
-
return;
|
|
5123
|
-
}
|
|
5124
|
-
if (!baselineReady) {
|
|
5125
|
-
offset = current.size;
|
|
5126
|
-
inode = current.ino;
|
|
5127
|
-
baselineReady = true;
|
|
5128
|
-
return;
|
|
5129
|
-
}
|
|
5130
|
-
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5131
|
-
offset = 0;
|
|
5132
|
-
}
|
|
5133
|
-
inode = current.ino;
|
|
5134
|
-
if (current.size === offset) return;
|
|
5135
|
-
const length = current.size - offset;
|
|
5136
|
-
const fh = await open2(logPath, "r");
|
|
5137
|
-
try {
|
|
5138
|
-
const buf = Buffer.alloc(length);
|
|
5139
|
-
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5140
|
-
offset += bytesRead;
|
|
5141
|
-
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5142
|
-
} finally {
|
|
5143
|
-
await fh.close();
|
|
5144
|
-
}
|
|
5145
|
-
} catch (error2) {
|
|
5146
|
-
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5147
|
-
} finally {
|
|
5148
|
-
polling = false;
|
|
5149
|
-
}
|
|
5150
|
-
};
|
|
5151
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5152
|
-
void poll();
|
|
5153
|
-
return {
|
|
5154
|
-
stop: () => {
|
|
5155
|
-
stopped = true;
|
|
5156
|
-
clearInterval(interval);
|
|
5157
|
-
}
|
|
5158
|
-
};
|
|
5159
|
-
}
|
|
5160
|
-
|
|
5161
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5162
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5163
|
-
import { dirname as dirname4 } from "node:path";
|
|
5164
|
-
function errorMessage(error2) {
|
|
5165
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5166
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5167
|
-
}
|
|
5168
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5169
|
-
try {
|
|
5170
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5171
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5172
|
-
if (availableBytes < requiredBytes) {
|
|
5173
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5174
|
-
}
|
|
5175
|
-
return null;
|
|
5176
|
-
} catch (err) {
|
|
5177
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5178
|
-
}
|
|
5179
|
-
}
|
|
5180
|
-
function readLogicalBytes(db) {
|
|
5181
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5182
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5183
|
-
return pageCount * pageSize;
|
|
5184
|
-
}
|
|
5185
|
-
function readCheckpointResult(db) {
|
|
5186
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5187
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5188
|
-
}
|
|
5189
|
-
async function probeReclaimAvailability(input) {
|
|
5190
|
-
const { dbPath, requiredBytes } = input;
|
|
5191
|
-
let sqlite;
|
|
5192
|
-
try {
|
|
5193
|
-
sqlite = await import("node:sqlite");
|
|
5194
|
-
} catch (err) {
|
|
5195
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5196
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5197
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5198
|
-
}
|
|
5199
|
-
let autoVacuum = null;
|
|
5200
|
-
try {
|
|
5201
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5202
|
-
try {
|
|
5203
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5204
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5205
|
-
} finally {
|
|
5206
|
-
db.close();
|
|
5207
|
-
}
|
|
5208
|
-
} catch (err) {
|
|
5209
|
-
console.warn(
|
|
5210
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5211
|
-
);
|
|
5212
|
-
}
|
|
5213
|
-
if (autoVacuum !== 0) return null;
|
|
5214
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5215
|
-
}
|
|
5216
|
-
async function reclaimSessionDbSpace(input) {
|
|
5217
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5218
|
-
let sqlite;
|
|
5219
|
-
try {
|
|
5220
|
-
sqlite = await import("node:sqlite");
|
|
5221
|
-
} catch (err) {
|
|
5222
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5223
|
-
console.warn(
|
|
5224
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5225
|
-
);
|
|
5226
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5227
|
-
}
|
|
5228
|
-
const { DatabaseSync } = sqlite;
|
|
5229
|
-
let db;
|
|
5230
|
-
try {
|
|
5231
|
-
db = new DatabaseSync(dbPath);
|
|
5232
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5233
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5234
|
-
if (autoVacuum === 0) {
|
|
5235
|
-
if (!allowFullVacuum) {
|
|
5236
|
-
console.warn(
|
|
5237
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5238
|
-
);
|
|
5239
|
-
return {
|
|
5240
|
-
ok: false,
|
|
5241
|
-
skipped: "full-vacuum-blocked",
|
|
5242
|
-
detail: "a session turn is live"
|
|
5243
|
-
};
|
|
5726
|
+
let polling = false;
|
|
5727
|
+
let stopped = false;
|
|
5728
|
+
const poll = async () => {
|
|
5729
|
+
if (polling || stopped) return;
|
|
5730
|
+
polling = true;
|
|
5731
|
+
try {
|
|
5732
|
+
let current;
|
|
5733
|
+
try {
|
|
5734
|
+
current = await stat(logPath);
|
|
5735
|
+
} catch (error2) {
|
|
5736
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5737
|
+
return;
|
|
5244
5738
|
}
|
|
5245
|
-
|
|
5246
|
-
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
);
|
|
5251
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5739
|
+
if (!baselineReady) {
|
|
5740
|
+
offset = current.size;
|
|
5741
|
+
inode = current.ino;
|
|
5742
|
+
baselineReady = true;
|
|
5743
|
+
return;
|
|
5252
5744
|
}
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5745
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5746
|
+
offset = 0;
|
|
5747
|
+
}
|
|
5748
|
+
inode = current.ino;
|
|
5749
|
+
if (current.size === offset) return;
|
|
5750
|
+
const length = current.size - offset;
|
|
5751
|
+
const fh = await open2(logPath, "r");
|
|
5752
|
+
try {
|
|
5753
|
+
const buf = Buffer.alloc(length);
|
|
5754
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5755
|
+
offset += bytesRead;
|
|
5756
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5757
|
+
} finally {
|
|
5758
|
+
await fh.close();
|
|
5759
|
+
}
|
|
5760
|
+
} catch (error2) {
|
|
5761
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5762
|
+
} finally {
|
|
5763
|
+
polling = false;
|
|
5259
5764
|
}
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5765
|
+
};
|
|
5766
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5767
|
+
void poll();
|
|
5768
|
+
return {
|
|
5769
|
+
stop: () => {
|
|
5770
|
+
stopped = true;
|
|
5771
|
+
clearInterval(interval);
|
|
5267
5772
|
}
|
|
5268
|
-
|
|
5269
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5270
|
-
);
|
|
5271
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5272
|
-
} catch (err) {
|
|
5273
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5274
|
-
return {
|
|
5275
|
-
ok: false,
|
|
5276
|
-
skipped: "reclaim-error",
|
|
5277
|
-
detail: errorMessage(err)
|
|
5278
|
-
};
|
|
5279
|
-
} finally {
|
|
5280
|
-
db?.close();
|
|
5281
|
-
}
|
|
5773
|
+
};
|
|
5282
5774
|
}
|
|
5283
5775
|
|
|
5284
5776
|
// src/lib/tunnel/connection.ts
|
|
@@ -5510,6 +6002,8 @@ var StreamForwarder = class {
|
|
|
5510
6002
|
if (done) break;
|
|
5511
6003
|
const chunk = Buffer.from(value);
|
|
5512
6004
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
6005
|
+
await stream.pausePromise;
|
|
6006
|
+
if (ac.signal.aborted) break;
|
|
5513
6007
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5514
6008
|
if (this.binaryFramesSupported) {
|
|
5515
6009
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5552,6 +6046,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5552
6046
|
}
|
|
5553
6047
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5554
6048
|
var BASE_RECONNECT_DELAY = 500;
|
|
6049
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5555
6050
|
function getReconnectDelay(attempt) {
|
|
5556
6051
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5557
6052
|
const jitter = Math.random() * 1e3;
|
|
@@ -5623,10 +6118,27 @@ function connectTunnel(options) {
|
|
|
5623
6118
|
ws.close();
|
|
5624
6119
|
reject(new Error("Connection timeout"));
|
|
5625
6120
|
}, 3e4);
|
|
6121
|
+
let relaySilenceTimeout;
|
|
6122
|
+
let relaySilenceTimedOut = false;
|
|
6123
|
+
const clearRelaySilenceTimeout = () => {
|
|
6124
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6125
|
+
clearTimeout(relaySilenceTimeout);
|
|
6126
|
+
relaySilenceTimeout = void 0;
|
|
6127
|
+
};
|
|
6128
|
+
const armRelaySilenceTimeout = () => {
|
|
6129
|
+
clearRelaySilenceTimeout();
|
|
6130
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6131
|
+
relaySilenceTimeout = void 0;
|
|
6132
|
+
relaySilenceTimedOut = true;
|
|
6133
|
+
ws.terminate();
|
|
6134
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6135
|
+
relaySilenceTimeout.unref?.();
|
|
6136
|
+
};
|
|
5626
6137
|
let upgradeRejection = null;
|
|
5627
6138
|
let upgradeRejectionReason = null;
|
|
5628
6139
|
ws.on("unexpected-response", (_req, res) => {
|
|
5629
6140
|
clearTimeout(connectionTimeout);
|
|
6141
|
+
clearRelaySilenceTimeout();
|
|
5630
6142
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5631
6143
|
upgradeRejectionReason = reason;
|
|
5632
6144
|
const chunks = [];
|
|
@@ -5656,6 +6168,7 @@ function connectTunnel(options) {
|
|
|
5656
6168
|
onInfo?.("WebSocket connection established");
|
|
5657
6169
|
});
|
|
5658
6170
|
ws.on("message", (data, isBinary) => {
|
|
6171
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5659
6172
|
if (isBinary) {
|
|
5660
6173
|
try {
|
|
5661
6174
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5689,6 +6202,7 @@ function connectTunnel(options) {
|
|
|
5689
6202
|
switch (message.type) {
|
|
5690
6203
|
case "connected": {
|
|
5691
6204
|
clearTimeout(connectionTimeout);
|
|
6205
|
+
armRelaySilenceTimeout();
|
|
5692
6206
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5693
6207
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5694
6208
|
onConnected?.(connectedAgentId);
|
|
@@ -5713,6 +6227,7 @@ function connectTunnel(options) {
|
|
|
5713
6227
|
});
|
|
5714
6228
|
ws.on("error", (error2) => {
|
|
5715
6229
|
clearTimeout(connectionTimeout);
|
|
6230
|
+
clearRelaySilenceTimeout();
|
|
5716
6231
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5717
6232
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5718
6233
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5724,7 +6239,8 @@ function connectTunnel(options) {
|
|
|
5724
6239
|
);
|
|
5725
6240
|
});
|
|
5726
6241
|
ws.on("close", (code, reason) => {
|
|
5727
|
-
|
|
6242
|
+
clearRelaySilenceTimeout();
|
|
6243
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5728
6244
|
forwarder.abortAll();
|
|
5729
6245
|
onDisconnected?.(code, reasonStr);
|
|
5730
6246
|
});
|
|
@@ -5732,7 +6248,7 @@ function connectTunnel(options) {
|
|
|
5732
6248
|
}
|
|
5733
6249
|
|
|
5734
6250
|
// src/lib/tunnel/runner-connection.ts
|
|
5735
|
-
var RunnerConnection = class {
|
|
6251
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5736
6252
|
opts;
|
|
5737
6253
|
sleep;
|
|
5738
6254
|
now;
|
|
@@ -5746,6 +6262,10 @@ var RunnerConnection = class {
|
|
|
5746
6262
|
reconnectAttempt = 0;
|
|
5747
6263
|
generation = 0;
|
|
5748
6264
|
disconnectedAtMs = null;
|
|
6265
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6266
|
+
"No pong received",
|
|
6267
|
+
"Orphaned socket"
|
|
6268
|
+
]);
|
|
5749
6269
|
constructor(opts) {
|
|
5750
6270
|
this.opts = opts;
|
|
5751
6271
|
this.resolvedAgentId = opts.agentId;
|
|
@@ -5802,7 +6322,8 @@ var RunnerConnection = class {
|
|
|
5802
6322
|
onDisconnected: (code, reason) => {
|
|
5803
6323
|
this.disconnectedAtMs = this.now();
|
|
5804
6324
|
events.onDisconnected(code, reason, { generation: this.generation });
|
|
5805
|
-
|
|
6325
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6326
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5806
6327
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5807
6328
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5808
6329
|
});
|
|
@@ -6141,6 +6662,101 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6141
6662
|
};
|
|
6142
6663
|
}
|
|
6143
6664
|
|
|
6665
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6666
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6667
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6668
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6669
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6670
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6671
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6672
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6673
|
+
var BAND_RANK = {
|
|
6674
|
+
low: 1,
|
|
6675
|
+
very_low: 2,
|
|
6676
|
+
critical: 3
|
|
6677
|
+
};
|
|
6678
|
+
function bandFor(freeBytes) {
|
|
6679
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6680
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6681
|
+
return "low";
|
|
6682
|
+
}
|
|
6683
|
+
function formatBytes(bytes) {
|
|
6684
|
+
if (bytes === null) return "unavailable";
|
|
6685
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6686
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6687
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6688
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6689
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6690
|
+
}
|
|
6691
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6692
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6693
|
+
}
|
|
6694
|
+
function evaluateLowDiskWarning(input) {
|
|
6695
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6696
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6697
|
+
return { warning: null, state };
|
|
6698
|
+
}
|
|
6699
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6700
|
+
if (recovered) {
|
|
6701
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6702
|
+
}
|
|
6703
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6704
|
+
if (!low) return { warning: null, state };
|
|
6705
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6706
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6707
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6708
|
+
const worse = currentRank > previousRank;
|
|
6709
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6710
|
+
const nextState = {
|
|
6711
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6712
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6713
|
+
};
|
|
6714
|
+
return {
|
|
6715
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6716
|
+
state: nextState
|
|
6717
|
+
};
|
|
6718
|
+
}
|
|
6719
|
+
|
|
6720
|
+
// src/lib/opencode/session-store-health.ts
|
|
6721
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6722
|
+
function createSessionStoreHealth(options = {}) {
|
|
6723
|
+
const now = options.now ?? (() => Date.now());
|
|
6724
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6725
|
+
let lastUnwritableAt = null;
|
|
6726
|
+
let lastWritableAt = null;
|
|
6727
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6728
|
+
return {
|
|
6729
|
+
observeUnwritable: () => {
|
|
6730
|
+
lastUnwritableAt = now();
|
|
6731
|
+
},
|
|
6732
|
+
observeWritable: () => {
|
|
6733
|
+
lastWritableAt = now();
|
|
6734
|
+
},
|
|
6735
|
+
isUnwritable: () => {
|
|
6736
|
+
if (lastUnwritableAt === null) return false;
|
|
6737
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6738
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6739
|
+
},
|
|
6740
|
+
observeCause: (sessionId, cause) => {
|
|
6741
|
+
if (sessionId === null) return;
|
|
6742
|
+
causes.delete(sessionId);
|
|
6743
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6744
|
+
while (causes.size > 256) {
|
|
6745
|
+
const oldest = causes.keys().next().value;
|
|
6746
|
+
if (typeof oldest !== "string") break;
|
|
6747
|
+
causes.delete(oldest);
|
|
6748
|
+
}
|
|
6749
|
+
},
|
|
6750
|
+
readCause: (sessionId) => {
|
|
6751
|
+
const observed = causes.get(sessionId);
|
|
6752
|
+
if (!observed) return null;
|
|
6753
|
+
causes.delete(sessionId);
|
|
6754
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6755
|
+
return observed.cause;
|
|
6756
|
+
}
|
|
6757
|
+
};
|
|
6758
|
+
}
|
|
6759
|
+
|
|
6144
6760
|
// src/lib/channels/driver.ts
|
|
6145
6761
|
import { homedir as homedir6 } from "node:os";
|
|
6146
6762
|
|
|
@@ -6577,6 +7193,11 @@ function messageIdOf(m) {
|
|
|
6577
7193
|
const infoId = m.info?.id;
|
|
6578
7194
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6579
7195
|
}
|
|
7196
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7197
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7198
|
+
if (replyShape === null) return base;
|
|
7199
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7200
|
+
}
|
|
6580
7201
|
function cleanImageMime(contentType) {
|
|
6581
7202
|
if (!contentType) return null;
|
|
6582
7203
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6597,8 +7218,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6597
7218
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6598
7219
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6599
7220
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7221
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7222
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7223
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7224
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6600
7225
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6601
7226
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7227
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6602
7228
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6603
7229
|
var HEARTBEAT_MS = 6e4;
|
|
6604
7230
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6660,6 +7286,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6660
7286
|
maxActiveSessions;
|
|
6661
7287
|
watcherStallMs;
|
|
6662
7288
|
wedgeWarningIntervalMs;
|
|
7289
|
+
sessionStoreHealth;
|
|
6663
7290
|
openCodeClient;
|
|
6664
7291
|
/** Cache of conversationId → opencode sessionId. */
|
|
6665
7292
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6752,6 +7379,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6752
7379
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6753
7380
|
*/
|
|
6754
7381
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7382
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6755
7383
|
/**
|
|
6756
7384
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6757
7385
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6784,10 +7412,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6784
7412
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6785
7413
|
*/
|
|
6786
7414
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7415
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7416
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6787
7417
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6788
7418
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6789
7419
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6790
7420
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7421
|
+
/**
|
|
7422
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7423
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7424
|
+
* identical-text turn, so it never produces a correlation.
|
|
7425
|
+
*/
|
|
7426
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6791
7427
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6792
7428
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6793
7429
|
/**
|
|
@@ -6873,21 +7509,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6873
7509
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6874
7510
|
*/
|
|
6875
7511
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6876
|
-
/**
|
|
6877
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6878
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6879
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6880
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6881
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6882
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6883
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6884
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6885
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6886
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6887
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6888
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6889
|
-
*/
|
|
6890
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6891
7512
|
/**
|
|
6892
7513
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6893
7514
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6905,8 +7526,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6905
7526
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6906
7527
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6907
7528
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6908
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6909
|
-
*
|
|
7529
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7530
|
+
* the terminal-report fence and is never sent again).
|
|
6910
7531
|
*/
|
|
6911
7532
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6912
7533
|
/**
|
|
@@ -7029,6 +7650,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7029
7650
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7030
7651
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7031
7652
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7653
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7032
7654
|
}
|
|
7033
7655
|
get isV2() {
|
|
7034
7656
|
return this.openCodeClient.version === "v2";
|
|
@@ -7044,7 +7666,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7044
7666
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7045
7667
|
}
|
|
7046
7668
|
async listSessions() {
|
|
7047
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7669
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7670
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7671
|
+
});
|
|
7048
7672
|
}
|
|
7049
7673
|
async sessionExists(sessionId) {
|
|
7050
7674
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7221,6 +7845,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7221
7845
|
}
|
|
7222
7846
|
return ids;
|
|
7223
7847
|
}
|
|
7848
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7849
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7850
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7851
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7852
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7853
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7854
|
+
}
|
|
7855
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7856
|
+
}
|
|
7857
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7858
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7859
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7860
|
+
this.awaitingReadopt.delete(messageId);
|
|
7861
|
+
this.sessions.delete(conversationId);
|
|
7862
|
+
this.supersede(conversationId, sessionId);
|
|
7863
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7864
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7865
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7866
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7867
|
+
this.log({
|
|
7868
|
+
level: "error",
|
|
7869
|
+
message: errorMessage3,
|
|
7870
|
+
conversation_id: conversationId,
|
|
7871
|
+
message_id: messageId
|
|
7872
|
+
});
|
|
7873
|
+
const reportError = await this.reportUntrackableAck(
|
|
7874
|
+
conversationId,
|
|
7875
|
+
messageId,
|
|
7876
|
+
null,
|
|
7877
|
+
errorMessage3
|
|
7878
|
+
);
|
|
7879
|
+
if (reportError !== null) {
|
|
7880
|
+
this.log({
|
|
7881
|
+
level: "warn",
|
|
7882
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7883
|
+
conversation_id: conversationId,
|
|
7884
|
+
message_id: messageId
|
|
7885
|
+
});
|
|
7886
|
+
}
|
|
7887
|
+
if (source === "readopt") {
|
|
7888
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7889
|
+
}
|
|
7890
|
+
}
|
|
7224
7891
|
/**
|
|
7225
7892
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7226
7893
|
*
|
|
@@ -7400,6 +8067,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7400
8067
|
}
|
|
7401
8068
|
break;
|
|
7402
8069
|
}
|
|
8070
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8071
|
+
if (correlation) {
|
|
8072
|
+
correlation.attempts += 1;
|
|
8073
|
+
let correlatedId = null;
|
|
8074
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8075
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8076
|
+
conv,
|
|
8077
|
+
message,
|
|
8078
|
+
correlation.sessionId
|
|
8079
|
+
);
|
|
8080
|
+
if (polled.ok)
|
|
8081
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8082
|
+
} else if (correlation.knownUserIds === null) {
|
|
8083
|
+
this.log({
|
|
8084
|
+
level: "debug",
|
|
8085
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8086
|
+
conversation_id: conv.id,
|
|
8087
|
+
message_id: message.id
|
|
8088
|
+
});
|
|
8089
|
+
}
|
|
8090
|
+
if (correlatedId !== null) {
|
|
8091
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8092
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8093
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8094
|
+
this.dispatched.add(message.id);
|
|
8095
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8096
|
+
dispatched += 1;
|
|
8097
|
+
this.log({
|
|
8098
|
+
level: "info",
|
|
8099
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8100
|
+
conversation_id: conv.id,
|
|
8101
|
+
message_id: message.id
|
|
8102
|
+
});
|
|
8103
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8104
|
+
opencode_message_id: correlatedId
|
|
8105
|
+
});
|
|
8106
|
+
continue;
|
|
8107
|
+
}
|
|
8108
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8109
|
+
await this.handoffUntrackableCorrelation(
|
|
8110
|
+
conv.id,
|
|
8111
|
+
message.id,
|
|
8112
|
+
correlation.sessionId,
|
|
8113
|
+
correlation.attempts,
|
|
8114
|
+
"fresh"
|
|
8115
|
+
);
|
|
8116
|
+
break;
|
|
8117
|
+
}
|
|
8118
|
+
break;
|
|
8119
|
+
}
|
|
7403
8120
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7404
8121
|
if (effectiveOpencodeMessageId) {
|
|
7405
8122
|
const outcome = await this.resolveRedrive(
|
|
@@ -7421,6 +8138,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7421
8138
|
model: message.opencode_model ?? void 0
|
|
7422
8139
|
};
|
|
7423
8140
|
let opencodeMessageId;
|
|
8141
|
+
let preDispatchUserIds = null;
|
|
7424
8142
|
try {
|
|
7425
8143
|
this.log({
|
|
7426
8144
|
level: "info",
|
|
@@ -7450,7 +8168,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7450
8168
|
message.content,
|
|
7451
8169
|
options,
|
|
7452
8170
|
sendAttachments,
|
|
7453
|
-
this.openCodeClient
|
|
8171
|
+
this.openCodeClient,
|
|
8172
|
+
{
|
|
8173
|
+
onKnownUserIds: (ids) => {
|
|
8174
|
+
preDispatchUserIds = ids;
|
|
8175
|
+
},
|
|
8176
|
+
onKnownUserIdsError: (error2) => {
|
|
8177
|
+
this.log({
|
|
8178
|
+
level: "warn",
|
|
8179
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8180
|
+
conversation_id: conv.id,
|
|
8181
|
+
message_id: message.id
|
|
8182
|
+
});
|
|
8183
|
+
}
|
|
8184
|
+
}
|
|
7454
8185
|
)
|
|
7455
8186
|
);
|
|
7456
8187
|
} catch (err) {
|
|
@@ -7458,6 +8189,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7458
8189
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7459
8190
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7460
8191
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8192
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7461
8193
|
this.log({
|
|
7462
8194
|
level: "error",
|
|
7463
8195
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7533,39 +8265,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7533
8265
|
if (this.isV2) {
|
|
7534
8266
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7535
8267
|
}
|
|
7536
|
-
|
|
7537
|
-
|
|
7538
|
-
|
|
7539
|
-
|
|
7540
|
-
|
|
7541
|
-
|
|
7542
|
-
message_id: message.id
|
|
7543
|
-
});
|
|
7544
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7545
|
-
continue;
|
|
7546
|
-
}
|
|
7547
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7548
|
-
this.sessions.delete(conv.id);
|
|
7549
|
-
this.supersede(conv.id, sessionId);
|
|
7550
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8268
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8269
|
+
sessionId,
|
|
8270
|
+
content: message.content,
|
|
8271
|
+
attempts: 0,
|
|
8272
|
+
knownUserIds: preDispatchUserIds
|
|
8273
|
+
});
|
|
7551
8274
|
this.log({
|
|
7552
|
-
level: "
|
|
7553
|
-
message:
|
|
8275
|
+
level: "warn",
|
|
8276
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7554
8277
|
conversation_id: conv.id,
|
|
7555
8278
|
message_id: message.id
|
|
7556
8279
|
});
|
|
7557
|
-
|
|
7558
|
-
this.log({
|
|
7559
|
-
level: "warn",
|
|
7560
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7561
|
-
conversation_id: conv.id,
|
|
7562
|
-
message_id: message.id
|
|
7563
|
-
});
|
|
7564
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7565
|
-
});
|
|
8280
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7566
8281
|
break;
|
|
7567
8282
|
}
|
|
7568
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7569
8283
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7570
8284
|
this.releasedOpencodeIds.delete(message.id);
|
|
7571
8285
|
this.dispatched.add(message.id);
|
|
@@ -8170,23 +8884,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8170
8884
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8171
8885
|
return 1;
|
|
8172
8886
|
}
|
|
8173
|
-
/**
|
|
8174
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8175
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8176
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8177
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8178
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8179
|
-
* count, since a new session is a genuinely different attempt.
|
|
8180
|
-
*/
|
|
8181
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8182
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8183
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8184
|
-
existing.count += 1;
|
|
8185
|
-
return existing.count;
|
|
8186
|
-
}
|
|
8187
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8188
|
-
return 1;
|
|
8189
|
-
}
|
|
8190
8887
|
/** Record one queued give-up toward the never-started bound. */
|
|
8191
8888
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8192
8889
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8498,8 +9195,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8498
9195
|
b2LastDescendantCheckMs: 0,
|
|
8499
9196
|
b2RootOngoingHeldLogged: false,
|
|
8500
9197
|
b2AbandonedSignalled: false,
|
|
9198
|
+
b2AbandonedWithoutAnswer: false,
|
|
8501
9199
|
ambiguousPinnedSinceMs: 0,
|
|
8502
9200
|
ambiguousResolved: false,
|
|
9201
|
+
sessionStoreFailurePending: false,
|
|
8503
9202
|
neverStartedResolved: false
|
|
8504
9203
|
});
|
|
8505
9204
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8521,6 +9220,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8521
9220
|
this.bufferedSessionErrors.delete(oldest);
|
|
8522
9221
|
}
|
|
8523
9222
|
}
|
|
9223
|
+
retainSessionError(event) {
|
|
9224
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9225
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9226
|
+
reason: event.reason,
|
|
9227
|
+
observedAt: this.now()
|
|
9228
|
+
});
|
|
9229
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9230
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9231
|
+
if (typeof oldest !== "string") break;
|
|
9232
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9233
|
+
}
|
|
9234
|
+
}
|
|
9235
|
+
readRetainedSessionError(sessionId) {
|
|
9236
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9237
|
+
if (!retained) return null;
|
|
9238
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9239
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9240
|
+
return null;
|
|
9241
|
+
}
|
|
9242
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9243
|
+
return retained.reason;
|
|
9244
|
+
}
|
|
8524
9245
|
/**
|
|
8525
9246
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8526
9247
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8616,8 +9337,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8616
9337
|
b2LastDescendantCheckMs: 0,
|
|
8617
9338
|
b2RootOngoingHeldLogged: false,
|
|
8618
9339
|
b2AbandonedSignalled: false,
|
|
9340
|
+
b2AbandonedWithoutAnswer: false,
|
|
8619
9341
|
ambiguousPinnedSinceMs: 0,
|
|
8620
9342
|
ambiguousResolved: false,
|
|
9343
|
+
sessionStoreFailurePending: false,
|
|
8621
9344
|
neverStartedResolved: false
|
|
8622
9345
|
});
|
|
8623
9346
|
}
|
|
@@ -8845,9 +9568,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8845
9568
|
return;
|
|
8846
9569
|
}
|
|
8847
9570
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9571
|
+
this.retainSessionError(event);
|
|
8848
9572
|
this.log({
|
|
8849
|
-
level: "
|
|
8850
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9573
|
+
level: "warn",
|
|
9574
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8851
9575
|
conversation_id: watcher.conv.id
|
|
8852
9576
|
});
|
|
8853
9577
|
return;
|
|
@@ -8877,9 +9601,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8877
9601
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8878
9602
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8879
9603
|
if (state !== "queued") {
|
|
9604
|
+
this.retainSessionError(event);
|
|
8880
9605
|
this.log({
|
|
8881
|
-
level: "
|
|
8882
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9606
|
+
level: "warn",
|
|
9607
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8883
9608
|
conversation_id: watcher.conv.id,
|
|
8884
9609
|
message_id: inFlight.evidentMessageId
|
|
8885
9610
|
});
|
|
@@ -9031,6 +9756,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9031
9756
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9032
9757
|
this.neverStartedAttempts.delete(id);
|
|
9033
9758
|
}
|
|
9759
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9760
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9761
|
+
return;
|
|
9762
|
+
}
|
|
9034
9763
|
if (inFlight.neverStartedResolved) {
|
|
9035
9764
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9036
9765
|
inFlight.neverStartedResolved = false;
|
|
@@ -9104,7 +9833,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9104
9833
|
if (state === "failed") {
|
|
9105
9834
|
this.anchorDeliveryDeadline(inFlight);
|
|
9106
9835
|
if (!inFlight.done) {
|
|
9107
|
-
const
|
|
9836
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9837
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9838
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9839
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9840
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9841
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9842
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9843
|
+
(detail) => detail !== null
|
|
9844
|
+
);
|
|
9845
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9108
9846
|
this.log({
|
|
9109
9847
|
level: "error",
|
|
9110
9848
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9190,10 +9928,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9190
9928
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9191
9929
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9192
9930
|
inFlight.b2AbandonedSignalled = false;
|
|
9931
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9193
9932
|
}
|
|
9194
9933
|
} else {
|
|
9195
9934
|
if (inFlight.b2AbandonedSignalled) {
|
|
9196
|
-
|
|
9935
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9936
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9937
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9938
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9939
|
+
sessionId,
|
|
9940
|
+
watcher,
|
|
9941
|
+
inFlight,
|
|
9942
|
+
messages,
|
|
9943
|
+
finish
|
|
9944
|
+
);
|
|
9945
|
+
} else {
|
|
9946
|
+
await this.settleMessageDone(
|
|
9947
|
+
sessionId,
|
|
9948
|
+
watcher,
|
|
9949
|
+
inFlight,
|
|
9950
|
+
messages,
|
|
9951
|
+
composeAbandonmentNote(
|
|
9952
|
+
finish,
|
|
9953
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
9954
|
+
"with-answer"
|
|
9955
|
+
)
|
|
9956
|
+
);
|
|
9957
|
+
}
|
|
9197
9958
|
return;
|
|
9198
9959
|
}
|
|
9199
9960
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9210,22 +9971,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9210
9971
|
descendantOngoing,
|
|
9211
9972
|
rootOngoing
|
|
9212
9973
|
})) {
|
|
9974
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9975
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9976
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9977
|
+
watched_for_ms: pinnedForMs,
|
|
9978
|
+
run_state: "failed",
|
|
9979
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9980
|
+
});
|
|
9981
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9982
|
+
return;
|
|
9983
|
+
}
|
|
9213
9984
|
inFlight.b2AbandonedSignalled = true;
|
|
9985
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9986
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9987
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9988
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9989
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9990
|
+
watched_for_ms: pinnedForMs,
|
|
9991
|
+
run_state: "failed",
|
|
9992
|
+
finish,
|
|
9993
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9994
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9995
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9996
|
+
});
|
|
9997
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9998
|
+
sessionId,
|
|
9999
|
+
watcher,
|
|
10000
|
+
inFlight,
|
|
10001
|
+
messages,
|
|
10002
|
+
finish
|
|
10003
|
+
);
|
|
10004
|
+
return;
|
|
10005
|
+
}
|
|
9214
10006
|
this.log({
|
|
9215
10007
|
level: "warn",
|
|
9216
10008
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9217
10009
|
conversation_id: conv.id,
|
|
9218
10010
|
message_id: id
|
|
9219
10011
|
});
|
|
9220
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9221
10012
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9222
10013
|
watched_for_ms: pinnedForMs,
|
|
9223
|
-
finish
|
|
10014
|
+
finish,
|
|
9224
10015
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9225
10016
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9226
10017
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9227
10018
|
});
|
|
9228
|
-
await this.settleMessageDone(
|
|
10019
|
+
await this.settleMessageDone(
|
|
10020
|
+
sessionId,
|
|
10021
|
+
watcher,
|
|
10022
|
+
inFlight,
|
|
10023
|
+
messages,
|
|
10024
|
+
composeAbandonmentNote(
|
|
10025
|
+
finish,
|
|
10026
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10027
|
+
"with-answer"
|
|
10028
|
+
)
|
|
10029
|
+
);
|
|
9229
10030
|
return;
|
|
9230
10031
|
}
|
|
9231
10032
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9268,6 +10069,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9268
10069
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9269
10070
|
sessionOngoing: ongoing
|
|
9270
10071
|
})) {
|
|
10072
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10073
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10074
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10075
|
+
watched_for_ms: pinnedForMs,
|
|
10076
|
+
run_state: "failed",
|
|
10077
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10078
|
+
});
|
|
10079
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10080
|
+
return;
|
|
10081
|
+
}
|
|
9271
10082
|
inFlight.ambiguousResolved = true;
|
|
9272
10083
|
this.log({
|
|
9273
10084
|
level: "warn",
|
|
@@ -9415,15 +10226,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9415
10226
|
}
|
|
9416
10227
|
}
|
|
9417
10228
|
/**
|
|
9418
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9419
|
-
*
|
|
9420
|
-
*
|
|
9421
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9422
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9423
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9424
|
-
* and risking the two copies silently drifting apart.
|
|
10229
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10230
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10231
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9425
10232
|
*/
|
|
9426
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10233
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9427
10234
|
const conv = watcher.conv;
|
|
9428
10235
|
this.anchorDeliveryDeadline(inFlight);
|
|
9429
10236
|
if (!inFlight.done) {
|
|
@@ -9450,7 +10257,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9450
10257
|
title,
|
|
9451
10258
|
usage,
|
|
9452
10259
|
usageAgentName,
|
|
9453
|
-
subagentInvocations
|
|
10260
|
+
subagentInvocations,
|
|
10261
|
+
note
|
|
9454
10262
|
);
|
|
9455
10263
|
} catch (err) {
|
|
9456
10264
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9476,20 +10284,92 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9476
10284
|
}
|
|
9477
10285
|
this.log({
|
|
9478
10286
|
level: "warn",
|
|
9479
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10287
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10288
|
+
conversation_id: conv.id,
|
|
10289
|
+
message_id: inFlight.evidentMessageId
|
|
10290
|
+
});
|
|
10291
|
+
return;
|
|
10292
|
+
}
|
|
10293
|
+
inFlight.done = true;
|
|
10294
|
+
await this.reportSubagentAuthFailures(
|
|
10295
|
+
watcher.conv.id,
|
|
10296
|
+
inFlight.opencodeMessageId,
|
|
10297
|
+
inFlight.evidentMessageId,
|
|
10298
|
+
messages
|
|
10299
|
+
);
|
|
10300
|
+
}
|
|
10301
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10302
|
+
}
|
|
10303
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10304
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10305
|
+
const conv = watcher.conv;
|
|
10306
|
+
const error2 = composeAbandonmentNote(
|
|
10307
|
+
finish,
|
|
10308
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10309
|
+
"without-answer"
|
|
10310
|
+
);
|
|
10311
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10312
|
+
this.log({
|
|
10313
|
+
level: "warn",
|
|
10314
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10315
|
+
conversation_id: conv.id,
|
|
10316
|
+
message_id: inFlight.evidentMessageId
|
|
10317
|
+
});
|
|
10318
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10319
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10320
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10321
|
+
messages,
|
|
10322
|
+
inFlight.opencodeMessageId,
|
|
10323
|
+
inFlight.evidentMessageId
|
|
10324
|
+
);
|
|
10325
|
+
try {
|
|
10326
|
+
await this.markFailed(
|
|
10327
|
+
conv.id,
|
|
10328
|
+
inFlight.evidentMessageId,
|
|
10329
|
+
sessionId,
|
|
10330
|
+
error2,
|
|
10331
|
+
usage,
|
|
10332
|
+
void 0,
|
|
10333
|
+
usageAgentName,
|
|
10334
|
+
subagentInvocations
|
|
10335
|
+
);
|
|
10336
|
+
} catch (err) {
|
|
10337
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10338
|
+
if (err instanceof ChannelTerminalError) {
|
|
10339
|
+
this.log({
|
|
10340
|
+
level: "warn",
|
|
10341
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9480
10342
|
conversation_id: conv.id,
|
|
9481
10343
|
message_id: inFlight.evidentMessageId
|
|
9482
10344
|
});
|
|
10345
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9483
10346
|
return;
|
|
9484
10347
|
}
|
|
9485
|
-
inFlight.
|
|
9486
|
-
|
|
9487
|
-
|
|
9488
|
-
|
|
9489
|
-
|
|
9490
|
-
|
|
9491
|
-
|
|
10348
|
+
if (this.now() >= inFlight.deadline) {
|
|
10349
|
+
this.log({
|
|
10350
|
+
level: "warn",
|
|
10351
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10352
|
+
conversation_id: conv.id,
|
|
10353
|
+
message_id: inFlight.evidentMessageId
|
|
10354
|
+
});
|
|
10355
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10356
|
+
return;
|
|
10357
|
+
}
|
|
10358
|
+
this.log({
|
|
10359
|
+
level: "warn",
|
|
10360
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10361
|
+
conversation_id: conv.id,
|
|
10362
|
+
message_id: inFlight.evidentMessageId
|
|
10363
|
+
});
|
|
10364
|
+
return;
|
|
9492
10365
|
}
|
|
10366
|
+
inFlight.done = true;
|
|
10367
|
+
await this.reportSubagentAuthFailures(
|
|
10368
|
+
watcher.conv.id,
|
|
10369
|
+
inFlight.opencodeMessageId,
|
|
10370
|
+
inFlight.evidentMessageId,
|
|
10371
|
+
messages
|
|
10372
|
+
);
|
|
9493
10373
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9494
10374
|
}
|
|
9495
10375
|
/**
|
|
@@ -9547,6 +10427,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9547
10427
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9548
10428
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9549
10429
|
}
|
|
10430
|
+
/**
|
|
10431
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10432
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10433
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10434
|
+
*/
|
|
10435
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10436
|
+
const conv = watcher.conv;
|
|
10437
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10438
|
+
this.log({
|
|
10439
|
+
level: "error",
|
|
10440
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10441
|
+
conversation_id: conv.id,
|
|
10442
|
+
message_id: inFlight.evidentMessageId
|
|
10443
|
+
});
|
|
10444
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10445
|
+
try {
|
|
10446
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10447
|
+
} catch (err) {
|
|
10448
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10449
|
+
if (err instanceof ChannelTerminalError) {
|
|
10450
|
+
this.log({
|
|
10451
|
+
level: "warn",
|
|
10452
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10453
|
+
conversation_id: conv.id,
|
|
10454
|
+
message_id: inFlight.evidentMessageId
|
|
10455
|
+
});
|
|
10456
|
+
this.recordReleasedOpencodeId(
|
|
10457
|
+
inFlight.evidentMessageId,
|
|
10458
|
+
sessionId,
|
|
10459
|
+
inFlight.opencodeMessageId
|
|
10460
|
+
);
|
|
10461
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10462
|
+
return;
|
|
10463
|
+
}
|
|
10464
|
+
if (this.now() >= inFlight.deadline) {
|
|
10465
|
+
this.log({
|
|
10466
|
+
level: "warn",
|
|
10467
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10468
|
+
conversation_id: conv.id,
|
|
10469
|
+
message_id: inFlight.evidentMessageId
|
|
10470
|
+
});
|
|
10471
|
+
this.recordReleasedOpencodeId(
|
|
10472
|
+
inFlight.evidentMessageId,
|
|
10473
|
+
sessionId,
|
|
10474
|
+
inFlight.opencodeMessageId
|
|
10475
|
+
);
|
|
10476
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10477
|
+
return;
|
|
10478
|
+
}
|
|
10479
|
+
this.log({
|
|
10480
|
+
level: "warn",
|
|
10481
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10482
|
+
conversation_id: conv.id,
|
|
10483
|
+
message_id: inFlight.evidentMessageId
|
|
10484
|
+
});
|
|
10485
|
+
return;
|
|
10486
|
+
}
|
|
10487
|
+
inFlight.done = true;
|
|
10488
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10489
|
+
}
|
|
9550
10490
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9551
10491
|
/**
|
|
9552
10492
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9585,11 +10525,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9585
10525
|
}
|
|
9586
10526
|
}
|
|
9587
10527
|
}
|
|
9588
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10528
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9589
10529
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10530
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10531
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10532
|
+
this.awaitingCorrelation.delete(id);
|
|
10533
|
+
this.log({
|
|
10534
|
+
level: "debug",
|
|
10535
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10536
|
+
message_id: id
|
|
10537
|
+
});
|
|
10538
|
+
}
|
|
10539
|
+
}
|
|
9590
10540
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9591
10541
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9592
10542
|
this.untrackableAck.delete(id);
|
|
10543
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9593
10544
|
this.untrackableAckSignalled.delete(id);
|
|
9594
10545
|
this.untrackableAckWarned.delete(id);
|
|
9595
10546
|
this.log({
|
|
@@ -9629,7 +10580,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9629
10580
|
bySession.set(row.opencode_session_id, list);
|
|
9630
10581
|
}
|
|
9631
10582
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9632
|
-
let messages;
|
|
10583
|
+
let messages = null;
|
|
9633
10584
|
try {
|
|
9634
10585
|
if (this.isV2) {
|
|
9635
10586
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9638,9 +10589,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9638
10589
|
level: "warn",
|
|
9639
10590
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9640
10591
|
});
|
|
9641
|
-
|
|
10592
|
+
} else {
|
|
10593
|
+
messages = snapshot;
|
|
9642
10594
|
}
|
|
9643
|
-
messages = snapshot;
|
|
9644
10595
|
} else {
|
|
9645
10596
|
const polled = await pollSessionMessagesForRedrive(
|
|
9646
10597
|
this.port,
|
|
@@ -9653,15 +10604,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9653
10604
|
level: "warn",
|
|
9654
10605
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9655
10606
|
});
|
|
9656
|
-
|
|
10607
|
+
} else {
|
|
10608
|
+
messages = polled.messages;
|
|
9657
10609
|
}
|
|
9658
|
-
messages = polled.messages;
|
|
9659
10610
|
}
|
|
9660
10611
|
} catch (err) {
|
|
9661
10612
|
this.log({
|
|
9662
10613
|
level: "warn",
|
|
9663
10614
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9664
10615
|
});
|
|
10616
|
+
}
|
|
10617
|
+
if (messages === null) {
|
|
10618
|
+
for (const row of sessionRows) {
|
|
10619
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10620
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10621
|
+
}
|
|
10622
|
+
}
|
|
9665
10623
|
continue;
|
|
9666
10624
|
}
|
|
9667
10625
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9671,6 +10629,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9671
10629
|
}
|
|
9672
10630
|
}
|
|
9673
10631
|
}
|
|
10632
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10633
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10634
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10635
|
+
if (!correlation) return false;
|
|
10636
|
+
correlation.attempts += 1;
|
|
10637
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10638
|
+
if (correlatedId !== null) {
|
|
10639
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10640
|
+
const conv = this.convForRow(sessionId, row);
|
|
10641
|
+
const message = this.queuedMessageForRow(row);
|
|
10642
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10643
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10644
|
+
this.dispatched.add(row.id);
|
|
10645
|
+
this.readopted.add(row.id);
|
|
10646
|
+
this.ensureWatcherRunning(sessionId);
|
|
10647
|
+
this.log({
|
|
10648
|
+
level: "info",
|
|
10649
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10650
|
+
conversation_id: row.conversation_id,
|
|
10651
|
+
message_id: row.id
|
|
10652
|
+
});
|
|
10653
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10654
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10655
|
+
opencode_message_id: correlatedId
|
|
10656
|
+
});
|
|
10657
|
+
return true;
|
|
10658
|
+
}
|
|
10659
|
+
if (correlation.knownUserIds === null) {
|
|
10660
|
+
this.log({
|
|
10661
|
+
level: "debug",
|
|
10662
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10663
|
+
conversation_id: row.conversation_id,
|
|
10664
|
+
message_id: row.id
|
|
10665
|
+
});
|
|
10666
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10667
|
+
this.log({
|
|
10668
|
+
level: "warn",
|
|
10669
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10670
|
+
conversation_id: row.conversation_id,
|
|
10671
|
+
message_id: row.id
|
|
10672
|
+
});
|
|
10673
|
+
}
|
|
10674
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10675
|
+
await this.handoffUntrackableCorrelation(
|
|
10676
|
+
row.conversation_id,
|
|
10677
|
+
row.id,
|
|
10678
|
+
correlation.sessionId,
|
|
10679
|
+
correlation.attempts,
|
|
10680
|
+
"readopt"
|
|
10681
|
+
);
|
|
10682
|
+
}
|
|
10683
|
+
return true;
|
|
10684
|
+
}
|
|
9674
10685
|
/**
|
|
9675
10686
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9676
10687
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9678,8 +10689,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9678
10689
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9679
10690
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9680
10691
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9681
|
-
* before the restart)
|
|
9682
|
-
*
|
|
10692
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10693
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10694
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10695
|
+
* `forceReadoptRun`):
|
|
9683
10696
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9684
10697
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9685
10698
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9688,7 +10701,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9688
10701
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9689
10702
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9690
10703
|
* tracking the stored id so the reply correlates by it;
|
|
9691
|
-
* - `unknown`/null id →
|
|
10704
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10705
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9692
10706
|
*
|
|
9693
10707
|
* Only `ChannelAuthError` propagates.
|
|
9694
10708
|
*/
|
|
@@ -9702,6 +10716,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9702
10716
|
});
|
|
9703
10717
|
return;
|
|
9704
10718
|
}
|
|
10719
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9705
10720
|
const ocId = row.opencode_message_id;
|
|
9706
10721
|
const state = messageRunState(messages, ocId ?? "");
|
|
9707
10722
|
if (state === "done") {
|
|
@@ -9768,12 +10783,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9768
10783
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9769
10784
|
this.dontRedispatch.delete(row.id);
|
|
9770
10785
|
this.untrackableAck.delete(row.id);
|
|
10786
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9771
10787
|
this.untrackableAckSignalled.delete(row.id);
|
|
9772
10788
|
this.untrackableAckWarned.delete(row.id);
|
|
9773
10789
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9774
10790
|
return;
|
|
9775
10791
|
}
|
|
9776
|
-
if (this.
|
|
10792
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10793
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10794
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10795
|
+
this.untrackableAckWarned.add(row.id);
|
|
10796
|
+
this.log({
|
|
10797
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10798
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10799
|
+
conversation_id: row.conversation_id,
|
|
10800
|
+
message_id: row.id
|
|
10801
|
+
});
|
|
10802
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10803
|
+
const reportError = await this.reportUntrackableAck(
|
|
10804
|
+
row.conversation_id,
|
|
10805
|
+
row.id,
|
|
10806
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10807
|
+
errorMessage3
|
|
10808
|
+
);
|
|
10809
|
+
if (reportError !== null) {
|
|
10810
|
+
this.log({
|
|
10811
|
+
level: "warn",
|
|
10812
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10813
|
+
conversation_id: row.conversation_id,
|
|
10814
|
+
message_id: row.id
|
|
10815
|
+
});
|
|
10816
|
+
}
|
|
10817
|
+
} else {
|
|
10818
|
+
this.log({
|
|
10819
|
+
level: "debug",
|
|
10820
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10821
|
+
conversation_id: row.conversation_id,
|
|
10822
|
+
message_id: row.id
|
|
10823
|
+
});
|
|
10824
|
+
}
|
|
10825
|
+
return;
|
|
10826
|
+
}
|
|
10827
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9777
10828
|
this.log({
|
|
9778
10829
|
level: "debug",
|
|
9779
10830
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9951,6 +11002,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9951
11002
|
}
|
|
9952
11003
|
this.dontRedispatch.delete(row.id);
|
|
9953
11004
|
this.untrackableAck.delete(row.id);
|
|
11005
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9954
11006
|
this.untrackableAckSignalled.delete(row.id);
|
|
9955
11007
|
this.untrackableAckWarned.delete(row.id);
|
|
9956
11008
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9970,8 +11022,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9970
11022
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9971
11023
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9972
11024
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9973
|
-
* re-entry)
|
|
9974
|
-
*
|
|
11025
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11026
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11027
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9975
11028
|
*
|
|
9976
11029
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9977
11030
|
* `processed_at` (Invariant 1).
|
|
@@ -10034,6 +11087,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10034
11087
|
);
|
|
10035
11088
|
}
|
|
10036
11089
|
let ocId;
|
|
11090
|
+
let preDispatchUserIds = null;
|
|
10037
11091
|
try {
|
|
10038
11092
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10039
11093
|
sessionId,
|
|
@@ -10043,7 +11097,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10043
11097
|
row.content,
|
|
10044
11098
|
options,
|
|
10045
11099
|
sendAttachments,
|
|
10046
|
-
this.openCodeClient
|
|
11100
|
+
this.openCodeClient,
|
|
11101
|
+
{
|
|
11102
|
+
onKnownUserIds: (ids) => {
|
|
11103
|
+
preDispatchUserIds = ids;
|
|
11104
|
+
},
|
|
11105
|
+
onKnownUserIdsError: (error2) => {
|
|
11106
|
+
this.log({
|
|
11107
|
+
level: "warn",
|
|
11108
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11109
|
+
conversation_id: row.conversation_id,
|
|
11110
|
+
message_id: row.id
|
|
11111
|
+
});
|
|
11112
|
+
}
|
|
11113
|
+
}
|
|
10047
11114
|
)
|
|
10048
11115
|
);
|
|
10049
11116
|
} catch (err) {
|
|
@@ -10076,6 +11143,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10076
11143
|
}
|
|
10077
11144
|
} else {
|
|
10078
11145
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11146
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10079
11147
|
}
|
|
10080
11148
|
this.log({
|
|
10081
11149
|
level: "error",
|
|
@@ -10124,39 +11192,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10124
11192
|
}
|
|
10125
11193
|
if (ocId === null) {
|
|
10126
11194
|
this.awaitingReadopt.delete(row.id);
|
|
10127
|
-
|
|
10128
|
-
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
|
|
10132
|
-
|
|
10133
|
-
this.log({
|
|
10134
|
-
level: "error",
|
|
10135
|
-
message: errorMessage3,
|
|
10136
|
-
conversation_id: row.conversation_id,
|
|
10137
|
-
message_id: row.id
|
|
10138
|
-
});
|
|
10139
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10140
|
-
this.log({
|
|
10141
|
-
level: "warn",
|
|
10142
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10143
|
-
conversation_id: row.conversation_id,
|
|
10144
|
-
message_id: row.id
|
|
10145
|
-
});
|
|
10146
|
-
});
|
|
10147
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10148
|
-
return;
|
|
10149
|
-
}
|
|
11195
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11196
|
+
sessionId,
|
|
11197
|
+
content: row.content,
|
|
11198
|
+
attempts: 0,
|
|
11199
|
+
knownUserIds: preDispatchUserIds
|
|
11200
|
+
});
|
|
10150
11201
|
this.log({
|
|
10151
11202
|
level: "warn",
|
|
10152
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11203
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10153
11204
|
conversation_id: row.conversation_id,
|
|
10154
11205
|
message_id: row.id
|
|
10155
11206
|
});
|
|
10156
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10157
11207
|
return;
|
|
10158
11208
|
}
|
|
10159
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10160
11209
|
const processedAtMs = this.processedAtMs(row);
|
|
10161
11210
|
this.registerReadopted(
|
|
10162
11211
|
readoptConv,
|
|
@@ -11062,7 +12111,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11062
12111
|
* watcher retries next tick within the
|
|
11063
12112
|
* deadline, Finding 4).
|
|
11064
12113
|
*/
|
|
11065
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12114
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11066
12115
|
const res = await this.fetchImpl(
|
|
11067
12116
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11068
12117
|
{
|
|
@@ -11080,7 +12129,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11080
12129
|
...title ? { title } : {},
|
|
11081
12130
|
...usage ? usage : {},
|
|
11082
12131
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11083
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12132
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12133
|
+
...error2 ? { error: error2 } : {}
|
|
11084
12134
|
})
|
|
11085
12135
|
}
|
|
11086
12136
|
);
|
|
@@ -12751,9 +13801,12 @@ async function driveChannels(state, driver) {
|
|
|
12751
13801
|
}
|
|
12752
13802
|
}
|
|
12753
13803
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12754
|
-
var
|
|
13804
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
13805
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
13806
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
13807
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
12755
13808
|
function shouldWarnForReclaimSkip(reason) {
|
|
12756
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13809
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12757
13810
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12758
13811
|
if (!version2) return false;
|
|
12759
13812
|
const major = Number(version2[1]);
|
|
@@ -12810,61 +13863,136 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12810
13863
|
async function runSweep(state, driver, config) {
|
|
12811
13864
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12812
13865
|
try {
|
|
12813
|
-
const
|
|
12814
|
-
|
|
12815
|
-
|
|
13866
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13867
|
+
const sessions = enumeration.sessions;
|
|
13868
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13869
|
+
if (enumerationReason !== null) {
|
|
13870
|
+
logActivity(state, {
|
|
12816
13871
|
type: "info",
|
|
12817
|
-
|
|
12818
|
-
|
|
12819
|
-
outcome: "sweep-skipped",
|
|
12820
|
-
reason: "opencode-unreachable"
|
|
12821
|
-
}
|
|
13872
|
+
level: "warn",
|
|
13873
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12822
13874
|
});
|
|
12823
|
-
return;
|
|
12824
13875
|
}
|
|
12825
|
-
const toDelete = selectSessionsToDelete(
|
|
12826
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12827
|
-
{
|
|
12828
|
-
maxAgeMs: config.maxAgeMs,
|
|
12829
|
-
maxCount: config.maxCount,
|
|
12830
|
-
nowMs: Date.now(),
|
|
12831
|
-
protectedIds: driver.protectedSessionIds()
|
|
12832
|
-
}
|
|
12833
|
-
);
|
|
12834
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12835
13876
|
let deleted = 0;
|
|
12836
13877
|
let failed = 0;
|
|
13878
|
+
let failed4xx = 0;
|
|
13879
|
+
let failed5xx = 0;
|
|
13880
|
+
let failedOtherStatus = 0;
|
|
13881
|
+
let failedThrew = 0;
|
|
12837
13882
|
let skippedNewlyActive = 0;
|
|
12838
|
-
|
|
12839
|
-
|
|
12840
|
-
|
|
12841
|
-
|
|
13883
|
+
let remaining = 0;
|
|
13884
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13885
|
+
if (sessions !== null) {
|
|
13886
|
+
const toDelete = selectSessionsToDelete(
|
|
13887
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13888
|
+
{
|
|
13889
|
+
maxAgeMs: config.maxAgeMs,
|
|
13890
|
+
maxCount: config.maxCount,
|
|
13891
|
+
nowMs: Date.now(),
|
|
13892
|
+
protectedIds: protectedNow
|
|
13893
|
+
}
|
|
13894
|
+
);
|
|
13895
|
+
const eligibleToDelete = toDelete.filter(
|
|
13896
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
13897
|
+
);
|
|
13898
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13899
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
13900
|
+
const answeredFailures = [];
|
|
13901
|
+
protectedNow = driver.protectedSessionIds();
|
|
13902
|
+
for (const id of deleteCandidates) {
|
|
13903
|
+
if (protectedNow.has(id)) {
|
|
13904
|
+
skippedNewlyActive++;
|
|
13905
|
+
logActivity(state, {
|
|
13906
|
+
type: "info",
|
|
13907
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13908
|
+
});
|
|
13909
|
+
continue;
|
|
13910
|
+
}
|
|
13911
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13912
|
+
if (outcome.ok) {
|
|
13913
|
+
deleted++;
|
|
13914
|
+
} else if (outcome.status === "threw") {
|
|
13915
|
+
failed++;
|
|
13916
|
+
failedThrew++;
|
|
13917
|
+
} else {
|
|
13918
|
+
failed++;
|
|
13919
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13920
|
+
if (statusClass === 4) failed4xx++;
|
|
13921
|
+
else if (statusClass === 5) failed5xx++;
|
|
13922
|
+
else failedOtherStatus++;
|
|
13923
|
+
answeredFailures.push({
|
|
13924
|
+
id,
|
|
13925
|
+
status: outcome.status,
|
|
13926
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
13927
|
+
});
|
|
13928
|
+
}
|
|
13929
|
+
}
|
|
13930
|
+
const givenUpFailures = [];
|
|
13931
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
13932
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
13933
|
+
} else {
|
|
13934
|
+
for (const failure of answeredFailures) {
|
|
13935
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
13936
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
13937
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
13938
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
13939
|
+
givenUpFailures.push(failure);
|
|
13940
|
+
}
|
|
13941
|
+
}
|
|
13942
|
+
}
|
|
13943
|
+
for (const failure of givenUpFailures) {
|
|
13944
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
13945
|
+
0,
|
|
13946
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
13947
|
+
);
|
|
13948
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12842
13949
|
type: "info",
|
|
12843
|
-
|
|
13950
|
+
level: "warn",
|
|
13951
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
13952
|
+
metadata: {
|
|
13953
|
+
outcome: "session-delete-given-up",
|
|
13954
|
+
session_id: failure.id,
|
|
13955
|
+
status: failure.status,
|
|
13956
|
+
body_excerpt: excerpt,
|
|
13957
|
+
given_up_count: givenUpFailures.length
|
|
13958
|
+
}
|
|
12844
13959
|
});
|
|
12845
|
-
continue;
|
|
12846
13960
|
}
|
|
12847
|
-
|
|
12848
|
-
|
|
13961
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13962
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13963
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13964
|
+
logActivity(state, {
|
|
13965
|
+
type: "info",
|
|
13966
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13967
|
+
});
|
|
12849
13968
|
}
|
|
12850
|
-
const
|
|
12851
|
-
|
|
12852
|
-
|
|
12853
|
-
|
|
12854
|
-
|
|
12855
|
-
|
|
13969
|
+
const sessionMetadata = {
|
|
13970
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13971
|
+
sessions_deleted: deleted,
|
|
13972
|
+
sessions_failed: failed,
|
|
13973
|
+
sessions_failed_4xx: failed4xx,
|
|
13974
|
+
sessions_failed_5xx: failed5xx,
|
|
13975
|
+
sessions_failed_other: failedOtherStatus,
|
|
13976
|
+
sessions_failed_threw: failedThrew,
|
|
13977
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13978
|
+
sessions_remaining: remaining,
|
|
13979
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13980
|
+
};
|
|
12856
13981
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12857
13982
|
dbPath: sessionDbPath(),
|
|
12858
|
-
|
|
13983
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13984
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13985
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12859
13986
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12860
13987
|
});
|
|
12861
13988
|
if (reclaimResult.ok) {
|
|
13989
|
+
state.sessionStoreHealth.observeWritable();
|
|
12862
13990
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12863
13991
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12864
13992
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12865
13993
|
logSessionCleanupActivity(state, "sweep", {
|
|
12866
13994
|
type: "info",
|
|
12867
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13995
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12868
13996
|
metadata: {
|
|
12869
13997
|
outcome: "reclaimed",
|
|
12870
13998
|
mode: reclaimResult.mode,
|
|
@@ -12872,25 +14000,27 @@ async function runSweep(state, driver, config) {
|
|
|
12872
14000
|
after_bytes: reclaimResult.afterBytes,
|
|
12873
14001
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12874
14002
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12875
|
-
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
14003
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
14004
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
14005
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
14006
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
14007
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14008
|
+
...sessionMetadata
|
|
12879
14009
|
}
|
|
12880
14010
|
});
|
|
12881
14011
|
} else {
|
|
14012
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
14013
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
14014
|
+
}
|
|
12882
14015
|
logSessionCleanupActivity(state, "sweep", {
|
|
12883
14016
|
type: "info",
|
|
12884
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
14017
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12885
14018
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12886
14019
|
metadata: {
|
|
12887
14020
|
outcome: "reclaim-skipped",
|
|
12888
14021
|
reason: reclaimResult.skipped,
|
|
12889
14022
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12890
|
-
|
|
12891
|
-
sessions_deleted: deleted,
|
|
12892
|
-
sessions_failed: failed,
|
|
12893
|
-
sessions_skipped_active: skippedNewlyActive
|
|
14023
|
+
...sessionMetadata
|
|
12894
14024
|
}
|
|
12895
14025
|
});
|
|
12896
14026
|
}
|
|
@@ -13147,6 +14277,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13147
14277
|
message: `Resource usage collection: ${warning2}`
|
|
13148
14278
|
});
|
|
13149
14279
|
}
|
|
14280
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14281
|
+
...usage,
|
|
14282
|
+
nowMs: Date.now(),
|
|
14283
|
+
state: state.lowDiskWarningState
|
|
14284
|
+
});
|
|
14285
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14286
|
+
if (lowDiskDecision.warning !== null) {
|
|
14287
|
+
logActivity(state, {
|
|
14288
|
+
type: "info",
|
|
14289
|
+
level: "warn",
|
|
14290
|
+
message: lowDiskDecision.warning
|
|
14291
|
+
});
|
|
14292
|
+
}
|
|
13150
14293
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13151
14294
|
if (result.ok) {
|
|
13152
14295
|
if (consecutiveFailures > 0) {
|
|
@@ -13393,6 +14536,7 @@ async function run(options) {
|
|
|
13393
14536
|
opencodeClient: null,
|
|
13394
14537
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13395
14538
|
sessionDbProvenanceAnomaly: false,
|
|
14539
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13396
14540
|
opencodeProcess: null,
|
|
13397
14541
|
stopOpenCodeLogTail: null,
|
|
13398
14542
|
litestreamProcess: null,
|
|
@@ -13404,12 +14548,14 @@ async function run(options) {
|
|
|
13404
14548
|
messageCount: 0,
|
|
13405
14549
|
lastProxiedActivityAt: null,
|
|
13406
14550
|
sessionCleanupTimers: [],
|
|
14551
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13407
14552
|
claudeUsageTimer: null,
|
|
13408
14553
|
claudeUsageRearm: null,
|
|
13409
14554
|
openaiUsageTimer: null,
|
|
13410
14555
|
openaiUsageRearm: null,
|
|
13411
14556
|
resourceUsageTimer: null,
|
|
13412
14557
|
stopResourceUsageSampling: null,
|
|
14558
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13413
14559
|
credentialSync: null,
|
|
13414
14560
|
authHeader: ""
|
|
13415
14561
|
};
|
|
@@ -13583,6 +14729,34 @@ async function run(options) {
|
|
|
13583
14729
|
} else {
|
|
13584
14730
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13585
14731
|
}
|
|
14732
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14733
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14734
|
+
log2(state, message, level);
|
|
14735
|
+
if (state.interactive && !state.json) {
|
|
14736
|
+
logActivity(state, { type: "info", level, message });
|
|
14737
|
+
}
|
|
14738
|
+
};
|
|
14739
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14740
|
+
if (vaultResolution.error) {
|
|
14741
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14742
|
+
logCredentialVaultResolution(
|
|
14743
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14744
|
+
"warn"
|
|
14745
|
+
);
|
|
14746
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14747
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14748
|
+
logCredentialVaultResolution(
|
|
14749
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14750
|
+
"info"
|
|
14751
|
+
);
|
|
14752
|
+
} else {
|
|
14753
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14754
|
+
logCredentialVaultResolution(
|
|
14755
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14756
|
+
"debug"
|
|
14757
|
+
);
|
|
14758
|
+
}
|
|
14759
|
+
}
|
|
13586
14760
|
if (options.restoreRunnerCredentials) {
|
|
13587
14761
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13588
14762
|
const credentialContext = {
|
|
@@ -13600,10 +14774,18 @@ async function run(options) {
|
|
|
13600
14774
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13601
14775
|
}
|
|
13602
14776
|
state.credentialSync?.arm();
|
|
13603
|
-
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13604
|
-
|
|
13605
|
-
|
|
13606
|
-
|
|
14777
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14778
|
+
() => ({
|
|
14779
|
+
agentId: state.agentId,
|
|
14780
|
+
authHeader: state.authHeader
|
|
14781
|
+
}),
|
|
14782
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14783
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14784
|
+
if (cause !== null && sessionId !== null) {
|
|
14785
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14786
|
+
}
|
|
14787
|
+
}
|
|
14788
|
+
);
|
|
13607
14789
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13608
14790
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13609
14791
|
openCodeActivityForwarder
|
|
@@ -13613,22 +14795,27 @@ async function run(options) {
|
|
|
13613
14795
|
openCodeLogTail.stop();
|
|
13614
14796
|
};
|
|
13615
14797
|
let sessionDbVerifyFatal = false;
|
|
14798
|
+
let sessionDbVerifyFatalReason;
|
|
13616
14799
|
if (!options.restoreSessionDb) {
|
|
13617
|
-
|
|
14800
|
+
logActivity(state, {
|
|
14801
|
+
type: "info",
|
|
14802
|
+
level: "warn",
|
|
14803
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14804
|
+
});
|
|
13618
14805
|
} else {
|
|
13619
14806
|
const health = await checkOpenCodeHealth(state.port);
|
|
13620
14807
|
if (health.healthy) {
|
|
13621
|
-
|
|
13622
|
-
|
|
13623
|
-
|
|
13624
|
-
"
|
|
13625
|
-
);
|
|
14808
|
+
logActivity(state, {
|
|
14809
|
+
type: "info",
|
|
14810
|
+
level: "warn",
|
|
14811
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14812
|
+
});
|
|
13626
14813
|
} else if (health.authFailed) {
|
|
13627
|
-
|
|
13628
|
-
|
|
13629
|
-
|
|
13630
|
-
"
|
|
13631
|
-
);
|
|
14814
|
+
logActivity(state, {
|
|
14815
|
+
type: "info",
|
|
14816
|
+
level: "warn",
|
|
14817
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14818
|
+
});
|
|
13632
14819
|
} else {
|
|
13633
14820
|
const result = await restoreAndVerifySessionDb({
|
|
13634
14821
|
dbPath: sessionDbPath(),
|
|
@@ -13645,12 +14832,13 @@ async function run(options) {
|
|
|
13645
14832
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13646
14833
|
});
|
|
13647
14834
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14835
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13648
14836
|
}
|
|
13649
14837
|
}
|
|
13650
14838
|
reportSessionDbRecovery(state);
|
|
13651
14839
|
if (sessionDbVerifyFatal) {
|
|
13652
14840
|
throw new Error(
|
|
13653
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14841
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13654
14842
|
);
|
|
13655
14843
|
}
|
|
13656
14844
|
applyRunnerOpenCodeConfig({
|
|
@@ -13895,6 +15083,7 @@ async function run(options) {
|
|
|
13895
15083
|
getAuthHeader: () => state.authHeader,
|
|
13896
15084
|
conversationFilter: state.conversationFilter,
|
|
13897
15085
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15086
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13898
15087
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13899
15088
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13900
15089
|
fileSyncDirectories,
|
|
@@ -13976,14 +15165,14 @@ async function run(options) {
|
|
|
13976
15165
|
},
|
|
13977
15166
|
onDisconnected: (code, reason, transition) => {
|
|
13978
15167
|
state.connected = false;
|
|
13979
|
-
const
|
|
15168
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13980
15169
|
logActivity(state, {
|
|
13981
15170
|
type: "info",
|
|
13982
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15171
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
13983
15172
|
});
|
|
13984
15173
|
emitAgentDisconnected(state.agentId, {
|
|
13985
15174
|
code,
|
|
13986
|
-
reason:
|
|
15175
|
+
reason: safeReason2,
|
|
13987
15176
|
connection_generation: transition.generation
|
|
13988
15177
|
});
|
|
13989
15178
|
if (state.interactive) displayStatus(state);
|
|
@@ -14146,6 +15335,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14146
15335
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14147
15336
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14148
15337
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15338
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14149
15339
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14150
15340
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14151
15341
|
"-a, --agent [id]",
|