@evident-ai/cli 3.5.2-dev.6f1ed0e → 3.5.2-dev.747ee34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1603 -508
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -703,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
703
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
704
704
|
});
|
|
705
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
706
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
707
732
|
const apiUrl = getApiUrlConfig();
|
|
708
733
|
try {
|
|
@@ -877,6 +902,76 @@ async function status(options = {}) {
|
|
|
877
902
|
process.exit(result.exitCode);
|
|
878
903
|
}
|
|
879
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// src/lib/claude-usage.ts
|
|
881
976
|
import { execFileSync } from "node:child_process";
|
|
882
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1069,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1069
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1070
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1071
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1072
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1073
1238
|
var TelemetryEventTypes = {
|
|
1074
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1157,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1157
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1158
1323
|
|
|
1159
1324
|
// src/lib/telemetry.ts
|
|
1160
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.747ee34" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1161
1326
|
function getCliVersion() {
|
|
1162
1327
|
return CLI_VERSION;
|
|
1163
1328
|
}
|
|
@@ -1318,8 +1483,316 @@ var EventTypes = {
|
|
|
1318
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1319
1484
|
};
|
|
1320
1485
|
|
|
1321
|
-
// src/lib/
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1322
1491
|
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
function errorMessage(error2) {
|
|
1522
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1523
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1524
|
+
}
|
|
1525
|
+
function errorMessages(error2) {
|
|
1526
|
+
const messages = [];
|
|
1527
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1528
|
+
let current = error2;
|
|
1529
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1530
|
+
visited.add(current);
|
|
1531
|
+
if (typeof current === "string") {
|
|
1532
|
+
messages.push(current);
|
|
1533
|
+
break;
|
|
1534
|
+
}
|
|
1535
|
+
if (current instanceof Error) {
|
|
1536
|
+
messages.push(current.name, current.message);
|
|
1537
|
+
current = current.cause;
|
|
1538
|
+
continue;
|
|
1539
|
+
}
|
|
1540
|
+
if (typeof current === "object") {
|
|
1541
|
+
const record = current;
|
|
1542
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1543
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1544
|
+
current = record.cause;
|
|
1545
|
+
continue;
|
|
1546
|
+
}
|
|
1547
|
+
break;
|
|
1548
|
+
}
|
|
1549
|
+
return messages;
|
|
1550
|
+
}
|
|
1551
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1552
|
+
return errorMessages(error2).some(
|
|
1553
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1554
|
+
message
|
|
1555
|
+
)
|
|
1556
|
+
);
|
|
1557
|
+
}
|
|
1558
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1559
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1560
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1561
|
+
function isCauseBearingText(text) {
|
|
1562
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1563
|
+
}
|
|
1564
|
+
function structuredLogFields(line) {
|
|
1565
|
+
const fields = [];
|
|
1566
|
+
for (let index = 0; index < line.length; index++) {
|
|
1567
|
+
if (line[index] === '"') {
|
|
1568
|
+
index++;
|
|
1569
|
+
while (index < line.length) {
|
|
1570
|
+
if (line[index] === "\\") {
|
|
1571
|
+
index++;
|
|
1572
|
+
} else if (line[index] === '"') {
|
|
1573
|
+
break;
|
|
1574
|
+
}
|
|
1575
|
+
index++;
|
|
1576
|
+
}
|
|
1577
|
+
continue;
|
|
1578
|
+
}
|
|
1579
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1580
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1581
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1582
|
+
const valueStart = index + key.length;
|
|
1583
|
+
if (line[valueStart] !== '"') {
|
|
1584
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1585
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1586
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
let valueEnd = valueStart + 1;
|
|
1590
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1591
|
+
if (line[valueEnd] === "\\") {
|
|
1592
|
+
valueEnd++;
|
|
1593
|
+
} else if (line[valueEnd] === '"') {
|
|
1594
|
+
break;
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1598
|
+
index = valueEnd;
|
|
1599
|
+
}
|
|
1600
|
+
return fields;
|
|
1601
|
+
}
|
|
1602
|
+
function structuredCauseMessages(text) {
|
|
1603
|
+
const fields = structuredLogFields(text);
|
|
1604
|
+
if (fields.length > 0) return fields;
|
|
1605
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1606
|
+
}
|
|
1607
|
+
function causeTokenFromText(text) {
|
|
1608
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1609
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1610
|
+
const match = pattern.exec(text);
|
|
1611
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1612
|
+
return match[1];
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
return null;
|
|
1616
|
+
}
|
|
1617
|
+
function sessionStoreCauseToken(value) {
|
|
1618
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1619
|
+
const token = causeTokenFromText(message);
|
|
1620
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1621
|
+
}
|
|
1622
|
+
return null;
|
|
1623
|
+
}
|
|
1624
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1625
|
+
try {
|
|
1626
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1627
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1628
|
+
if (availableBytes < requiredBytes) {
|
|
1629
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1630
|
+
}
|
|
1631
|
+
return null;
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
function readLogicalBytes(db) {
|
|
1637
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1638
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1639
|
+
return pageCount * pageSize;
|
|
1640
|
+
}
|
|
1641
|
+
function readFreelistCount(db) {
|
|
1642
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1643
|
+
}
|
|
1644
|
+
function readCheckpointResult(db) {
|
|
1645
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1646
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1647
|
+
}
|
|
1648
|
+
async function probeReclaimAvailability(input) {
|
|
1649
|
+
const { dbPath, requiredBytes } = input;
|
|
1650
|
+
let sqlite;
|
|
1651
|
+
try {
|
|
1652
|
+
sqlite = await import("node:sqlite");
|
|
1653
|
+
} catch (err) {
|
|
1654
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1655
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1656
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1657
|
+
}
|
|
1658
|
+
let autoVacuum = null;
|
|
1659
|
+
try {
|
|
1660
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1661
|
+
try {
|
|
1662
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1663
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1664
|
+
} finally {
|
|
1665
|
+
db.close();
|
|
1666
|
+
}
|
|
1667
|
+
} catch (err) {
|
|
1668
|
+
console.warn(
|
|
1669
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
if (autoVacuum !== 0) return null;
|
|
1673
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1674
|
+
}
|
|
1675
|
+
async function reclaimSessionDbSpace(input) {
|
|
1676
|
+
const {
|
|
1677
|
+
dbPath,
|
|
1678
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1679
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1680
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1681
|
+
now = () => performance.now(),
|
|
1682
|
+
allowFullVacuum = true
|
|
1683
|
+
} = input;
|
|
1684
|
+
let sqlite;
|
|
1685
|
+
try {
|
|
1686
|
+
sqlite = await import("node:sqlite");
|
|
1687
|
+
} catch (err) {
|
|
1688
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1689
|
+
console.warn(
|
|
1690
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1691
|
+
);
|
|
1692
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1693
|
+
}
|
|
1694
|
+
const { DatabaseSync } = sqlite;
|
|
1695
|
+
let db;
|
|
1696
|
+
try {
|
|
1697
|
+
db = new DatabaseSync(dbPath);
|
|
1698
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1699
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1700
|
+
if (autoVacuum === 0) {
|
|
1701
|
+
if (!allowFullVacuum) {
|
|
1702
|
+
console.warn(
|
|
1703
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1704
|
+
);
|
|
1705
|
+
return {
|
|
1706
|
+
ok: false,
|
|
1707
|
+
skipped: "full-vacuum-blocked",
|
|
1708
|
+
detail: "a session turn is live"
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1712
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1713
|
+
if (skipReason !== null) {
|
|
1714
|
+
console.warn(
|
|
1715
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1716
|
+
);
|
|
1717
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1718
|
+
}
|
|
1719
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1720
|
+
const freelistBefore = readFreelistCount(db);
|
|
1721
|
+
const startedAt = now();
|
|
1722
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1723
|
+
db.exec("VACUUM");
|
|
1724
|
+
const afterBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistAfter = readFreelistCount(db);
|
|
1726
|
+
const checkpoint = readCheckpointResult(db);
|
|
1727
|
+
return {
|
|
1728
|
+
ok: true,
|
|
1729
|
+
mode: "convert",
|
|
1730
|
+
beforeBytes,
|
|
1731
|
+
afterBytes,
|
|
1732
|
+
freelistBefore,
|
|
1733
|
+
freelistAfter,
|
|
1734
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1735
|
+
stoppedBy: "freelist-drained",
|
|
1736
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1737
|
+
checkpoint
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
if (autoVacuum === 2) {
|
|
1741
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1742
|
+
const freelistBefore = readFreelistCount(db);
|
|
1743
|
+
const startedAt = now();
|
|
1744
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1745
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1746
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1747
|
+
let pagesAttempted = 0;
|
|
1748
|
+
let freelistAfter = freelistBefore;
|
|
1749
|
+
let stoppedBy = "freelist-drained";
|
|
1750
|
+
while (freelistAfter > 0) {
|
|
1751
|
+
if (pagesAttempted >= pageLimit) {
|
|
1752
|
+
stoppedBy = "page-ceiling";
|
|
1753
|
+
break;
|
|
1754
|
+
}
|
|
1755
|
+
if (now() - startedAt >= budget) {
|
|
1756
|
+
stoppedBy = "time-budget";
|
|
1757
|
+
break;
|
|
1758
|
+
}
|
|
1759
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1760
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1761
|
+
pagesAttempted += pages;
|
|
1762
|
+
freelistAfter = readFreelistCount(db);
|
|
1763
|
+
}
|
|
1764
|
+
const afterBytes = readLogicalBytes(db);
|
|
1765
|
+
const checkpoint = readCheckpointResult(db);
|
|
1766
|
+
return {
|
|
1767
|
+
ok: true,
|
|
1768
|
+
mode: "incremental",
|
|
1769
|
+
beforeBytes,
|
|
1770
|
+
afterBytes,
|
|
1771
|
+
freelistBefore,
|
|
1772
|
+
freelistAfter,
|
|
1773
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1774
|
+
stoppedBy,
|
|
1775
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1776
|
+
checkpoint
|
|
1777
|
+
};
|
|
1778
|
+
}
|
|
1779
|
+
console.warn(
|
|
1780
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1781
|
+
);
|
|
1782
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1783
|
+
} catch (err) {
|
|
1784
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1785
|
+
return {
|
|
1786
|
+
ok: false,
|
|
1787
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1788
|
+
detail: errorMessage(err)
|
|
1789
|
+
};
|
|
1790
|
+
} finally {
|
|
1791
|
+
db?.close();
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1323
1796
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1324
1797
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1325
1798
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1336,11 +1809,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1336
1809
|
if (!levelMatch) return null;
|
|
1337
1810
|
const level = levelMatch[1].toUpperCase();
|
|
1338
1811
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1339
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1812
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1813
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1340
1814
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1341
1815
|
const parsed = {
|
|
1342
1816
|
level: level === "WARN" ? "warn" : "error",
|
|
1343
|
-
sessionID: sessionMatch?.[1]
|
|
1817
|
+
sessionID: sessionMatch?.[1],
|
|
1818
|
+
messageID: messageIDMatch?.[1]
|
|
1344
1819
|
};
|
|
1345
1820
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1346
1821
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1348,12 +1823,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1348
1823
|
}
|
|
1349
1824
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1350
1825
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1351
|
-
function createOpenCodeActivityForwarder(getContext) {
|
|
1826
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1352
1827
|
let buffer = Buffer.alloc(0);
|
|
1353
1828
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1354
1829
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1355
1830
|
const metadata = {
|
|
1356
1831
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1832
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1357
1833
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1358
1834
|
};
|
|
1359
1835
|
forwardRunnerActivity(
|
|
@@ -1379,9 +1855,24 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1379
1855
|
};
|
|
1380
1856
|
const flushLine = (line) => {
|
|
1381
1857
|
const parsed = parseOpenCodeLogLine(line);
|
|
1858
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1859
|
+
const cause = sessionStoreCauseToken(line);
|
|
1860
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1861
|
+
try {
|
|
1862
|
+
onSessionStoreWriteFailure({
|
|
1863
|
+
isDiskFailure,
|
|
1864
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1865
|
+
cause
|
|
1866
|
+
});
|
|
1867
|
+
} catch (err) {
|
|
1868
|
+
reportTelemetryDiagnostic(
|
|
1869
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1382
1873
|
if (!parsed) return;
|
|
1383
1874
|
const now = Date.now();
|
|
1384
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1875
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1385
1876
|
expireRepeatStates(now, key);
|
|
1386
1877
|
if (key === void 0) {
|
|
1387
1878
|
forwardLine(line, parsed);
|
|
@@ -1427,34 +1918,21 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1427
1918
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1428
1919
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1429
1920
|
var MAX_METADATA_ENTRIES = 20;
|
|
1430
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1431
|
-
function redactHomeDirectory(message) {
|
|
1432
|
-
const home = homedir2();
|
|
1433
|
-
if (!home || home === "/") {
|
|
1434
|
-
return message;
|
|
1435
|
-
}
|
|
1436
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1437
|
-
let result = "";
|
|
1438
|
-
let searchFrom = 0;
|
|
1439
|
-
let homeIndex;
|
|
1440
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1441
|
-
const homeEnd = homeIndex + home.length;
|
|
1442
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1443
|
-
const nextCharacter = message[homeEnd];
|
|
1444
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1445
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1446
|
-
result += atPathBoundary ? "~" : home;
|
|
1447
|
-
searchFrom = homeEnd;
|
|
1448
|
-
}
|
|
1449
|
-
return result + message.slice(searchFrom);
|
|
1450
|
-
}
|
|
1451
|
-
function redactDiagnosticText(message) {
|
|
1452
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1453
|
-
return redactHomeDirectory(redacted);
|
|
1454
|
-
}
|
|
1455
1921
|
function truncate(message) {
|
|
1456
1922
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1457
|
-
|
|
1923
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1924
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1925
|
+
let previousDropped = -1;
|
|
1926
|
+
while (dropped !== previousDropped) {
|
|
1927
|
+
previousDropped = dropped;
|
|
1928
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1929
|
+
dropped = nextDropped;
|
|
1930
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1931
|
+
}
|
|
1932
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1933
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1934
|
+
const tailLength = remaining - headLength;
|
|
1935
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1458
1936
|
}
|
|
1459
1937
|
function sanitiseMetadata(metadata) {
|
|
1460
1938
|
if (!metadata) return {};
|
|
@@ -1664,12 +2142,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1664
2142
|
metadata: withoutContractFields(record),
|
|
1665
2143
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1666
2144
|
};
|
|
1667
|
-
case "session_db_boot_refused":
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
2145
|
+
case "session_db_boot_refused": {
|
|
2146
|
+
const message = (() => {
|
|
2147
|
+
switch (record.reason) {
|
|
2148
|
+
case "local_discard_failed":
|
|
2149
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2150
|
+
case "replica_separation_unproven":
|
|
2151
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2152
|
+
case "verification_timeout_before_separation":
|
|
2153
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2154
|
+
case "verification_timeout_during_separation":
|
|
2155
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2156
|
+
default:
|
|
2157
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2158
|
+
}
|
|
2159
|
+
})();
|
|
2160
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2161
|
+
}
|
|
1673
2162
|
case "schema_provenance_mismatch":
|
|
1674
2163
|
return {
|
|
1675
2164
|
level,
|
|
@@ -1919,7 +2408,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1919
2408
|
return null;
|
|
1920
2409
|
}
|
|
1921
2410
|
}
|
|
1922
|
-
function
|
|
2411
|
+
function roleOf2(m) {
|
|
1923
2412
|
if (!m || typeof m !== "object") return void 0;
|
|
1924
2413
|
if (typeof m.role === "string") return m.role;
|
|
1925
2414
|
const infoRole = m.info?.role;
|
|
@@ -1933,13 +2422,13 @@ function createdOf(m) {
|
|
|
1933
2422
|
if (!m || typeof m !== "object") return void 0;
|
|
1934
2423
|
return m.info?.time?.created ?? m.time?.created;
|
|
1935
2424
|
}
|
|
1936
|
-
function
|
|
2425
|
+
function idOf2(m) {
|
|
1937
2426
|
if (!m || typeof m !== "object") return void 0;
|
|
1938
2427
|
if (typeof m.id === "string") return m.id;
|
|
1939
2428
|
const infoId = m.info?.id;
|
|
1940
2429
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1941
2430
|
}
|
|
1942
|
-
function
|
|
2431
|
+
function parentIdOf2(m) {
|
|
1943
2432
|
if (!m || typeof m !== "object") return void 0;
|
|
1944
2433
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1945
2434
|
const infoParent = m.info?.parentID;
|
|
@@ -1955,8 +2444,12 @@ function errorOf(m) {
|
|
|
1955
2444
|
if (!m || typeof m !== "object") return void 0;
|
|
1956
2445
|
return m.info?.error ?? m.error;
|
|
1957
2446
|
}
|
|
2447
|
+
function isTerminalStopAnswer(m) {
|
|
2448
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2449
|
+
}
|
|
1958
2450
|
function isAssistantInFlight(m) {
|
|
1959
2451
|
if (completedOf(m) == null) return true;
|
|
2452
|
+
if (errorOf(m) != null) return false;
|
|
1960
2453
|
return finishOf(m) === "tool-calls";
|
|
1961
2454
|
}
|
|
1962
2455
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2004,9 +2497,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2004
2497
|
function isSessionActivelyGenerating(messages) {
|
|
2005
2498
|
if (!messages || messages.length === 0) return false;
|
|
2006
2499
|
const last = messages[messages.length - 1];
|
|
2007
|
-
if (
|
|
2500
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2008
2501
|
return completedOf(last) == null;
|
|
2009
2502
|
}
|
|
2503
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2010
2504
|
function sessionLastActivityMs(session) {
|
|
2011
2505
|
const candidates = [
|
|
2012
2506
|
session.time?.updated,
|
|
@@ -2021,30 +2515,60 @@ function sessionLastActivityMs(session) {
|
|
|
2021
2515
|
}
|
|
2022
2516
|
return null;
|
|
2023
2517
|
}
|
|
2024
|
-
|
|
2025
|
-
|
|
2518
|
+
function sessionListPath(options) {
|
|
2519
|
+
const params = new URLSearchParams();
|
|
2520
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2521
|
+
const query = params.toString();
|
|
2522
|
+
return query ? `/session?${query}` : "/session";
|
|
2523
|
+
}
|
|
2524
|
+
async function listSessionPage(port, client, options) {
|
|
2525
|
+
const path = sessionListPath(options);
|
|
2026
2526
|
try {
|
|
2027
|
-
const res = await requestWithClient(port, client,
|
|
2527
|
+
const res = await requestWithClient(port, client, path);
|
|
2028
2528
|
if (!res.ok) return null;
|
|
2029
2529
|
const body = await res.json();
|
|
2030
2530
|
return Array.isArray(body) ? body : null;
|
|
2031
2531
|
} catch (error2) {
|
|
2032
2532
|
console.error(
|
|
2033
|
-
`[
|
|
2533
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2034
2534
|
);
|
|
2035
2535
|
return null;
|
|
2036
2536
|
}
|
|
2037
2537
|
}
|
|
2538
|
+
async function listSessions(port, client, options) {
|
|
2539
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2540
|
+
return listSessionPage(port, client, options);
|
|
2541
|
+
}
|
|
2542
|
+
async function listAllSessions(port, client) {
|
|
2543
|
+
if (client?.version === "v2") {
|
|
2544
|
+
const sessions2 = await listV2Sessions(client);
|
|
2545
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2546
|
+
}
|
|
2547
|
+
const sessions = await listSessionPage(port, client, {
|
|
2548
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2549
|
+
});
|
|
2550
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2551
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2552
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2553
|
+
}
|
|
2554
|
+
return { sessions };
|
|
2555
|
+
}
|
|
2038
2556
|
async function deleteSession(port, id, client) {
|
|
2039
2557
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2040
2558
|
try {
|
|
2041
|
-
const res = await requestWithClient(
|
|
2042
|
-
|
|
2559
|
+
const res = await requestWithClient(
|
|
2560
|
+
port,
|
|
2561
|
+
client,
|
|
2562
|
+
`/session/${id}`,
|
|
2563
|
+
{ method: "DELETE" },
|
|
2564
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2565
|
+
);
|
|
2566
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
2043
2567
|
} catch (error2) {
|
|
2044
2568
|
console.error(
|
|
2045
2569
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2046
2570
|
);
|
|
2047
|
-
return false;
|
|
2571
|
+
return { ok: false, status: "threw" };
|
|
2048
2572
|
}
|
|
2049
2573
|
}
|
|
2050
2574
|
async function sessionExists(port, id, client) {
|
|
@@ -2282,11 +2806,31 @@ function messageText(m) {
|
|
|
2282
2806
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2283
2807
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2284
2808
|
}
|
|
2285
|
-
|
|
2809
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2810
|
+
if (!messages || messages.length === 0) return null;
|
|
2811
|
+
let best = null;
|
|
2812
|
+
for (const message of messages) {
|
|
2813
|
+
if (roleOf2(message) !== "user") continue;
|
|
2814
|
+
const id = idOf2(message);
|
|
2815
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2816
|
+
if (messageText(message) !== content) continue;
|
|
2817
|
+
const created = createdOf(message) ?? 0;
|
|
2818
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2819
|
+
}
|
|
2820
|
+
return best?.id ?? null;
|
|
2821
|
+
}
|
|
2822
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2286
2823
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2287
2824
|
const knownUserIds = new Set(
|
|
2288
|
-
(before ?? []).filter((m) =>
|
|
2825
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2289
2826
|
);
|
|
2827
|
+
if (callbacks) {
|
|
2828
|
+
try {
|
|
2829
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2830
|
+
} catch (error2) {
|
|
2831
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2290
2834
|
const parts = [{ type: "text", text: content }];
|
|
2291
2835
|
let pendingOutcomes = null;
|
|
2292
2836
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2325,22 +2869,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2325
2869
|
const READ_BACK_DELAY_MS = 150;
|
|
2326
2870
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2327
2871
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
const id = idOf(m);
|
|
2333
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2334
|
-
if (messageText(m) !== content) continue;
|
|
2335
|
-
const created = createdOf(m) ?? 0;
|
|
2336
|
-
if (best === null || created > best.created) {
|
|
2337
|
-
best = { id, created };
|
|
2338
|
-
}
|
|
2339
|
-
}
|
|
2340
|
-
if (best) {
|
|
2341
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2342
|
-
return best.id;
|
|
2343
|
-
}
|
|
2872
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2873
|
+
if (dispatchedUserMessageId !== null) {
|
|
2874
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2875
|
+
return dispatchedUserMessageId;
|
|
2344
2876
|
}
|
|
2345
2877
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2346
2878
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2351,57 +2883,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2351
2883
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2352
2884
|
if (!messages || messages.length === 0) return null;
|
|
2353
2885
|
const byParent = messages.find(
|
|
2354
|
-
(m) =>
|
|
2886
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2355
2887
|
);
|
|
2356
2888
|
if (byParent) return byParent;
|
|
2357
|
-
const userIndex = messages.findIndex((m) =>
|
|
2889
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2358
2890
|
if (userIndex === -1) return null;
|
|
2359
2891
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2360
|
-
if (
|
|
2892
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2361
2893
|
}
|
|
2362
2894
|
return null;
|
|
2363
2895
|
}
|
|
2364
2896
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2365
2897
|
if (!messages || messages.length === 0) return null;
|
|
2366
2898
|
let lastCorrelated = null;
|
|
2367
|
-
let lastNonErrored = null;
|
|
2368
2899
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2369
2900
|
const m = messages[i];
|
|
2370
|
-
if (
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2901
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2902
|
+
lastCorrelated = m;
|
|
2903
|
+
break;
|
|
2904
|
+
}
|
|
2905
|
+
if (lastCorrelated) {
|
|
2906
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2907
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2908
|
+
const m = messages[i];
|
|
2909
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2910
|
+
return m;
|
|
2911
|
+
}
|
|
2375
2912
|
}
|
|
2913
|
+
return lastCorrelated;
|
|
2376
2914
|
}
|
|
2377
|
-
|
|
2378
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2915
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2379
2916
|
if (userIndex === -1) return null;
|
|
2380
2917
|
let last = null;
|
|
2381
|
-
let
|
|
2918
|
+
let lastAssistantIndex = -1;
|
|
2382
2919
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2383
|
-
const role =
|
|
2920
|
+
const role = roleOf2(messages[i]);
|
|
2384
2921
|
if (role === "user") break;
|
|
2385
2922
|
if (role === "assistant") {
|
|
2386
2923
|
last = messages[i];
|
|
2387
|
-
|
|
2924
|
+
lastAssistantIndex = i;
|
|
2388
2925
|
}
|
|
2389
2926
|
}
|
|
2390
|
-
|
|
2927
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2928
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2929
|
+
const m = messages[i];
|
|
2930
|
+
if (roleOf2(m) === "user") break;
|
|
2931
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2932
|
+
}
|
|
2933
|
+
return last;
|
|
2391
2934
|
}
|
|
2392
2935
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2393
2936
|
if (!messages || messages.length === 0) return [];
|
|
2394
2937
|
const byParent = messages.filter(
|
|
2395
|
-
(message) =>
|
|
2938
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2396
2939
|
);
|
|
2397
2940
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2398
2941
|
if (assistants.length === 0) {
|
|
2399
|
-
const userIndex = messages.findIndex((message) =>
|
|
2942
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2400
2943
|
if (userIndex === -1) return [];
|
|
2401
2944
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2402
2945
|
const message = messages[i];
|
|
2403
|
-
if (
|
|
2404
|
-
if (
|
|
2946
|
+
if (roleOf2(message) === "user") break;
|
|
2947
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2405
2948
|
}
|
|
2406
2949
|
}
|
|
2407
2950
|
const refs = [];
|
|
@@ -2441,7 +2984,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2441
2984
|
if (!messages || messages.length === 0) return [];
|
|
2442
2985
|
const calls = [];
|
|
2443
2986
|
for (const message of messages) {
|
|
2444
|
-
if (
|
|
2987
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2445
2988
|
for (const part of message.parts ?? []) {
|
|
2446
2989
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2447
2990
|
continue;
|
|
@@ -2471,7 +3014,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2471
3014
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2472
3015
|
const unattributed = [];
|
|
2473
3016
|
for (const message of messages ?? []) {
|
|
2474
|
-
if (
|
|
3017
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2475
3018
|
const created = finiteNumber(createdOf(message));
|
|
2476
3019
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2477
3020
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2493,8 +3036,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2493
3036
|
}
|
|
2494
3037
|
function sumAssistantUsage(messages) {
|
|
2495
3038
|
if (!messages || messages.length === 0) return null;
|
|
2496
|
-
const
|
|
2497
|
-
const
|
|
3039
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3040
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3041
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2498
3042
|
let sawAnyUsage = false;
|
|
2499
3043
|
let inputSum = 0;
|
|
2500
3044
|
let outputSum = 0;
|
|
@@ -2548,13 +3092,11 @@ function sumAssistantUsage(messages) {
|
|
|
2548
3092
|
function messageUsage(messages, userMessageId) {
|
|
2549
3093
|
if (!messages || messages.length === 0) return null;
|
|
2550
3094
|
const byParentAll = messages.filter(
|
|
2551
|
-
(m) =>
|
|
3095
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2552
3096
|
);
|
|
2553
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2554
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2555
3097
|
let correlated;
|
|
2556
|
-
if (
|
|
2557
|
-
correlated =
|
|
3098
|
+
if (byParentAll.length > 0) {
|
|
3099
|
+
correlated = byParentAll;
|
|
2558
3100
|
} else {
|
|
2559
3101
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2560
3102
|
correlated = reply ? [reply] : [];
|
|
@@ -2564,7 +3106,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2564
3106
|
}
|
|
2565
3107
|
function messageRunState(messages, userMessageId) {
|
|
2566
3108
|
if (!messages || messages.length === 0) return "unknown";
|
|
2567
|
-
const hasUser = messages.some((m) =>
|
|
3109
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2568
3110
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2569
3111
|
if (!hasUser) {
|
|
2570
3112
|
if (!reply) return "unknown";
|
|
@@ -2600,14 +3142,38 @@ function messageError(messages, userMessageId) {
|
|
|
2600
3142
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2601
3143
|
const error2 = errorOf(reply);
|
|
2602
3144
|
if (error2 == null) return null;
|
|
2603
|
-
|
|
2604
|
-
if (typeof error2 === "
|
|
3145
|
+
let rendered;
|
|
3146
|
+
if (typeof error2 === "string") {
|
|
3147
|
+
rendered = error2;
|
|
3148
|
+
} else if (typeof error2 === "object") {
|
|
2605
3149
|
const e = error2;
|
|
2606
3150
|
const dataMessage = e.data?.message;
|
|
2607
|
-
if (typeof dataMessage === "string")
|
|
2608
|
-
|
|
3151
|
+
if (typeof dataMessage === "string") {
|
|
3152
|
+
rendered = dataMessage;
|
|
3153
|
+
} else if (typeof e.message === "string") {
|
|
3154
|
+
rendered = e.message;
|
|
3155
|
+
} else {
|
|
3156
|
+
rendered = "The agent run failed.";
|
|
3157
|
+
}
|
|
3158
|
+
} else {
|
|
3159
|
+
rendered = "The agent run failed.";
|
|
2609
3160
|
}
|
|
2610
|
-
|
|
3161
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3162
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3163
|
+
) : void 0;
|
|
3164
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3165
|
+
}
|
|
3166
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3167
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3168
|
+
if (!reply) return null;
|
|
3169
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3170
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3171
|
+
return {
|
|
3172
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3173
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3174
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3175
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3176
|
+
};
|
|
2611
3177
|
}
|
|
2612
3178
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2613
3179
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2656,7 +3222,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2656
3222
|
if (!messages || messages.length === 0) return null;
|
|
2657
3223
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2658
3224
|
const message = messages[i];
|
|
2659
|
-
if (
|
|
3225
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2660
3226
|
const created = createdOf(message);
|
|
2661
3227
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2662
3228
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2688,25 +3254,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2688
3254
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2689
3255
|
if (!messages || messages.length === 0) return false;
|
|
2690
3256
|
return messages.some(
|
|
2691
|
-
(m) =>
|
|
3257
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2692
3258
|
);
|
|
2693
3259
|
}
|
|
2694
3260
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2695
3261
|
if (!messages || messages.length === 0) return false;
|
|
2696
|
-
const userIndex = messages.findIndex((message) =>
|
|
3262
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2697
3263
|
if (userIndex === -1) return false;
|
|
2698
3264
|
let hasLaterUser = false;
|
|
2699
3265
|
let hasStartedLaterUser = false;
|
|
2700
3266
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2701
3267
|
const message = messages[i];
|
|
2702
|
-
if (
|
|
3268
|
+
if (roleOf2(message) !== "user") continue;
|
|
2703
3269
|
hasLaterUser = true;
|
|
2704
|
-
const laterUserMessageId =
|
|
3270
|
+
const laterUserMessageId = idOf2(message);
|
|
2705
3271
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2706
3272
|
return false;
|
|
2707
3273
|
}
|
|
2708
3274
|
if (messages.some(
|
|
2709
|
-
(candidate) =>
|
|
3275
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2710
3276
|
)) {
|
|
2711
3277
|
hasStartedLaterUser = true;
|
|
2712
3278
|
}
|
|
@@ -3177,13 +3743,17 @@ async function listV2Sessions(client) {
|
|
|
3177
3743
|
}
|
|
3178
3744
|
async function deleteV2Session(client, sessionId) {
|
|
3179
3745
|
try {
|
|
3180
|
-
await client.request(
|
|
3181
|
-
|
|
3746
|
+
const response = await client.request(
|
|
3747
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3748
|
+
{ method: "DELETE" },
|
|
3749
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3750
|
+
);
|
|
3751
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3182
3752
|
} catch (error2) {
|
|
3183
3753
|
console.error(
|
|
3184
3754
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3185
3755
|
);
|
|
3186
|
-
return false;
|
|
3756
|
+
return { ok: false, status: "threw" };
|
|
3187
3757
|
}
|
|
3188
3758
|
}
|
|
3189
3759
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3805,9 +4375,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3805
4375
|
|
|
3806
4376
|
// src/lib/opencode/session-db-boot.ts
|
|
3807
4377
|
import { spawn as spawn2 } from "node:child_process";
|
|
3808
|
-
import { mkdirSync, statSync as
|
|
4378
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3809
4379
|
import { homedir as homedir4 } from "node:os";
|
|
3810
|
-
import { dirname as
|
|
4380
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3811
4381
|
|
|
3812
4382
|
// src/lib/runner-synchroniser.ts
|
|
3813
4383
|
import { spawn } from "node:child_process";
|
|
@@ -3887,9 +4457,12 @@ function runSynchroniser(args, opts) {
|
|
|
3887
4457
|
|
|
3888
4458
|
// src/lib/opencode/session-db-boot.ts
|
|
3889
4459
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3890
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4460
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3891
4461
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3892
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4462
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4463
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4464
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4465
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3893
4466
|
function commandError(result) {
|
|
3894
4467
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3895
4468
|
}
|
|
@@ -3932,7 +4505,7 @@ function clearMarker(options) {
|
|
|
3932
4505
|
function markNoReplicate(options, message) {
|
|
3933
4506
|
if (options.noReplicateMarker) {
|
|
3934
4507
|
try {
|
|
3935
|
-
mkdirSync(
|
|
4508
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3936
4509
|
writeFileSync(options.noReplicateMarker, "");
|
|
3937
4510
|
} catch (error2) {
|
|
3938
4511
|
options.log(
|
|
@@ -4057,7 +4630,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4057
4630
|
return null;
|
|
4058
4631
|
}
|
|
4059
4632
|
try {
|
|
4060
|
-
if (
|
|
4633
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4061
4634
|
} catch (error2) {
|
|
4062
4635
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4063
4636
|
options.log(
|
|
@@ -4087,7 +4660,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4087
4660
|
return null;
|
|
4088
4661
|
}
|
|
4089
4662
|
try {
|
|
4090
|
-
mkdirSync(
|
|
4663
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
4091
4664
|
writeFileSync(configPath, rendered.stdout);
|
|
4092
4665
|
} catch (error2) {
|
|
4093
4666
|
options.log(
|
|
@@ -4200,20 +4773,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4200
4773
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4201
4774
|
env: {
|
|
4202
4775
|
...env,
|
|
4203
|
-
// The synchroniser reads
|
|
4204
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4776
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4205
4777
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4206
4778
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4779
|
+
),
|
|
4780
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4781
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4782
|
+
),
|
|
4783
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4784
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4785
|
+
),
|
|
4786
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4787
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4207
4788
|
)
|
|
4208
4789
|
}
|
|
4209
4790
|
});
|
|
4210
4791
|
logSynchroniserDiagnostics(result, options);
|
|
4211
4792
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4793
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4794
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4795
|
+
options.log(
|
|
4796
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4797
|
+
"error"
|
|
4798
|
+
);
|
|
4799
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4800
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4801
|
+
}
|
|
4212
4802
|
options.log(
|
|
4213
4803
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4214
4804
|
"warn"
|
|
4215
4805
|
);
|
|
4216
|
-
return false;
|
|
4806
|
+
return { verifyFatal: false };
|
|
4217
4807
|
}
|
|
4218
4808
|
if (result.code === 34) {
|
|
4219
4809
|
reportRecord(
|
|
@@ -4223,14 +4813,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4223
4813
|
null,
|
|
4224
4814
|
options
|
|
4225
4815
|
);
|
|
4226
|
-
return true;
|
|
4816
|
+
return { verifyFatal: true };
|
|
4227
4817
|
}
|
|
4228
4818
|
if (result.code === 33) {
|
|
4229
4819
|
options.log(
|
|
4230
4820
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4231
4821
|
"warn"
|
|
4232
4822
|
);
|
|
4233
|
-
return false;
|
|
4823
|
+
return { verifyFatal: false };
|
|
4234
4824
|
}
|
|
4235
4825
|
if (result.code !== 0) {
|
|
4236
4826
|
options.log(
|
|
@@ -4238,17 +4828,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4238
4828
|
"warn"
|
|
4239
4829
|
);
|
|
4240
4830
|
}
|
|
4241
|
-
return false;
|
|
4831
|
+
return { verifyFatal: false };
|
|
4242
4832
|
}
|
|
4243
4833
|
options.log(
|
|
4244
4834
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4245
4835
|
"debug"
|
|
4246
4836
|
);
|
|
4247
|
-
return false;
|
|
4837
|
+
return { verifyFatal: false };
|
|
4248
4838
|
}
|
|
4249
4839
|
function fileExists(path) {
|
|
4250
4840
|
try {
|
|
4251
|
-
|
|
4841
|
+
statSync3(path);
|
|
4252
4842
|
return true;
|
|
4253
4843
|
} catch (error2) {
|
|
4254
4844
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4331,13 +4921,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4331
4921
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4332
4922
|
return { verifyFatal: false };
|
|
4333
4923
|
}
|
|
4334
|
-
return
|
|
4924
|
+
return verifySessionDb(options, configPath, env);
|
|
4335
4925
|
}
|
|
4336
4926
|
|
|
4337
4927
|
// src/lib/opencode/session-db-provenance.ts
|
|
4338
4928
|
import { createRequire } from "node:module";
|
|
4339
4929
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4340
|
-
import { dirname as
|
|
4930
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4341
4931
|
var require2 = createRequire(import.meta.url);
|
|
4342
4932
|
function readSessionDbMigrationIds(dbPath) {
|
|
4343
4933
|
let db;
|
|
@@ -4403,7 +4993,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4403
4993
|
}
|
|
4404
4994
|
function saveSessionDbProvenanceState(path, state) {
|
|
4405
4995
|
try {
|
|
4406
|
-
mkdirSync2(
|
|
4996
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4407
4997
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4408
4998
|
`, "utf8");
|
|
4409
4999
|
} catch (error2) {
|
|
@@ -4984,7 +5574,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4984
5574
|
toDelete.push(s.id);
|
|
4985
5575
|
}
|
|
4986
5576
|
}
|
|
4987
|
-
|
|
5577
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5578
|
+
return toDelete.sort(
|
|
5579
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5580
|
+
);
|
|
4988
5581
|
}
|
|
4989
5582
|
var DEFAULT_INTERVAL = "1h";
|
|
4990
5583
|
function resolve(flag, envValue, fallback) {
|
|
@@ -5044,13 +5637,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5044
5637
|
}
|
|
5045
5638
|
|
|
5046
5639
|
// src/lib/opencode/session-db-size.ts
|
|
5047
|
-
import { statSync as
|
|
5640
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5048
5641
|
import { join as join6 } from "node:path";
|
|
5049
5642
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5050
5643
|
function statSessionDbBytes(homeDir) {
|
|
5051
5644
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5052
5645
|
try {
|
|
5053
|
-
return
|
|
5646
|
+
return statSync4(dbPath).size;
|
|
5054
5647
|
} catch (err) {
|
|
5055
5648
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5056
5649
|
if (!isMissingFile) {
|
|
@@ -5076,7 +5669,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5076
5669
|
}
|
|
5077
5670
|
|
|
5078
5671
|
// src/lib/opencode/log-tail.ts
|
|
5079
|
-
import { statSync as
|
|
5672
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5080
5673
|
import { homedir as homedir5 } from "node:os";
|
|
5081
5674
|
import { join as join7 } from "node:path";
|
|
5082
5675
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -5096,189 +5689,66 @@ function reportFailure(operation, logPath, error2) {
|
|
|
5096
5689
|
function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
5097
5690
|
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
5098
5691
|
let offset = 0;
|
|
5099
|
-
let inode = null;
|
|
5100
|
-
let baselineReady = true;
|
|
5101
|
-
try {
|
|
5102
|
-
const initial =
|
|
5103
|
-
offset = initial.size;
|
|
5104
|
-
inode = initial.ino;
|
|
5105
|
-
} catch (error2) {
|
|
5106
|
-
if (!isEnoent(error2)) {
|
|
5107
|
-
reportFailure("initial stat", logPath, error2);
|
|
5108
|
-
baselineReady = false;
|
|
5109
|
-
}
|
|
5110
|
-
}
|
|
5111
|
-
let polling = false;
|
|
5112
|
-
let stopped = false;
|
|
5113
|
-
const poll = async () => {
|
|
5114
|
-
if (polling || stopped) return;
|
|
5115
|
-
polling = true;
|
|
5116
|
-
try {
|
|
5117
|
-
let current;
|
|
5118
|
-
try {
|
|
5119
|
-
current = await stat(logPath);
|
|
5120
|
-
} catch (error2) {
|
|
5121
|
-
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5122
|
-
return;
|
|
5123
|
-
}
|
|
5124
|
-
if (!baselineReady) {
|
|
5125
|
-
offset = current.size;
|
|
5126
|
-
inode = current.ino;
|
|
5127
|
-
baselineReady = true;
|
|
5128
|
-
return;
|
|
5129
|
-
}
|
|
5130
|
-
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5131
|
-
offset = 0;
|
|
5132
|
-
}
|
|
5133
|
-
inode = current.ino;
|
|
5134
|
-
if (current.size === offset) return;
|
|
5135
|
-
const length = current.size - offset;
|
|
5136
|
-
const fh = await open2(logPath, "r");
|
|
5137
|
-
try {
|
|
5138
|
-
const buf = Buffer.alloc(length);
|
|
5139
|
-
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5140
|
-
offset += bytesRead;
|
|
5141
|
-
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5142
|
-
} finally {
|
|
5143
|
-
await fh.close();
|
|
5144
|
-
}
|
|
5145
|
-
} catch (error2) {
|
|
5146
|
-
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5147
|
-
} finally {
|
|
5148
|
-
polling = false;
|
|
5149
|
-
}
|
|
5150
|
-
};
|
|
5151
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5152
|
-
void poll();
|
|
5153
|
-
return {
|
|
5154
|
-
stop: () => {
|
|
5155
|
-
stopped = true;
|
|
5156
|
-
clearInterval(interval);
|
|
5157
|
-
}
|
|
5158
|
-
};
|
|
5159
|
-
}
|
|
5160
|
-
|
|
5161
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5162
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5163
|
-
import { dirname as dirname4 } from "node:path";
|
|
5164
|
-
function errorMessage(error2) {
|
|
5165
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5166
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5167
|
-
}
|
|
5168
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5169
|
-
try {
|
|
5170
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5171
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5172
|
-
if (availableBytes < requiredBytes) {
|
|
5173
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5174
|
-
}
|
|
5175
|
-
return null;
|
|
5176
|
-
} catch (err) {
|
|
5177
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5178
|
-
}
|
|
5179
|
-
}
|
|
5180
|
-
function readLogicalBytes(db) {
|
|
5181
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5182
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5183
|
-
return pageCount * pageSize;
|
|
5184
|
-
}
|
|
5185
|
-
function readCheckpointResult(db) {
|
|
5186
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5187
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5188
|
-
}
|
|
5189
|
-
async function probeReclaimAvailability(input) {
|
|
5190
|
-
const { dbPath, requiredBytes } = input;
|
|
5191
|
-
let sqlite;
|
|
5192
|
-
try {
|
|
5193
|
-
sqlite = await import("node:sqlite");
|
|
5194
|
-
} catch (err) {
|
|
5195
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5196
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5197
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5198
|
-
}
|
|
5199
|
-
let autoVacuum = null;
|
|
5200
|
-
try {
|
|
5201
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5202
|
-
try {
|
|
5203
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5204
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5205
|
-
} finally {
|
|
5206
|
-
db.close();
|
|
5207
|
-
}
|
|
5208
|
-
} catch (err) {
|
|
5209
|
-
console.warn(
|
|
5210
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5211
|
-
);
|
|
5212
|
-
}
|
|
5213
|
-
if (autoVacuum !== 0) return null;
|
|
5214
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5215
|
-
}
|
|
5216
|
-
async function reclaimSessionDbSpace(input) {
|
|
5217
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5218
|
-
let sqlite;
|
|
5219
|
-
try {
|
|
5220
|
-
sqlite = await import("node:sqlite");
|
|
5221
|
-
} catch (err) {
|
|
5222
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5223
|
-
console.warn(
|
|
5224
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5225
|
-
);
|
|
5226
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5227
|
-
}
|
|
5228
|
-
const { DatabaseSync } = sqlite;
|
|
5229
|
-
let db;
|
|
5230
|
-
try {
|
|
5231
|
-
db = new DatabaseSync(dbPath);
|
|
5232
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5233
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5234
|
-
if (autoVacuum === 0) {
|
|
5235
|
-
if (!allowFullVacuum) {
|
|
5236
|
-
console.warn(
|
|
5237
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5238
|
-
);
|
|
5239
|
-
return {
|
|
5240
|
-
ok: false,
|
|
5241
|
-
skipped: "full-vacuum-blocked",
|
|
5242
|
-
detail: "a session turn is live"
|
|
5243
|
-
};
|
|
5692
|
+
let inode = null;
|
|
5693
|
+
let baselineReady = true;
|
|
5694
|
+
try {
|
|
5695
|
+
const initial = statSync5(logPath);
|
|
5696
|
+
offset = initial.size;
|
|
5697
|
+
inode = initial.ino;
|
|
5698
|
+
} catch (error2) {
|
|
5699
|
+
if (!isEnoent(error2)) {
|
|
5700
|
+
reportFailure("initial stat", logPath, error2);
|
|
5701
|
+
baselineReady = false;
|
|
5702
|
+
}
|
|
5703
|
+
}
|
|
5704
|
+
let polling = false;
|
|
5705
|
+
let stopped = false;
|
|
5706
|
+
const poll = async () => {
|
|
5707
|
+
if (polling || stopped) return;
|
|
5708
|
+
polling = true;
|
|
5709
|
+
try {
|
|
5710
|
+
let current;
|
|
5711
|
+
try {
|
|
5712
|
+
current = await stat(logPath);
|
|
5713
|
+
} catch (error2) {
|
|
5714
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5715
|
+
return;
|
|
5244
5716
|
}
|
|
5245
|
-
|
|
5246
|
-
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
);
|
|
5251
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5717
|
+
if (!baselineReady) {
|
|
5718
|
+
offset = current.size;
|
|
5719
|
+
inode = current.ino;
|
|
5720
|
+
baselineReady = true;
|
|
5721
|
+
return;
|
|
5252
5722
|
}
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5723
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5724
|
+
offset = 0;
|
|
5725
|
+
}
|
|
5726
|
+
inode = current.ino;
|
|
5727
|
+
if (current.size === offset) return;
|
|
5728
|
+
const length = current.size - offset;
|
|
5729
|
+
const fh = await open2(logPath, "r");
|
|
5730
|
+
try {
|
|
5731
|
+
const buf = Buffer.alloc(length);
|
|
5732
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5733
|
+
offset += bytesRead;
|
|
5734
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5735
|
+
} finally {
|
|
5736
|
+
await fh.close();
|
|
5737
|
+
}
|
|
5738
|
+
} catch (error2) {
|
|
5739
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5740
|
+
} finally {
|
|
5741
|
+
polling = false;
|
|
5259
5742
|
}
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5743
|
+
};
|
|
5744
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5745
|
+
void poll();
|
|
5746
|
+
return {
|
|
5747
|
+
stop: () => {
|
|
5748
|
+
stopped = true;
|
|
5749
|
+
clearInterval(interval);
|
|
5267
5750
|
}
|
|
5268
|
-
|
|
5269
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5270
|
-
);
|
|
5271
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5272
|
-
} catch (err) {
|
|
5273
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5274
|
-
return {
|
|
5275
|
-
ok: false,
|
|
5276
|
-
skipped: "reclaim-error",
|
|
5277
|
-
detail: errorMessage(err)
|
|
5278
|
-
};
|
|
5279
|
-
} finally {
|
|
5280
|
-
db?.close();
|
|
5281
|
-
}
|
|
5751
|
+
};
|
|
5282
5752
|
}
|
|
5283
5753
|
|
|
5284
5754
|
// src/lib/tunnel/connection.ts
|
|
@@ -6170,6 +6640,101 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6170
6640
|
};
|
|
6171
6641
|
}
|
|
6172
6642
|
|
|
6643
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6644
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6645
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6646
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6647
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6648
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6649
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6650
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6651
|
+
var BAND_RANK = {
|
|
6652
|
+
low: 1,
|
|
6653
|
+
very_low: 2,
|
|
6654
|
+
critical: 3
|
|
6655
|
+
};
|
|
6656
|
+
function bandFor(freeBytes) {
|
|
6657
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6658
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6659
|
+
return "low";
|
|
6660
|
+
}
|
|
6661
|
+
function formatBytes(bytes) {
|
|
6662
|
+
if (bytes === null) return "unavailable";
|
|
6663
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6664
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6665
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6666
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6667
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6668
|
+
}
|
|
6669
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6670
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6671
|
+
}
|
|
6672
|
+
function evaluateLowDiskWarning(input) {
|
|
6673
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6674
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6675
|
+
return { warning: null, state };
|
|
6676
|
+
}
|
|
6677
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6678
|
+
if (recovered) {
|
|
6679
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6680
|
+
}
|
|
6681
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6682
|
+
if (!low) return { warning: null, state };
|
|
6683
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6684
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6685
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6686
|
+
const worse = currentRank > previousRank;
|
|
6687
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6688
|
+
const nextState = {
|
|
6689
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6690
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6691
|
+
};
|
|
6692
|
+
return {
|
|
6693
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6694
|
+
state: nextState
|
|
6695
|
+
};
|
|
6696
|
+
}
|
|
6697
|
+
|
|
6698
|
+
// src/lib/opencode/session-store-health.ts
|
|
6699
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6700
|
+
function createSessionStoreHealth(options = {}) {
|
|
6701
|
+
const now = options.now ?? (() => Date.now());
|
|
6702
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6703
|
+
let lastUnwritableAt = null;
|
|
6704
|
+
let lastWritableAt = null;
|
|
6705
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6706
|
+
return {
|
|
6707
|
+
observeUnwritable: () => {
|
|
6708
|
+
lastUnwritableAt = now();
|
|
6709
|
+
},
|
|
6710
|
+
observeWritable: () => {
|
|
6711
|
+
lastWritableAt = now();
|
|
6712
|
+
},
|
|
6713
|
+
isUnwritable: () => {
|
|
6714
|
+
if (lastUnwritableAt === null) return false;
|
|
6715
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6716
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6717
|
+
},
|
|
6718
|
+
observeCause: (sessionId, cause) => {
|
|
6719
|
+
if (sessionId === null) return;
|
|
6720
|
+
causes.delete(sessionId);
|
|
6721
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6722
|
+
while (causes.size > 256) {
|
|
6723
|
+
const oldest = causes.keys().next().value;
|
|
6724
|
+
if (typeof oldest !== "string") break;
|
|
6725
|
+
causes.delete(oldest);
|
|
6726
|
+
}
|
|
6727
|
+
},
|
|
6728
|
+
readCause: (sessionId) => {
|
|
6729
|
+
const observed = causes.get(sessionId);
|
|
6730
|
+
if (!observed) return null;
|
|
6731
|
+
causes.delete(sessionId);
|
|
6732
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6733
|
+
return observed.cause;
|
|
6734
|
+
}
|
|
6735
|
+
};
|
|
6736
|
+
}
|
|
6737
|
+
|
|
6173
6738
|
// src/lib/channels/driver.ts
|
|
6174
6739
|
import { homedir as homedir6 } from "node:os";
|
|
6175
6740
|
|
|
@@ -6606,6 +7171,11 @@ function messageIdOf(m) {
|
|
|
6606
7171
|
const infoId = m.info?.id;
|
|
6607
7172
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6608
7173
|
}
|
|
7174
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7175
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7176
|
+
if (replyShape === null) return base;
|
|
7177
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7178
|
+
}
|
|
6609
7179
|
function cleanImageMime(contentType) {
|
|
6610
7180
|
if (!contentType) return null;
|
|
6611
7181
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6626,8 +7196,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6626
7196
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6627
7197
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6628
7198
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7199
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7200
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7201
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7202
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6629
7203
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6630
7204
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7205
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6631
7206
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6632
7207
|
var HEARTBEAT_MS = 6e4;
|
|
6633
7208
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6689,6 +7264,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6689
7264
|
maxActiveSessions;
|
|
6690
7265
|
watcherStallMs;
|
|
6691
7266
|
wedgeWarningIntervalMs;
|
|
7267
|
+
sessionStoreHealth;
|
|
6692
7268
|
openCodeClient;
|
|
6693
7269
|
/** Cache of conversationId → opencode sessionId. */
|
|
6694
7270
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6781,6 +7357,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6781
7357
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6782
7358
|
*/
|
|
6783
7359
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7360
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6784
7361
|
/**
|
|
6785
7362
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6786
7363
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6813,10 +7390,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6813
7390
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6814
7391
|
*/
|
|
6815
7392
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7393
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7394
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6816
7395
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6817
7396
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6818
7397
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6819
7398
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7399
|
+
/**
|
|
7400
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7401
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7402
|
+
* identical-text turn, so it never produces a correlation.
|
|
7403
|
+
*/
|
|
7404
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6820
7405
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6821
7406
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6822
7407
|
/**
|
|
@@ -6902,21 +7487,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6902
7487
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6903
7488
|
*/
|
|
6904
7489
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6905
|
-
/**
|
|
6906
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6907
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6908
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6909
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6910
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6911
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6912
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6913
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6914
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6915
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6916
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6917
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6918
|
-
*/
|
|
6919
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6920
7490
|
/**
|
|
6921
7491
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6922
7492
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6934,8 +7504,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6934
7504
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6935
7505
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6936
7506
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6937
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6938
|
-
*
|
|
7507
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7508
|
+
* the terminal-report fence and is never sent again).
|
|
6939
7509
|
*/
|
|
6940
7510
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6941
7511
|
/**
|
|
@@ -7058,6 +7628,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7058
7628
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7059
7629
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7060
7630
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7631
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7061
7632
|
}
|
|
7062
7633
|
get isV2() {
|
|
7063
7634
|
return this.openCodeClient.version === "v2";
|
|
@@ -7073,7 +7644,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7073
7644
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7074
7645
|
}
|
|
7075
7646
|
async listSessions() {
|
|
7076
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7647
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7648
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7649
|
+
});
|
|
7077
7650
|
}
|
|
7078
7651
|
async sessionExists(sessionId) {
|
|
7079
7652
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7250,6 +7823,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7250
7823
|
}
|
|
7251
7824
|
return ids;
|
|
7252
7825
|
}
|
|
7826
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7827
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7828
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7829
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7830
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7831
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7832
|
+
}
|
|
7833
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7834
|
+
}
|
|
7835
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7836
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7837
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7838
|
+
this.awaitingReadopt.delete(messageId);
|
|
7839
|
+
this.sessions.delete(conversationId);
|
|
7840
|
+
this.supersede(conversationId, sessionId);
|
|
7841
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7842
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7843
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7844
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7845
|
+
this.log({
|
|
7846
|
+
level: "error",
|
|
7847
|
+
message: errorMessage3,
|
|
7848
|
+
conversation_id: conversationId,
|
|
7849
|
+
message_id: messageId
|
|
7850
|
+
});
|
|
7851
|
+
const reportError = await this.reportUntrackableAck(
|
|
7852
|
+
conversationId,
|
|
7853
|
+
messageId,
|
|
7854
|
+
null,
|
|
7855
|
+
errorMessage3
|
|
7856
|
+
);
|
|
7857
|
+
if (reportError !== null) {
|
|
7858
|
+
this.log({
|
|
7859
|
+
level: "warn",
|
|
7860
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7861
|
+
conversation_id: conversationId,
|
|
7862
|
+
message_id: messageId
|
|
7863
|
+
});
|
|
7864
|
+
}
|
|
7865
|
+
if (source === "readopt") {
|
|
7866
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7867
|
+
}
|
|
7868
|
+
}
|
|
7253
7869
|
/**
|
|
7254
7870
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7255
7871
|
*
|
|
@@ -7429,6 +8045,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7429
8045
|
}
|
|
7430
8046
|
break;
|
|
7431
8047
|
}
|
|
8048
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8049
|
+
if (correlation) {
|
|
8050
|
+
correlation.attempts += 1;
|
|
8051
|
+
let correlatedId = null;
|
|
8052
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8053
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8054
|
+
conv,
|
|
8055
|
+
message,
|
|
8056
|
+
correlation.sessionId
|
|
8057
|
+
);
|
|
8058
|
+
if (polled.ok)
|
|
8059
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8060
|
+
} else if (correlation.knownUserIds === null) {
|
|
8061
|
+
this.log({
|
|
8062
|
+
level: "debug",
|
|
8063
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8064
|
+
conversation_id: conv.id,
|
|
8065
|
+
message_id: message.id
|
|
8066
|
+
});
|
|
8067
|
+
}
|
|
8068
|
+
if (correlatedId !== null) {
|
|
8069
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8070
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8071
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8072
|
+
this.dispatched.add(message.id);
|
|
8073
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8074
|
+
dispatched += 1;
|
|
8075
|
+
this.log({
|
|
8076
|
+
level: "info",
|
|
8077
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8078
|
+
conversation_id: conv.id,
|
|
8079
|
+
message_id: message.id
|
|
8080
|
+
});
|
|
8081
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8082
|
+
opencode_message_id: correlatedId
|
|
8083
|
+
});
|
|
8084
|
+
continue;
|
|
8085
|
+
}
|
|
8086
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8087
|
+
await this.handoffUntrackableCorrelation(
|
|
8088
|
+
conv.id,
|
|
8089
|
+
message.id,
|
|
8090
|
+
correlation.sessionId,
|
|
8091
|
+
correlation.attempts,
|
|
8092
|
+
"fresh"
|
|
8093
|
+
);
|
|
8094
|
+
break;
|
|
8095
|
+
}
|
|
8096
|
+
break;
|
|
8097
|
+
}
|
|
7432
8098
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7433
8099
|
if (effectiveOpencodeMessageId) {
|
|
7434
8100
|
const outcome = await this.resolveRedrive(
|
|
@@ -7450,6 +8116,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7450
8116
|
model: message.opencode_model ?? void 0
|
|
7451
8117
|
};
|
|
7452
8118
|
let opencodeMessageId;
|
|
8119
|
+
let preDispatchUserIds = null;
|
|
7453
8120
|
try {
|
|
7454
8121
|
this.log({
|
|
7455
8122
|
level: "info",
|
|
@@ -7479,7 +8146,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7479
8146
|
message.content,
|
|
7480
8147
|
options,
|
|
7481
8148
|
sendAttachments,
|
|
7482
|
-
this.openCodeClient
|
|
8149
|
+
this.openCodeClient,
|
|
8150
|
+
{
|
|
8151
|
+
onKnownUserIds: (ids) => {
|
|
8152
|
+
preDispatchUserIds = ids;
|
|
8153
|
+
},
|
|
8154
|
+
onKnownUserIdsError: (error2) => {
|
|
8155
|
+
this.log({
|
|
8156
|
+
level: "warn",
|
|
8157
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8158
|
+
conversation_id: conv.id,
|
|
8159
|
+
message_id: message.id
|
|
8160
|
+
});
|
|
8161
|
+
}
|
|
8162
|
+
}
|
|
7483
8163
|
)
|
|
7484
8164
|
);
|
|
7485
8165
|
} catch (err) {
|
|
@@ -7487,6 +8167,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7487
8167
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7488
8168
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7489
8169
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8170
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7490
8171
|
this.log({
|
|
7491
8172
|
level: "error",
|
|
7492
8173
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7562,39 +8243,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7562
8243
|
if (this.isV2) {
|
|
7563
8244
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7564
8245
|
}
|
|
7565
|
-
|
|
7566
|
-
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
7570
|
-
|
|
7571
|
-
message_id: message.id
|
|
7572
|
-
});
|
|
7573
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7574
|
-
continue;
|
|
7575
|
-
}
|
|
7576
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7577
|
-
this.sessions.delete(conv.id);
|
|
7578
|
-
this.supersede(conv.id, sessionId);
|
|
7579
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8246
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8247
|
+
sessionId,
|
|
8248
|
+
content: message.content,
|
|
8249
|
+
attempts: 0,
|
|
8250
|
+
knownUserIds: preDispatchUserIds
|
|
8251
|
+
});
|
|
7580
8252
|
this.log({
|
|
7581
|
-
level: "
|
|
7582
|
-
message:
|
|
8253
|
+
level: "warn",
|
|
8254
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7583
8255
|
conversation_id: conv.id,
|
|
7584
8256
|
message_id: message.id
|
|
7585
8257
|
});
|
|
7586
|
-
|
|
7587
|
-
this.log({
|
|
7588
|
-
level: "warn",
|
|
7589
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7590
|
-
conversation_id: conv.id,
|
|
7591
|
-
message_id: message.id
|
|
7592
|
-
});
|
|
7593
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7594
|
-
});
|
|
8258
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7595
8259
|
break;
|
|
7596
8260
|
}
|
|
7597
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7598
8261
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7599
8262
|
this.releasedOpencodeIds.delete(message.id);
|
|
7600
8263
|
this.dispatched.add(message.id);
|
|
@@ -8199,23 +8862,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8199
8862
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8200
8863
|
return 1;
|
|
8201
8864
|
}
|
|
8202
|
-
/**
|
|
8203
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8204
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8205
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8206
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8207
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8208
|
-
* count, since a new session is a genuinely different attempt.
|
|
8209
|
-
*/
|
|
8210
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8211
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8212
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8213
|
-
existing.count += 1;
|
|
8214
|
-
return existing.count;
|
|
8215
|
-
}
|
|
8216
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8217
|
-
return 1;
|
|
8218
|
-
}
|
|
8219
8865
|
/** Record one queued give-up toward the never-started bound. */
|
|
8220
8866
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8221
8867
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8527,8 +9173,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8527
9173
|
b2LastDescendantCheckMs: 0,
|
|
8528
9174
|
b2RootOngoingHeldLogged: false,
|
|
8529
9175
|
b2AbandonedSignalled: false,
|
|
9176
|
+
b2AbandonedWithoutAnswer: false,
|
|
8530
9177
|
ambiguousPinnedSinceMs: 0,
|
|
8531
9178
|
ambiguousResolved: false,
|
|
9179
|
+
sessionStoreFailurePending: false,
|
|
8532
9180
|
neverStartedResolved: false
|
|
8533
9181
|
});
|
|
8534
9182
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8550,6 +9198,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8550
9198
|
this.bufferedSessionErrors.delete(oldest);
|
|
8551
9199
|
}
|
|
8552
9200
|
}
|
|
9201
|
+
retainSessionError(event) {
|
|
9202
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9203
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9204
|
+
reason: event.reason,
|
|
9205
|
+
observedAt: this.now()
|
|
9206
|
+
});
|
|
9207
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9208
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9209
|
+
if (typeof oldest !== "string") break;
|
|
9210
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9211
|
+
}
|
|
9212
|
+
}
|
|
9213
|
+
readRetainedSessionError(sessionId) {
|
|
9214
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9215
|
+
if (!retained) return null;
|
|
9216
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9217
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9218
|
+
return null;
|
|
9219
|
+
}
|
|
9220
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9221
|
+
return retained.reason;
|
|
9222
|
+
}
|
|
8553
9223
|
/**
|
|
8554
9224
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8555
9225
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8645,8 +9315,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8645
9315
|
b2LastDescendantCheckMs: 0,
|
|
8646
9316
|
b2RootOngoingHeldLogged: false,
|
|
8647
9317
|
b2AbandonedSignalled: false,
|
|
9318
|
+
b2AbandonedWithoutAnswer: false,
|
|
8648
9319
|
ambiguousPinnedSinceMs: 0,
|
|
8649
9320
|
ambiguousResolved: false,
|
|
9321
|
+
sessionStoreFailurePending: false,
|
|
8650
9322
|
neverStartedResolved: false
|
|
8651
9323
|
});
|
|
8652
9324
|
}
|
|
@@ -8874,9 +9546,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8874
9546
|
return;
|
|
8875
9547
|
}
|
|
8876
9548
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9549
|
+
this.retainSessionError(event);
|
|
8877
9550
|
this.log({
|
|
8878
|
-
level: "
|
|
8879
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9551
|
+
level: "warn",
|
|
9552
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8880
9553
|
conversation_id: watcher.conv.id
|
|
8881
9554
|
});
|
|
8882
9555
|
return;
|
|
@@ -8906,9 +9579,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8906
9579
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8907
9580
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8908
9581
|
if (state !== "queued") {
|
|
9582
|
+
this.retainSessionError(event);
|
|
8909
9583
|
this.log({
|
|
8910
|
-
level: "
|
|
8911
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9584
|
+
level: "warn",
|
|
9585
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8912
9586
|
conversation_id: watcher.conv.id,
|
|
8913
9587
|
message_id: inFlight.evidentMessageId
|
|
8914
9588
|
});
|
|
@@ -9060,6 +9734,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9060
9734
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9061
9735
|
this.neverStartedAttempts.delete(id);
|
|
9062
9736
|
}
|
|
9737
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9738
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9739
|
+
return;
|
|
9740
|
+
}
|
|
9063
9741
|
if (inFlight.neverStartedResolved) {
|
|
9064
9742
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9065
9743
|
inFlight.neverStartedResolved = false;
|
|
@@ -9133,7 +9811,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9133
9811
|
if (state === "failed") {
|
|
9134
9812
|
this.anchorDeliveryDeadline(inFlight);
|
|
9135
9813
|
if (!inFlight.done) {
|
|
9136
|
-
const
|
|
9814
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9815
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9816
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9817
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9818
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9819
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9820
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9821
|
+
(detail) => detail !== null
|
|
9822
|
+
);
|
|
9823
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9137
9824
|
this.log({
|
|
9138
9825
|
level: "error",
|
|
9139
9826
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9219,10 +9906,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9219
9906
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9220
9907
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9221
9908
|
inFlight.b2AbandonedSignalled = false;
|
|
9909
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9222
9910
|
}
|
|
9223
9911
|
} else {
|
|
9224
9912
|
if (inFlight.b2AbandonedSignalled) {
|
|
9225
|
-
|
|
9913
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9914
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9915
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9916
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9917
|
+
sessionId,
|
|
9918
|
+
watcher,
|
|
9919
|
+
inFlight,
|
|
9920
|
+
messages,
|
|
9921
|
+
finish
|
|
9922
|
+
);
|
|
9923
|
+
} else {
|
|
9924
|
+
await this.settleMessageDone(
|
|
9925
|
+
sessionId,
|
|
9926
|
+
watcher,
|
|
9927
|
+
inFlight,
|
|
9928
|
+
messages,
|
|
9929
|
+
composeAbandonmentNote(
|
|
9930
|
+
finish,
|
|
9931
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
9932
|
+
"with-answer"
|
|
9933
|
+
)
|
|
9934
|
+
);
|
|
9935
|
+
}
|
|
9226
9936
|
return;
|
|
9227
9937
|
}
|
|
9228
9938
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9239,22 +9949,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9239
9949
|
descendantOngoing,
|
|
9240
9950
|
rootOngoing
|
|
9241
9951
|
})) {
|
|
9952
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9953
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9954
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9955
|
+
watched_for_ms: pinnedForMs,
|
|
9956
|
+
run_state: "failed",
|
|
9957
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9958
|
+
});
|
|
9959
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9960
|
+
return;
|
|
9961
|
+
}
|
|
9242
9962
|
inFlight.b2AbandonedSignalled = true;
|
|
9963
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9964
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9965
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9966
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9967
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9968
|
+
watched_for_ms: pinnedForMs,
|
|
9969
|
+
run_state: "failed",
|
|
9970
|
+
finish,
|
|
9971
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9972
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9973
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9974
|
+
});
|
|
9975
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9976
|
+
sessionId,
|
|
9977
|
+
watcher,
|
|
9978
|
+
inFlight,
|
|
9979
|
+
messages,
|
|
9980
|
+
finish
|
|
9981
|
+
);
|
|
9982
|
+
return;
|
|
9983
|
+
}
|
|
9243
9984
|
this.log({
|
|
9244
9985
|
level: "warn",
|
|
9245
9986
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9246
9987
|
conversation_id: conv.id,
|
|
9247
9988
|
message_id: id
|
|
9248
9989
|
});
|
|
9249
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9250
9990
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9251
9991
|
watched_for_ms: pinnedForMs,
|
|
9252
|
-
finish
|
|
9992
|
+
finish,
|
|
9253
9993
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9254
9994
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9255
9995
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9256
9996
|
});
|
|
9257
|
-
await this.settleMessageDone(
|
|
9997
|
+
await this.settleMessageDone(
|
|
9998
|
+
sessionId,
|
|
9999
|
+
watcher,
|
|
10000
|
+
inFlight,
|
|
10001
|
+
messages,
|
|
10002
|
+
composeAbandonmentNote(
|
|
10003
|
+
finish,
|
|
10004
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10005
|
+
"with-answer"
|
|
10006
|
+
)
|
|
10007
|
+
);
|
|
9258
10008
|
return;
|
|
9259
10009
|
}
|
|
9260
10010
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9297,6 +10047,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9297
10047
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9298
10048
|
sessionOngoing: ongoing
|
|
9299
10049
|
})) {
|
|
10050
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10051
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10052
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10053
|
+
watched_for_ms: pinnedForMs,
|
|
10054
|
+
run_state: "failed",
|
|
10055
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10056
|
+
});
|
|
10057
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10058
|
+
return;
|
|
10059
|
+
}
|
|
9300
10060
|
inFlight.ambiguousResolved = true;
|
|
9301
10061
|
this.log({
|
|
9302
10062
|
level: "warn",
|
|
@@ -9444,15 +10204,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9444
10204
|
}
|
|
9445
10205
|
}
|
|
9446
10206
|
/**
|
|
9447
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9448
|
-
*
|
|
9449
|
-
*
|
|
9450
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9451
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9452
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9453
|
-
* and risking the two copies silently drifting apart.
|
|
10207
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10208
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10209
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9454
10210
|
*/
|
|
9455
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10211
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9456
10212
|
const conv = watcher.conv;
|
|
9457
10213
|
this.anchorDeliveryDeadline(inFlight);
|
|
9458
10214
|
if (!inFlight.done) {
|
|
@@ -9479,7 +10235,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9479
10235
|
title,
|
|
9480
10236
|
usage,
|
|
9481
10237
|
usageAgentName,
|
|
9482
|
-
subagentInvocations
|
|
10238
|
+
subagentInvocations,
|
|
10239
|
+
note
|
|
9483
10240
|
);
|
|
9484
10241
|
} catch (err) {
|
|
9485
10242
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9505,20 +10262,92 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9505
10262
|
}
|
|
9506
10263
|
this.log({
|
|
9507
10264
|
level: "warn",
|
|
9508
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10265
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10266
|
+
conversation_id: conv.id,
|
|
10267
|
+
message_id: inFlight.evidentMessageId
|
|
10268
|
+
});
|
|
10269
|
+
return;
|
|
10270
|
+
}
|
|
10271
|
+
inFlight.done = true;
|
|
10272
|
+
await this.reportSubagentAuthFailures(
|
|
10273
|
+
watcher.conv.id,
|
|
10274
|
+
inFlight.opencodeMessageId,
|
|
10275
|
+
inFlight.evidentMessageId,
|
|
10276
|
+
messages
|
|
10277
|
+
);
|
|
10278
|
+
}
|
|
10279
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10280
|
+
}
|
|
10281
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10282
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10283
|
+
const conv = watcher.conv;
|
|
10284
|
+
const error2 = composeAbandonmentNote(
|
|
10285
|
+
finish,
|
|
10286
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10287
|
+
"without-answer"
|
|
10288
|
+
);
|
|
10289
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10290
|
+
this.log({
|
|
10291
|
+
level: "warn",
|
|
10292
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10293
|
+
conversation_id: conv.id,
|
|
10294
|
+
message_id: inFlight.evidentMessageId
|
|
10295
|
+
});
|
|
10296
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10297
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10298
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10299
|
+
messages,
|
|
10300
|
+
inFlight.opencodeMessageId,
|
|
10301
|
+
inFlight.evidentMessageId
|
|
10302
|
+
);
|
|
10303
|
+
try {
|
|
10304
|
+
await this.markFailed(
|
|
10305
|
+
conv.id,
|
|
10306
|
+
inFlight.evidentMessageId,
|
|
10307
|
+
sessionId,
|
|
10308
|
+
error2,
|
|
10309
|
+
usage,
|
|
10310
|
+
void 0,
|
|
10311
|
+
usageAgentName,
|
|
10312
|
+
subagentInvocations
|
|
10313
|
+
);
|
|
10314
|
+
} catch (err) {
|
|
10315
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10316
|
+
if (err instanceof ChannelTerminalError) {
|
|
10317
|
+
this.log({
|
|
10318
|
+
level: "warn",
|
|
10319
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9509
10320
|
conversation_id: conv.id,
|
|
9510
10321
|
message_id: inFlight.evidentMessageId
|
|
9511
10322
|
});
|
|
10323
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9512
10324
|
return;
|
|
9513
10325
|
}
|
|
9514
|
-
inFlight.
|
|
9515
|
-
|
|
9516
|
-
|
|
9517
|
-
|
|
9518
|
-
|
|
9519
|
-
|
|
9520
|
-
|
|
10326
|
+
if (this.now() >= inFlight.deadline) {
|
|
10327
|
+
this.log({
|
|
10328
|
+
level: "warn",
|
|
10329
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10330
|
+
conversation_id: conv.id,
|
|
10331
|
+
message_id: inFlight.evidentMessageId
|
|
10332
|
+
});
|
|
10333
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10334
|
+
return;
|
|
10335
|
+
}
|
|
10336
|
+
this.log({
|
|
10337
|
+
level: "warn",
|
|
10338
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10339
|
+
conversation_id: conv.id,
|
|
10340
|
+
message_id: inFlight.evidentMessageId
|
|
10341
|
+
});
|
|
10342
|
+
return;
|
|
9521
10343
|
}
|
|
10344
|
+
inFlight.done = true;
|
|
10345
|
+
await this.reportSubagentAuthFailures(
|
|
10346
|
+
watcher.conv.id,
|
|
10347
|
+
inFlight.opencodeMessageId,
|
|
10348
|
+
inFlight.evidentMessageId,
|
|
10349
|
+
messages
|
|
10350
|
+
);
|
|
9522
10351
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9523
10352
|
}
|
|
9524
10353
|
/**
|
|
@@ -9576,6 +10405,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9576
10405
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9577
10406
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9578
10407
|
}
|
|
10408
|
+
/**
|
|
10409
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10410
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10411
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10412
|
+
*/
|
|
10413
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10414
|
+
const conv = watcher.conv;
|
|
10415
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10416
|
+
this.log({
|
|
10417
|
+
level: "error",
|
|
10418
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10419
|
+
conversation_id: conv.id,
|
|
10420
|
+
message_id: inFlight.evidentMessageId
|
|
10421
|
+
});
|
|
10422
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10423
|
+
try {
|
|
10424
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10425
|
+
} catch (err) {
|
|
10426
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10427
|
+
if (err instanceof ChannelTerminalError) {
|
|
10428
|
+
this.log({
|
|
10429
|
+
level: "warn",
|
|
10430
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10431
|
+
conversation_id: conv.id,
|
|
10432
|
+
message_id: inFlight.evidentMessageId
|
|
10433
|
+
});
|
|
10434
|
+
this.recordReleasedOpencodeId(
|
|
10435
|
+
inFlight.evidentMessageId,
|
|
10436
|
+
sessionId,
|
|
10437
|
+
inFlight.opencodeMessageId
|
|
10438
|
+
);
|
|
10439
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10440
|
+
return;
|
|
10441
|
+
}
|
|
10442
|
+
if (this.now() >= inFlight.deadline) {
|
|
10443
|
+
this.log({
|
|
10444
|
+
level: "warn",
|
|
10445
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10446
|
+
conversation_id: conv.id,
|
|
10447
|
+
message_id: inFlight.evidentMessageId
|
|
10448
|
+
});
|
|
10449
|
+
this.recordReleasedOpencodeId(
|
|
10450
|
+
inFlight.evidentMessageId,
|
|
10451
|
+
sessionId,
|
|
10452
|
+
inFlight.opencodeMessageId
|
|
10453
|
+
);
|
|
10454
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10455
|
+
return;
|
|
10456
|
+
}
|
|
10457
|
+
this.log({
|
|
10458
|
+
level: "warn",
|
|
10459
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10460
|
+
conversation_id: conv.id,
|
|
10461
|
+
message_id: inFlight.evidentMessageId
|
|
10462
|
+
});
|
|
10463
|
+
return;
|
|
10464
|
+
}
|
|
10465
|
+
inFlight.done = true;
|
|
10466
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10467
|
+
}
|
|
9579
10468
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9580
10469
|
/**
|
|
9581
10470
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9614,11 +10503,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9614
10503
|
}
|
|
9615
10504
|
}
|
|
9616
10505
|
}
|
|
9617
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10506
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9618
10507
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10508
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10509
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10510
|
+
this.awaitingCorrelation.delete(id);
|
|
10511
|
+
this.log({
|
|
10512
|
+
level: "debug",
|
|
10513
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10514
|
+
message_id: id
|
|
10515
|
+
});
|
|
10516
|
+
}
|
|
10517
|
+
}
|
|
9619
10518
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9620
10519
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9621
10520
|
this.untrackableAck.delete(id);
|
|
10521
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9622
10522
|
this.untrackableAckSignalled.delete(id);
|
|
9623
10523
|
this.untrackableAckWarned.delete(id);
|
|
9624
10524
|
this.log({
|
|
@@ -9658,7 +10558,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9658
10558
|
bySession.set(row.opencode_session_id, list);
|
|
9659
10559
|
}
|
|
9660
10560
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9661
|
-
let messages;
|
|
10561
|
+
let messages = null;
|
|
9662
10562
|
try {
|
|
9663
10563
|
if (this.isV2) {
|
|
9664
10564
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9667,9 +10567,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9667
10567
|
level: "warn",
|
|
9668
10568
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9669
10569
|
});
|
|
9670
|
-
|
|
10570
|
+
} else {
|
|
10571
|
+
messages = snapshot;
|
|
9671
10572
|
}
|
|
9672
|
-
messages = snapshot;
|
|
9673
10573
|
} else {
|
|
9674
10574
|
const polled = await pollSessionMessagesForRedrive(
|
|
9675
10575
|
this.port,
|
|
@@ -9682,15 +10582,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9682
10582
|
level: "warn",
|
|
9683
10583
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9684
10584
|
});
|
|
9685
|
-
|
|
10585
|
+
} else {
|
|
10586
|
+
messages = polled.messages;
|
|
9686
10587
|
}
|
|
9687
|
-
messages = polled.messages;
|
|
9688
10588
|
}
|
|
9689
10589
|
} catch (err) {
|
|
9690
10590
|
this.log({
|
|
9691
10591
|
level: "warn",
|
|
9692
10592
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9693
10593
|
});
|
|
10594
|
+
}
|
|
10595
|
+
if (messages === null) {
|
|
10596
|
+
for (const row of sessionRows) {
|
|
10597
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10598
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10599
|
+
}
|
|
10600
|
+
}
|
|
9694
10601
|
continue;
|
|
9695
10602
|
}
|
|
9696
10603
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9700,6 +10607,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9700
10607
|
}
|
|
9701
10608
|
}
|
|
9702
10609
|
}
|
|
10610
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10611
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10612
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10613
|
+
if (!correlation) return false;
|
|
10614
|
+
correlation.attempts += 1;
|
|
10615
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10616
|
+
if (correlatedId !== null) {
|
|
10617
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10618
|
+
const conv = this.convForRow(sessionId, row);
|
|
10619
|
+
const message = this.queuedMessageForRow(row);
|
|
10620
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10621
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10622
|
+
this.dispatched.add(row.id);
|
|
10623
|
+
this.readopted.add(row.id);
|
|
10624
|
+
this.ensureWatcherRunning(sessionId);
|
|
10625
|
+
this.log({
|
|
10626
|
+
level: "info",
|
|
10627
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10628
|
+
conversation_id: row.conversation_id,
|
|
10629
|
+
message_id: row.id
|
|
10630
|
+
});
|
|
10631
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10632
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10633
|
+
opencode_message_id: correlatedId
|
|
10634
|
+
});
|
|
10635
|
+
return true;
|
|
10636
|
+
}
|
|
10637
|
+
if (correlation.knownUserIds === null) {
|
|
10638
|
+
this.log({
|
|
10639
|
+
level: "debug",
|
|
10640
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10641
|
+
conversation_id: row.conversation_id,
|
|
10642
|
+
message_id: row.id
|
|
10643
|
+
});
|
|
10644
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10645
|
+
this.log({
|
|
10646
|
+
level: "warn",
|
|
10647
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10648
|
+
conversation_id: row.conversation_id,
|
|
10649
|
+
message_id: row.id
|
|
10650
|
+
});
|
|
10651
|
+
}
|
|
10652
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10653
|
+
await this.handoffUntrackableCorrelation(
|
|
10654
|
+
row.conversation_id,
|
|
10655
|
+
row.id,
|
|
10656
|
+
correlation.sessionId,
|
|
10657
|
+
correlation.attempts,
|
|
10658
|
+
"readopt"
|
|
10659
|
+
);
|
|
10660
|
+
}
|
|
10661
|
+
return true;
|
|
10662
|
+
}
|
|
9703
10663
|
/**
|
|
9704
10664
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9705
10665
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9707,8 +10667,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9707
10667
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9708
10668
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9709
10669
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9710
|
-
* before the restart)
|
|
9711
|
-
*
|
|
10670
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10671
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10672
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10673
|
+
* `forceReadoptRun`):
|
|
9712
10674
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9713
10675
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9714
10676
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9717,7 +10679,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9717
10679
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9718
10680
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9719
10681
|
* tracking the stored id so the reply correlates by it;
|
|
9720
|
-
* - `unknown`/null id →
|
|
10682
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10683
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9721
10684
|
*
|
|
9722
10685
|
* Only `ChannelAuthError` propagates.
|
|
9723
10686
|
*/
|
|
@@ -9731,6 +10694,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9731
10694
|
});
|
|
9732
10695
|
return;
|
|
9733
10696
|
}
|
|
10697
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9734
10698
|
const ocId = row.opencode_message_id;
|
|
9735
10699
|
const state = messageRunState(messages, ocId ?? "");
|
|
9736
10700
|
if (state === "done") {
|
|
@@ -9797,12 +10761,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9797
10761
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9798
10762
|
this.dontRedispatch.delete(row.id);
|
|
9799
10763
|
this.untrackableAck.delete(row.id);
|
|
10764
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9800
10765
|
this.untrackableAckSignalled.delete(row.id);
|
|
9801
10766
|
this.untrackableAckWarned.delete(row.id);
|
|
9802
10767
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9803
10768
|
return;
|
|
9804
10769
|
}
|
|
9805
|
-
if (this.
|
|
10770
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10771
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10772
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10773
|
+
this.untrackableAckWarned.add(row.id);
|
|
10774
|
+
this.log({
|
|
10775
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10776
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10777
|
+
conversation_id: row.conversation_id,
|
|
10778
|
+
message_id: row.id
|
|
10779
|
+
});
|
|
10780
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10781
|
+
const reportError = await this.reportUntrackableAck(
|
|
10782
|
+
row.conversation_id,
|
|
10783
|
+
row.id,
|
|
10784
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10785
|
+
errorMessage3
|
|
10786
|
+
);
|
|
10787
|
+
if (reportError !== null) {
|
|
10788
|
+
this.log({
|
|
10789
|
+
level: "warn",
|
|
10790
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10791
|
+
conversation_id: row.conversation_id,
|
|
10792
|
+
message_id: row.id
|
|
10793
|
+
});
|
|
10794
|
+
}
|
|
10795
|
+
} else {
|
|
10796
|
+
this.log({
|
|
10797
|
+
level: "debug",
|
|
10798
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10799
|
+
conversation_id: row.conversation_id,
|
|
10800
|
+
message_id: row.id
|
|
10801
|
+
});
|
|
10802
|
+
}
|
|
10803
|
+
return;
|
|
10804
|
+
}
|
|
10805
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9806
10806
|
this.log({
|
|
9807
10807
|
level: "debug",
|
|
9808
10808
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9980,6 +10980,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9980
10980
|
}
|
|
9981
10981
|
this.dontRedispatch.delete(row.id);
|
|
9982
10982
|
this.untrackableAck.delete(row.id);
|
|
10983
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9983
10984
|
this.untrackableAckSignalled.delete(row.id);
|
|
9984
10985
|
this.untrackableAckWarned.delete(row.id);
|
|
9985
10986
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9999,8 +11000,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9999
11000
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
10000
11001
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
10001
11002
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
10002
|
-
* re-entry)
|
|
10003
|
-
*
|
|
11003
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11004
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11005
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
10004
11006
|
*
|
|
10005
11007
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
10006
11008
|
* `processed_at` (Invariant 1).
|
|
@@ -10063,6 +11065,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10063
11065
|
);
|
|
10064
11066
|
}
|
|
10065
11067
|
let ocId;
|
|
11068
|
+
let preDispatchUserIds = null;
|
|
10066
11069
|
try {
|
|
10067
11070
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10068
11071
|
sessionId,
|
|
@@ -10072,7 +11075,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10072
11075
|
row.content,
|
|
10073
11076
|
options,
|
|
10074
11077
|
sendAttachments,
|
|
10075
|
-
this.openCodeClient
|
|
11078
|
+
this.openCodeClient,
|
|
11079
|
+
{
|
|
11080
|
+
onKnownUserIds: (ids) => {
|
|
11081
|
+
preDispatchUserIds = ids;
|
|
11082
|
+
},
|
|
11083
|
+
onKnownUserIdsError: (error2) => {
|
|
11084
|
+
this.log({
|
|
11085
|
+
level: "warn",
|
|
11086
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11087
|
+
conversation_id: row.conversation_id,
|
|
11088
|
+
message_id: row.id
|
|
11089
|
+
});
|
|
11090
|
+
}
|
|
11091
|
+
}
|
|
10076
11092
|
)
|
|
10077
11093
|
);
|
|
10078
11094
|
} catch (err) {
|
|
@@ -10105,6 +11121,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10105
11121
|
}
|
|
10106
11122
|
} else {
|
|
10107
11123
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11124
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10108
11125
|
}
|
|
10109
11126
|
this.log({
|
|
10110
11127
|
level: "error",
|
|
@@ -10153,39 +11170,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10153
11170
|
}
|
|
10154
11171
|
if (ocId === null) {
|
|
10155
11172
|
this.awaitingReadopt.delete(row.id);
|
|
10156
|
-
|
|
10157
|
-
|
|
10158
|
-
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
this.log({
|
|
10163
|
-
level: "error",
|
|
10164
|
-
message: errorMessage3,
|
|
10165
|
-
conversation_id: row.conversation_id,
|
|
10166
|
-
message_id: row.id
|
|
10167
|
-
});
|
|
10168
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10169
|
-
this.log({
|
|
10170
|
-
level: "warn",
|
|
10171
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10172
|
-
conversation_id: row.conversation_id,
|
|
10173
|
-
message_id: row.id
|
|
10174
|
-
});
|
|
10175
|
-
});
|
|
10176
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10177
|
-
return;
|
|
10178
|
-
}
|
|
11173
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11174
|
+
sessionId,
|
|
11175
|
+
content: row.content,
|
|
11176
|
+
attempts: 0,
|
|
11177
|
+
knownUserIds: preDispatchUserIds
|
|
11178
|
+
});
|
|
10179
11179
|
this.log({
|
|
10180
11180
|
level: "warn",
|
|
10181
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11181
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10182
11182
|
conversation_id: row.conversation_id,
|
|
10183
11183
|
message_id: row.id
|
|
10184
11184
|
});
|
|
10185
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10186
11185
|
return;
|
|
10187
11186
|
}
|
|
10188
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10189
11187
|
const processedAtMs = this.processedAtMs(row);
|
|
10190
11188
|
this.registerReadopted(
|
|
10191
11189
|
readoptConv,
|
|
@@ -11091,7 +12089,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11091
12089
|
* watcher retries next tick within the
|
|
11092
12090
|
* deadline, Finding 4).
|
|
11093
12091
|
*/
|
|
11094
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12092
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11095
12093
|
const res = await this.fetchImpl(
|
|
11096
12094
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11097
12095
|
{
|
|
@@ -11109,7 +12107,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11109
12107
|
...title ? { title } : {},
|
|
11110
12108
|
...usage ? usage : {},
|
|
11111
12109
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11112
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12110
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12111
|
+
...error2 ? { error: error2 } : {}
|
|
11113
12112
|
})
|
|
11114
12113
|
}
|
|
11115
12114
|
);
|
|
@@ -12780,9 +13779,9 @@ async function driveChannels(state, driver) {
|
|
|
12780
13779
|
}
|
|
12781
13780
|
}
|
|
12782
13781
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12783
|
-
var
|
|
13782
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12784
13783
|
function shouldWarnForReclaimSkip(reason) {
|
|
12785
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13784
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12786
13785
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12787
13786
|
if (!version2) return false;
|
|
12788
13787
|
const major = Number(version2[1]);
|
|
@@ -12839,61 +13838,96 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12839
13838
|
async function runSweep(state, driver, config) {
|
|
12840
13839
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12841
13840
|
try {
|
|
12842
|
-
const
|
|
12843
|
-
|
|
12844
|
-
|
|
13841
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13842
|
+
const sessions = enumeration.sessions;
|
|
13843
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13844
|
+
if (enumerationReason !== null) {
|
|
13845
|
+
logActivity(state, {
|
|
12845
13846
|
type: "info",
|
|
12846
|
-
|
|
12847
|
-
|
|
12848
|
-
outcome: "sweep-skipped",
|
|
12849
|
-
reason: "opencode-unreachable"
|
|
12850
|
-
}
|
|
13847
|
+
level: "warn",
|
|
13848
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12851
13849
|
});
|
|
12852
|
-
return;
|
|
12853
13850
|
}
|
|
12854
|
-
const toDelete = selectSessionsToDelete(
|
|
12855
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12856
|
-
{
|
|
12857
|
-
maxAgeMs: config.maxAgeMs,
|
|
12858
|
-
maxCount: config.maxCount,
|
|
12859
|
-
nowMs: Date.now(),
|
|
12860
|
-
protectedIds: driver.protectedSessionIds()
|
|
12861
|
-
}
|
|
12862
|
-
);
|
|
12863
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12864
13851
|
let deleted = 0;
|
|
12865
13852
|
let failed = 0;
|
|
13853
|
+
let failed4xx = 0;
|
|
13854
|
+
let failed5xx = 0;
|
|
13855
|
+
let failedOtherStatus = 0;
|
|
13856
|
+
let failedThrew = 0;
|
|
12866
13857
|
let skippedNewlyActive = 0;
|
|
12867
|
-
|
|
12868
|
-
|
|
12869
|
-
|
|
12870
|
-
|
|
12871
|
-
|
|
12872
|
-
|
|
12873
|
-
|
|
12874
|
-
|
|
13858
|
+
let remaining = 0;
|
|
13859
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13860
|
+
if (sessions !== null) {
|
|
13861
|
+
const toDelete = selectSessionsToDelete(
|
|
13862
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13863
|
+
{
|
|
13864
|
+
maxAgeMs: config.maxAgeMs,
|
|
13865
|
+
maxCount: config.maxCount,
|
|
13866
|
+
nowMs: Date.now(),
|
|
13867
|
+
protectedIds: protectedNow
|
|
13868
|
+
}
|
|
13869
|
+
);
|
|
13870
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13871
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13872
|
+
protectedNow = driver.protectedSessionIds();
|
|
13873
|
+
for (const id of deleteCandidates) {
|
|
13874
|
+
if (protectedNow.has(id)) {
|
|
13875
|
+
skippedNewlyActive++;
|
|
13876
|
+
logActivity(state, {
|
|
13877
|
+
type: "info",
|
|
13878
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13879
|
+
});
|
|
13880
|
+
continue;
|
|
13881
|
+
}
|
|
13882
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13883
|
+
if (outcome.ok) {
|
|
13884
|
+
deleted++;
|
|
13885
|
+
} else if (outcome.status === "threw") {
|
|
13886
|
+
failed++;
|
|
13887
|
+
failedThrew++;
|
|
13888
|
+
} else {
|
|
13889
|
+
failed++;
|
|
13890
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13891
|
+
if (statusClass === 4) failed4xx++;
|
|
13892
|
+
else if (statusClass === 5) failed5xx++;
|
|
13893
|
+
else failedOtherStatus++;
|
|
13894
|
+
}
|
|
12875
13895
|
}
|
|
12876
|
-
|
|
12877
|
-
|
|
13896
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13897
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13898
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13899
|
+
logActivity(state, {
|
|
13900
|
+
type: "info",
|
|
13901
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13902
|
+
});
|
|
12878
13903
|
}
|
|
12879
|
-
const
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
13904
|
+
const sessionMetadata = {
|
|
13905
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13906
|
+
sessions_deleted: deleted,
|
|
13907
|
+
sessions_failed: failed,
|
|
13908
|
+
sessions_failed_4xx: failed4xx,
|
|
13909
|
+
sessions_failed_5xx: failed5xx,
|
|
13910
|
+
sessions_failed_other: failedOtherStatus,
|
|
13911
|
+
sessions_failed_threw: failedThrew,
|
|
13912
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13913
|
+
sessions_remaining: remaining,
|
|
13914
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13915
|
+
};
|
|
12885
13916
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12886
13917
|
dbPath: sessionDbPath(),
|
|
12887
|
-
|
|
13918
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13919
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13920
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12888
13921
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12889
13922
|
});
|
|
12890
13923
|
if (reclaimResult.ok) {
|
|
13924
|
+
state.sessionStoreHealth.observeWritable();
|
|
12891
13925
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12892
13926
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12893
13927
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12894
13928
|
logSessionCleanupActivity(state, "sweep", {
|
|
12895
13929
|
type: "info",
|
|
12896
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13930
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12897
13931
|
metadata: {
|
|
12898
13932
|
outcome: "reclaimed",
|
|
12899
13933
|
mode: reclaimResult.mode,
|
|
@@ -12901,25 +13935,27 @@ async function runSweep(state, driver, config) {
|
|
|
12901
13935
|
after_bytes: reclaimResult.afterBytes,
|
|
12902
13936
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12903
13937
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
13938
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13939
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13940
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13941
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13942
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13943
|
+
...sessionMetadata
|
|
12908
13944
|
}
|
|
12909
13945
|
});
|
|
12910
13946
|
} else {
|
|
13947
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13948
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13949
|
+
}
|
|
12911
13950
|
logSessionCleanupActivity(state, "sweep", {
|
|
12912
13951
|
type: "info",
|
|
12913
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13952
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12914
13953
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12915
13954
|
metadata: {
|
|
12916
13955
|
outcome: "reclaim-skipped",
|
|
12917
13956
|
reason: reclaimResult.skipped,
|
|
12918
13957
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12919
|
-
|
|
12920
|
-
sessions_deleted: deleted,
|
|
12921
|
-
sessions_failed: failed,
|
|
12922
|
-
sessions_skipped_active: skippedNewlyActive
|
|
13958
|
+
...sessionMetadata
|
|
12923
13959
|
}
|
|
12924
13960
|
});
|
|
12925
13961
|
}
|
|
@@ -13176,6 +14212,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13176
14212
|
message: `Resource usage collection: ${warning2}`
|
|
13177
14213
|
});
|
|
13178
14214
|
}
|
|
14215
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14216
|
+
...usage,
|
|
14217
|
+
nowMs: Date.now(),
|
|
14218
|
+
state: state.lowDiskWarningState
|
|
14219
|
+
});
|
|
14220
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14221
|
+
if (lowDiskDecision.warning !== null) {
|
|
14222
|
+
logActivity(state, {
|
|
14223
|
+
type: "info",
|
|
14224
|
+
level: "warn",
|
|
14225
|
+
message: lowDiskDecision.warning
|
|
14226
|
+
});
|
|
14227
|
+
}
|
|
13179
14228
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13180
14229
|
if (result.ok) {
|
|
13181
14230
|
if (consecutiveFailures > 0) {
|
|
@@ -13422,6 +14471,7 @@ async function run(options) {
|
|
|
13422
14471
|
opencodeClient: null,
|
|
13423
14472
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13424
14473
|
sessionDbProvenanceAnomaly: false,
|
|
14474
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13425
14475
|
opencodeProcess: null,
|
|
13426
14476
|
stopOpenCodeLogTail: null,
|
|
13427
14477
|
litestreamProcess: null,
|
|
@@ -13439,6 +14489,7 @@ async function run(options) {
|
|
|
13439
14489
|
openaiUsageRearm: null,
|
|
13440
14490
|
resourceUsageTimer: null,
|
|
13441
14491
|
stopResourceUsageSampling: null,
|
|
14492
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13442
14493
|
credentialSync: null,
|
|
13443
14494
|
authHeader: ""
|
|
13444
14495
|
};
|
|
@@ -13612,6 +14663,34 @@ async function run(options) {
|
|
|
13612
14663
|
} else {
|
|
13613
14664
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13614
14665
|
}
|
|
14666
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14667
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14668
|
+
log2(state, message, level);
|
|
14669
|
+
if (state.interactive && !state.json) {
|
|
14670
|
+
logActivity(state, { type: "info", level, message });
|
|
14671
|
+
}
|
|
14672
|
+
};
|
|
14673
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14674
|
+
if (vaultResolution.error) {
|
|
14675
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14676
|
+
logCredentialVaultResolution(
|
|
14677
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14678
|
+
"warn"
|
|
14679
|
+
);
|
|
14680
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14681
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14682
|
+
logCredentialVaultResolution(
|
|
14683
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14684
|
+
"info"
|
|
14685
|
+
);
|
|
14686
|
+
} else {
|
|
14687
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14688
|
+
logCredentialVaultResolution(
|
|
14689
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14690
|
+
"debug"
|
|
14691
|
+
);
|
|
14692
|
+
}
|
|
14693
|
+
}
|
|
13615
14694
|
if (options.restoreRunnerCredentials) {
|
|
13616
14695
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13617
14696
|
const credentialContext = {
|
|
@@ -13629,10 +14708,18 @@ async function run(options) {
|
|
|
13629
14708
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13630
14709
|
}
|
|
13631
14710
|
state.credentialSync?.arm();
|
|
13632
|
-
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13633
|
-
|
|
13634
|
-
|
|
13635
|
-
|
|
14711
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14712
|
+
() => ({
|
|
14713
|
+
agentId: state.agentId,
|
|
14714
|
+
authHeader: state.authHeader
|
|
14715
|
+
}),
|
|
14716
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14717
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14718
|
+
if (cause !== null && sessionId !== null) {
|
|
14719
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14720
|
+
}
|
|
14721
|
+
}
|
|
14722
|
+
);
|
|
13636
14723
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13637
14724
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13638
14725
|
openCodeActivityForwarder
|
|
@@ -13642,22 +14729,27 @@ async function run(options) {
|
|
|
13642
14729
|
openCodeLogTail.stop();
|
|
13643
14730
|
};
|
|
13644
14731
|
let sessionDbVerifyFatal = false;
|
|
14732
|
+
let sessionDbVerifyFatalReason;
|
|
13645
14733
|
if (!options.restoreSessionDb) {
|
|
13646
|
-
|
|
14734
|
+
logActivity(state, {
|
|
14735
|
+
type: "info",
|
|
14736
|
+
level: "warn",
|
|
14737
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14738
|
+
});
|
|
13647
14739
|
} else {
|
|
13648
14740
|
const health = await checkOpenCodeHealth(state.port);
|
|
13649
14741
|
if (health.healthy) {
|
|
13650
|
-
|
|
13651
|
-
|
|
13652
|
-
|
|
13653
|
-
"
|
|
13654
|
-
);
|
|
14742
|
+
logActivity(state, {
|
|
14743
|
+
type: "info",
|
|
14744
|
+
level: "warn",
|
|
14745
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14746
|
+
});
|
|
13655
14747
|
} else if (health.authFailed) {
|
|
13656
|
-
|
|
13657
|
-
|
|
13658
|
-
|
|
13659
|
-
"
|
|
13660
|
-
);
|
|
14748
|
+
logActivity(state, {
|
|
14749
|
+
type: "info",
|
|
14750
|
+
level: "warn",
|
|
14751
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14752
|
+
});
|
|
13661
14753
|
} else {
|
|
13662
14754
|
const result = await restoreAndVerifySessionDb({
|
|
13663
14755
|
dbPath: sessionDbPath(),
|
|
@@ -13674,12 +14766,13 @@ async function run(options) {
|
|
|
13674
14766
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13675
14767
|
});
|
|
13676
14768
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14769
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13677
14770
|
}
|
|
13678
14771
|
}
|
|
13679
14772
|
reportSessionDbRecovery(state);
|
|
13680
14773
|
if (sessionDbVerifyFatal) {
|
|
13681
14774
|
throw new Error(
|
|
13682
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14775
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13683
14776
|
);
|
|
13684
14777
|
}
|
|
13685
14778
|
applyRunnerOpenCodeConfig({
|
|
@@ -13924,6 +15017,7 @@ async function run(options) {
|
|
|
13924
15017
|
getAuthHeader: () => state.authHeader,
|
|
13925
15018
|
conversationFilter: state.conversationFilter,
|
|
13926
15019
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15020
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13927
15021
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13928
15022
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13929
15023
|
fileSyncDirectories,
|
|
@@ -14005,14 +15099,14 @@ async function run(options) {
|
|
|
14005
15099
|
},
|
|
14006
15100
|
onDisconnected: (code, reason, transition) => {
|
|
14007
15101
|
state.connected = false;
|
|
14008
|
-
const
|
|
15102
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
14009
15103
|
logActivity(state, {
|
|
14010
15104
|
type: "info",
|
|
14011
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15105
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14012
15106
|
});
|
|
14013
15107
|
emitAgentDisconnected(state.agentId, {
|
|
14014
15108
|
code,
|
|
14015
|
-
reason:
|
|
15109
|
+
reason: safeReason2,
|
|
14016
15110
|
connection_generation: transition.generation
|
|
14017
15111
|
});
|
|
14018
15112
|
if (state.interactive) displayStatus(state);
|
|
@@ -14175,6 +15269,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14175
15269
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14176
15270
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14177
15271
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15272
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14178
15273
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14179
15274
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14180
15275
|
"-a, --agent [id]",
|