@evident-ai/cli 3.5.2-dev.817a271 → 3.5.2-dev.8c3e0f7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/index.js +1730 -507
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -703,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
703
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
704
704
|
});
|
|
705
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
706
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
707
732
|
const apiUrl = getApiUrlConfig();
|
|
708
733
|
try {
|
|
@@ -877,6 +902,76 @@ async function status(options = {}) {
|
|
|
877
902
|
process.exit(result.exitCode);
|
|
878
903
|
}
|
|
879
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// src/lib/claude-usage.ts
|
|
881
976
|
import { execFileSync } from "node:child_process";
|
|
882
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1069,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1069
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1070
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1071
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1072
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1073
1238
|
var TelemetryEventTypes = {
|
|
1074
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1157,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1157
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1158
1323
|
|
|
1159
1324
|
// src/lib/telemetry.ts
|
|
1160
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.8c3e0f7" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1161
1326
|
function getCliVersion() {
|
|
1162
1327
|
return CLI_VERSION;
|
|
1163
1328
|
}
|
|
@@ -1318,8 +1483,334 @@ var EventTypes = {
|
|
|
1318
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1319
1484
|
};
|
|
1320
1485
|
|
|
1321
|
-
// src/lib/
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1322
1491
|
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1522
|
+
function errorMessage(error2) {
|
|
1523
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1524
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1525
|
+
}
|
|
1526
|
+
function errorMessages(error2) {
|
|
1527
|
+
const messages = [];
|
|
1528
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1529
|
+
let current = error2;
|
|
1530
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1531
|
+
visited.add(current);
|
|
1532
|
+
if (typeof current === "string") {
|
|
1533
|
+
messages.push(current);
|
|
1534
|
+
break;
|
|
1535
|
+
}
|
|
1536
|
+
if (current instanceof Error) {
|
|
1537
|
+
messages.push(current.name, current.message);
|
|
1538
|
+
current = current.cause;
|
|
1539
|
+
continue;
|
|
1540
|
+
}
|
|
1541
|
+
if (typeof current === "object") {
|
|
1542
|
+
const record = current;
|
|
1543
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1544
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1545
|
+
current = record.cause;
|
|
1546
|
+
continue;
|
|
1547
|
+
}
|
|
1548
|
+
break;
|
|
1549
|
+
}
|
|
1550
|
+
return messages;
|
|
1551
|
+
}
|
|
1552
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1553
|
+
return errorMessages(error2).some(
|
|
1554
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1555
|
+
message
|
|
1556
|
+
)
|
|
1557
|
+
);
|
|
1558
|
+
}
|
|
1559
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1560
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1561
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1562
|
+
function isCauseBearingText(text) {
|
|
1563
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1564
|
+
}
|
|
1565
|
+
function structuredLogFields(line) {
|
|
1566
|
+
const fields = [];
|
|
1567
|
+
for (let index = 0; index < line.length; index++) {
|
|
1568
|
+
if (line[index] === '"') {
|
|
1569
|
+
index++;
|
|
1570
|
+
while (index < line.length) {
|
|
1571
|
+
if (line[index] === "\\") {
|
|
1572
|
+
index++;
|
|
1573
|
+
} else if (line[index] === '"') {
|
|
1574
|
+
break;
|
|
1575
|
+
}
|
|
1576
|
+
index++;
|
|
1577
|
+
}
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1581
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1582
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1583
|
+
const valueStart = index + key.length;
|
|
1584
|
+
if (line[valueStart] !== '"') {
|
|
1585
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1586
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1587
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
let valueEnd = valueStart + 1;
|
|
1591
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1592
|
+
if (line[valueEnd] === "\\") {
|
|
1593
|
+
valueEnd++;
|
|
1594
|
+
} else if (line[valueEnd] === '"') {
|
|
1595
|
+
break;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1599
|
+
index = valueEnd;
|
|
1600
|
+
}
|
|
1601
|
+
return fields;
|
|
1602
|
+
}
|
|
1603
|
+
function structuredCauseMessages(text) {
|
|
1604
|
+
const fields = structuredLogFields(text);
|
|
1605
|
+
if (fields.length > 0) return fields;
|
|
1606
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1607
|
+
}
|
|
1608
|
+
function causeTokenFromText(text) {
|
|
1609
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1610
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1611
|
+
const match = pattern.exec(text);
|
|
1612
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1613
|
+
return match[1];
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
return null;
|
|
1617
|
+
}
|
|
1618
|
+
function sessionStoreCauseToken(value) {
|
|
1619
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1620
|
+
const token = causeTokenFromText(message);
|
|
1621
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1622
|
+
}
|
|
1623
|
+
return null;
|
|
1624
|
+
}
|
|
1625
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1626
|
+
try {
|
|
1627
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1628
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1629
|
+
if (availableBytes < requiredBytes) {
|
|
1630
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1631
|
+
}
|
|
1632
|
+
return null;
|
|
1633
|
+
} catch (err) {
|
|
1634
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
function readLogicalBytes(db) {
|
|
1638
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1639
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1640
|
+
return pageCount * pageSize;
|
|
1641
|
+
}
|
|
1642
|
+
function readFreelistCount(db) {
|
|
1643
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1644
|
+
}
|
|
1645
|
+
function readCheckpointResult(db) {
|
|
1646
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1647
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1648
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1649
|
+
}
|
|
1650
|
+
async function probeReclaimAvailability(input) {
|
|
1651
|
+
const { dbPath, requiredBytes } = input;
|
|
1652
|
+
let sqlite;
|
|
1653
|
+
try {
|
|
1654
|
+
sqlite = await import("node:sqlite");
|
|
1655
|
+
} catch (err) {
|
|
1656
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1657
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1658
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1659
|
+
}
|
|
1660
|
+
let autoVacuum = null;
|
|
1661
|
+
try {
|
|
1662
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1663
|
+
try {
|
|
1664
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1665
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1666
|
+
} finally {
|
|
1667
|
+
db.close();
|
|
1668
|
+
}
|
|
1669
|
+
} catch (err) {
|
|
1670
|
+
console.warn(
|
|
1671
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1672
|
+
);
|
|
1673
|
+
}
|
|
1674
|
+
if (autoVacuum !== 0) return null;
|
|
1675
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1676
|
+
}
|
|
1677
|
+
async function reclaimSessionDbSpace(input) {
|
|
1678
|
+
const {
|
|
1679
|
+
dbPath,
|
|
1680
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1681
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1682
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1683
|
+
now = () => performance.now(),
|
|
1684
|
+
hasLiveTurn = false,
|
|
1685
|
+
hasProvenanceAnomaly = false
|
|
1686
|
+
} = input;
|
|
1687
|
+
let sqlite;
|
|
1688
|
+
try {
|
|
1689
|
+
sqlite = await import("node:sqlite");
|
|
1690
|
+
} catch (err) {
|
|
1691
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1692
|
+
console.warn(
|
|
1693
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1694
|
+
);
|
|
1695
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1696
|
+
}
|
|
1697
|
+
const { DatabaseSync } = sqlite;
|
|
1698
|
+
let db;
|
|
1699
|
+
try {
|
|
1700
|
+
db = new DatabaseSync(dbPath);
|
|
1701
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1702
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1703
|
+
if (autoVacuum === 0) {
|
|
1704
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1705
|
+
const detail = [
|
|
1706
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1707
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1708
|
+
].join(" and ");
|
|
1709
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1710
|
+
return {
|
|
1711
|
+
ok: false,
|
|
1712
|
+
skipped: "full-vacuum-blocked",
|
|
1713
|
+
detail
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1717
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1718
|
+
if (skipReason !== null) {
|
|
1719
|
+
console.warn(
|
|
1720
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1721
|
+
);
|
|
1722
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1723
|
+
}
|
|
1724
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistBefore = readFreelistCount(db);
|
|
1726
|
+
const startedAt = now();
|
|
1727
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1728
|
+
db.exec("VACUUM");
|
|
1729
|
+
const afterBytes = readLogicalBytes(db);
|
|
1730
|
+
const freelistAfter = readFreelistCount(db);
|
|
1731
|
+
const checkpointStartedAt = now();
|
|
1732
|
+
const checkpoint = readCheckpointResult(db);
|
|
1733
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1734
|
+
return {
|
|
1735
|
+
ok: true,
|
|
1736
|
+
mode: "convert",
|
|
1737
|
+
beforeBytes,
|
|
1738
|
+
afterBytes,
|
|
1739
|
+
freelistBefore,
|
|
1740
|
+
freelistAfter,
|
|
1741
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1742
|
+
stoppedBy: "freelist-drained",
|
|
1743
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1744
|
+
checkpointElapsedMs,
|
|
1745
|
+
checkpoint
|
|
1746
|
+
};
|
|
1747
|
+
}
|
|
1748
|
+
if (autoVacuum === 2) {
|
|
1749
|
+
if (hasLiveTurn) {
|
|
1750
|
+
const detail = "a session turn is live";
|
|
1751
|
+
console.warn(
|
|
1752
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1753
|
+
);
|
|
1754
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1755
|
+
}
|
|
1756
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1757
|
+
const freelistBefore = readFreelistCount(db);
|
|
1758
|
+
const startedAt = now();
|
|
1759
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1760
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1761
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1762
|
+
let pagesAttempted = 0;
|
|
1763
|
+
let freelistAfter = freelistBefore;
|
|
1764
|
+
let stoppedBy = "freelist-drained";
|
|
1765
|
+
while (freelistAfter > 0) {
|
|
1766
|
+
if (pagesAttempted >= pageLimit) {
|
|
1767
|
+
stoppedBy = "page-ceiling";
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
if (now() - startedAt >= budget) {
|
|
1771
|
+
stoppedBy = "time-budget";
|
|
1772
|
+
break;
|
|
1773
|
+
}
|
|
1774
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1775
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1776
|
+
pagesAttempted += pages;
|
|
1777
|
+
freelistAfter = readFreelistCount(db);
|
|
1778
|
+
}
|
|
1779
|
+
const afterBytes = readLogicalBytes(db);
|
|
1780
|
+
const checkpointStartedAt = now();
|
|
1781
|
+
const checkpoint = readCheckpointResult(db);
|
|
1782
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1783
|
+
return {
|
|
1784
|
+
ok: true,
|
|
1785
|
+
mode: "incremental",
|
|
1786
|
+
beforeBytes,
|
|
1787
|
+
afterBytes,
|
|
1788
|
+
freelistBefore,
|
|
1789
|
+
freelistAfter,
|
|
1790
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1791
|
+
stoppedBy,
|
|
1792
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1793
|
+
checkpointElapsedMs,
|
|
1794
|
+
checkpoint
|
|
1795
|
+
};
|
|
1796
|
+
}
|
|
1797
|
+
console.warn(
|
|
1798
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1799
|
+
);
|
|
1800
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1801
|
+
} catch (err) {
|
|
1802
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1803
|
+
return {
|
|
1804
|
+
ok: false,
|
|
1805
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1806
|
+
detail: errorMessage(err)
|
|
1807
|
+
};
|
|
1808
|
+
} finally {
|
|
1809
|
+
db?.close();
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1323
1814
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1324
1815
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1325
1816
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1336,11 +1827,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1336
1827
|
if (!levelMatch) return null;
|
|
1337
1828
|
const level = levelMatch[1].toUpperCase();
|
|
1338
1829
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1339
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1830
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1831
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1340
1832
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1341
1833
|
const parsed = {
|
|
1342
1834
|
level: level === "WARN" ? "warn" : "error",
|
|
1343
|
-
sessionID: sessionMatch?.[1]
|
|
1835
|
+
sessionID: sessionMatch?.[1],
|
|
1836
|
+
messageID: messageIDMatch?.[1]
|
|
1344
1837
|
};
|
|
1345
1838
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1346
1839
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1348,12 +1841,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1348
1841
|
}
|
|
1349
1842
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1350
1843
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1351
|
-
function createOpenCodeActivityForwarder(getContext) {
|
|
1844
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1352
1845
|
let buffer = Buffer.alloc(0);
|
|
1353
1846
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1354
1847
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1355
1848
|
const metadata = {
|
|
1356
1849
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1850
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1357
1851
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1358
1852
|
};
|
|
1359
1853
|
forwardRunnerActivity(
|
|
@@ -1379,9 +1873,24 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1379
1873
|
};
|
|
1380
1874
|
const flushLine = (line) => {
|
|
1381
1875
|
const parsed = parseOpenCodeLogLine(line);
|
|
1876
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1877
|
+
const cause = sessionStoreCauseToken(line);
|
|
1878
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1879
|
+
try {
|
|
1880
|
+
onSessionStoreWriteFailure({
|
|
1881
|
+
isDiskFailure,
|
|
1882
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1883
|
+
cause
|
|
1884
|
+
});
|
|
1885
|
+
} catch (err) {
|
|
1886
|
+
reportTelemetryDiagnostic(
|
|
1887
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1888
|
+
);
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1382
1891
|
if (!parsed) return;
|
|
1383
1892
|
const now = Date.now();
|
|
1384
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1893
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1385
1894
|
expireRepeatStates(now, key);
|
|
1386
1895
|
if (key === void 0) {
|
|
1387
1896
|
forwardLine(line, parsed);
|
|
@@ -1427,34 +1936,21 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1427
1936
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1428
1937
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1429
1938
|
var MAX_METADATA_ENTRIES = 20;
|
|
1430
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1431
|
-
function redactHomeDirectory(message) {
|
|
1432
|
-
const home = homedir2();
|
|
1433
|
-
if (!home || home === "/") {
|
|
1434
|
-
return message;
|
|
1435
|
-
}
|
|
1436
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1437
|
-
let result = "";
|
|
1438
|
-
let searchFrom = 0;
|
|
1439
|
-
let homeIndex;
|
|
1440
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1441
|
-
const homeEnd = homeIndex + home.length;
|
|
1442
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1443
|
-
const nextCharacter = message[homeEnd];
|
|
1444
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1445
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1446
|
-
result += atPathBoundary ? "~" : home;
|
|
1447
|
-
searchFrom = homeEnd;
|
|
1448
|
-
}
|
|
1449
|
-
return result + message.slice(searchFrom);
|
|
1450
|
-
}
|
|
1451
|
-
function redactDiagnosticText(message) {
|
|
1452
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1453
|
-
return redactHomeDirectory(redacted);
|
|
1454
|
-
}
|
|
1455
1939
|
function truncate(message) {
|
|
1456
1940
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1457
|
-
|
|
1941
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1942
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1943
|
+
let previousDropped = -1;
|
|
1944
|
+
while (dropped !== previousDropped) {
|
|
1945
|
+
previousDropped = dropped;
|
|
1946
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1947
|
+
dropped = nextDropped;
|
|
1948
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1949
|
+
}
|
|
1950
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1951
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1952
|
+
const tailLength = remaining - headLength;
|
|
1953
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1458
1954
|
}
|
|
1459
1955
|
function sanitiseMetadata(metadata) {
|
|
1460
1956
|
if (!metadata) return {};
|
|
@@ -1664,12 +2160,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1664
2160
|
metadata: withoutContractFields(record),
|
|
1665
2161
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1666
2162
|
};
|
|
1667
|
-
case "session_db_boot_refused":
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
2163
|
+
case "session_db_boot_refused": {
|
|
2164
|
+
const message = (() => {
|
|
2165
|
+
switch (record.reason) {
|
|
2166
|
+
case "local_discard_failed":
|
|
2167
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2168
|
+
case "replica_separation_unproven":
|
|
2169
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2170
|
+
case "verification_timeout_before_separation":
|
|
2171
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2172
|
+
case "verification_timeout_during_separation":
|
|
2173
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2174
|
+
default:
|
|
2175
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2176
|
+
}
|
|
2177
|
+
})();
|
|
2178
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2179
|
+
}
|
|
1673
2180
|
case "schema_provenance_mismatch":
|
|
1674
2181
|
return {
|
|
1675
2182
|
level,
|
|
@@ -1919,7 +2426,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1919
2426
|
return null;
|
|
1920
2427
|
}
|
|
1921
2428
|
}
|
|
1922
|
-
function
|
|
2429
|
+
function roleOf2(m) {
|
|
1923
2430
|
if (!m || typeof m !== "object") return void 0;
|
|
1924
2431
|
if (typeof m.role === "string") return m.role;
|
|
1925
2432
|
const infoRole = m.info?.role;
|
|
@@ -1933,13 +2440,13 @@ function createdOf(m) {
|
|
|
1933
2440
|
if (!m || typeof m !== "object") return void 0;
|
|
1934
2441
|
return m.info?.time?.created ?? m.time?.created;
|
|
1935
2442
|
}
|
|
1936
|
-
function
|
|
2443
|
+
function idOf2(m) {
|
|
1937
2444
|
if (!m || typeof m !== "object") return void 0;
|
|
1938
2445
|
if (typeof m.id === "string") return m.id;
|
|
1939
2446
|
const infoId = m.info?.id;
|
|
1940
2447
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1941
2448
|
}
|
|
1942
|
-
function
|
|
2449
|
+
function parentIdOf2(m) {
|
|
1943
2450
|
if (!m || typeof m !== "object") return void 0;
|
|
1944
2451
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1945
2452
|
const infoParent = m.info?.parentID;
|
|
@@ -1955,8 +2462,12 @@ function errorOf(m) {
|
|
|
1955
2462
|
if (!m || typeof m !== "object") return void 0;
|
|
1956
2463
|
return m.info?.error ?? m.error;
|
|
1957
2464
|
}
|
|
2465
|
+
function isTerminalStopAnswer(m) {
|
|
2466
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2467
|
+
}
|
|
1958
2468
|
function isAssistantInFlight(m) {
|
|
1959
2469
|
if (completedOf(m) == null) return true;
|
|
2470
|
+
if (errorOf(m) != null) return false;
|
|
1960
2471
|
return finishOf(m) === "tool-calls";
|
|
1961
2472
|
}
|
|
1962
2473
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2004,9 +2515,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2004
2515
|
function isSessionActivelyGenerating(messages) {
|
|
2005
2516
|
if (!messages || messages.length === 0) return false;
|
|
2006
2517
|
const last = messages[messages.length - 1];
|
|
2007
|
-
if (
|
|
2518
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2008
2519
|
return completedOf(last) == null;
|
|
2009
2520
|
}
|
|
2521
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2010
2522
|
function sessionLastActivityMs(session) {
|
|
2011
2523
|
const candidates = [
|
|
2012
2524
|
session.time?.updated,
|
|
@@ -2021,30 +2533,71 @@ function sessionLastActivityMs(session) {
|
|
|
2021
2533
|
}
|
|
2022
2534
|
return null;
|
|
2023
2535
|
}
|
|
2024
|
-
|
|
2025
|
-
|
|
2536
|
+
function sessionListPath(options) {
|
|
2537
|
+
const params = new URLSearchParams();
|
|
2538
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2539
|
+
const query = params.toString();
|
|
2540
|
+
return query ? `/session?${query}` : "/session";
|
|
2541
|
+
}
|
|
2542
|
+
async function listSessionPage(port, client, options) {
|
|
2543
|
+
const path = sessionListPath(options);
|
|
2026
2544
|
try {
|
|
2027
|
-
const res = await requestWithClient(port, client,
|
|
2545
|
+
const res = await requestWithClient(port, client, path);
|
|
2028
2546
|
if (!res.ok) return null;
|
|
2029
2547
|
const body = await res.json();
|
|
2030
2548
|
return Array.isArray(body) ? body : null;
|
|
2031
2549
|
} catch (error2) {
|
|
2032
2550
|
console.error(
|
|
2033
|
-
`[
|
|
2551
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2034
2552
|
);
|
|
2035
2553
|
return null;
|
|
2036
2554
|
}
|
|
2037
2555
|
}
|
|
2556
|
+
async function listSessions(port, client, options) {
|
|
2557
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2558
|
+
return listSessionPage(port, client, options);
|
|
2559
|
+
}
|
|
2560
|
+
async function listAllSessions(port, client) {
|
|
2561
|
+
if (client?.version === "v2") {
|
|
2562
|
+
const sessions2 = await listV2Sessions(client);
|
|
2563
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2564
|
+
}
|
|
2565
|
+
const sessions = await listSessionPage(port, client, {
|
|
2566
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2567
|
+
});
|
|
2568
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2569
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2570
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2571
|
+
}
|
|
2572
|
+
return { sessions };
|
|
2573
|
+
}
|
|
2038
2574
|
async function deleteSession(port, id, client) {
|
|
2039
2575
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2040
2576
|
try {
|
|
2041
|
-
const res = await requestWithClient(
|
|
2042
|
-
|
|
2577
|
+
const res = await requestWithClient(
|
|
2578
|
+
port,
|
|
2579
|
+
client,
|
|
2580
|
+
`/session/${id}`,
|
|
2581
|
+
{ method: "DELETE" },
|
|
2582
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2583
|
+
);
|
|
2584
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2585
|
+
return { ok: true, status: res.status };
|
|
2586
|
+
}
|
|
2587
|
+
try {
|
|
2588
|
+
const body = await res.text();
|
|
2589
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2590
|
+
} catch (error2) {
|
|
2591
|
+
console.error(
|
|
2592
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2593
|
+
);
|
|
2594
|
+
return { ok: false, status: res.status };
|
|
2595
|
+
}
|
|
2043
2596
|
} catch (error2) {
|
|
2044
2597
|
console.error(
|
|
2045
2598
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2046
2599
|
);
|
|
2047
|
-
return false;
|
|
2600
|
+
return { ok: false, status: "threw" };
|
|
2048
2601
|
}
|
|
2049
2602
|
}
|
|
2050
2603
|
async function sessionExists(port, id, client) {
|
|
@@ -2282,11 +2835,31 @@ function messageText(m) {
|
|
|
2282
2835
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2283
2836
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2284
2837
|
}
|
|
2285
|
-
|
|
2838
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2839
|
+
if (!messages || messages.length === 0) return null;
|
|
2840
|
+
let best = null;
|
|
2841
|
+
for (const message of messages) {
|
|
2842
|
+
if (roleOf2(message) !== "user") continue;
|
|
2843
|
+
const id = idOf2(message);
|
|
2844
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2845
|
+
if (messageText(message) !== content) continue;
|
|
2846
|
+
const created = createdOf(message) ?? 0;
|
|
2847
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2848
|
+
}
|
|
2849
|
+
return best?.id ?? null;
|
|
2850
|
+
}
|
|
2851
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2286
2852
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2287
2853
|
const knownUserIds = new Set(
|
|
2288
|
-
(before ?? []).filter((m) =>
|
|
2854
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2289
2855
|
);
|
|
2856
|
+
if (callbacks) {
|
|
2857
|
+
try {
|
|
2858
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2859
|
+
} catch (error2) {
|
|
2860
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2290
2863
|
const parts = [{ type: "text", text: content }];
|
|
2291
2864
|
let pendingOutcomes = null;
|
|
2292
2865
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2325,22 +2898,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2325
2898
|
const READ_BACK_DELAY_MS = 150;
|
|
2326
2899
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2327
2900
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
const id = idOf(m);
|
|
2333
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2334
|
-
if (messageText(m) !== content) continue;
|
|
2335
|
-
const created = createdOf(m) ?? 0;
|
|
2336
|
-
if (best === null || created > best.created) {
|
|
2337
|
-
best = { id, created };
|
|
2338
|
-
}
|
|
2339
|
-
}
|
|
2340
|
-
if (best) {
|
|
2341
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2342
|
-
return best.id;
|
|
2343
|
-
}
|
|
2901
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2902
|
+
if (dispatchedUserMessageId !== null) {
|
|
2903
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2904
|
+
return dispatchedUserMessageId;
|
|
2344
2905
|
}
|
|
2345
2906
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2346
2907
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2351,57 +2912,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2351
2912
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2352
2913
|
if (!messages || messages.length === 0) return null;
|
|
2353
2914
|
const byParent = messages.find(
|
|
2354
|
-
(m) =>
|
|
2915
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2355
2916
|
);
|
|
2356
2917
|
if (byParent) return byParent;
|
|
2357
|
-
const userIndex = messages.findIndex((m) =>
|
|
2918
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2358
2919
|
if (userIndex === -1) return null;
|
|
2359
2920
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2360
|
-
if (
|
|
2921
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2361
2922
|
}
|
|
2362
2923
|
return null;
|
|
2363
2924
|
}
|
|
2364
2925
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2365
2926
|
if (!messages || messages.length === 0) return null;
|
|
2366
2927
|
let lastCorrelated = null;
|
|
2367
|
-
let lastNonErrored = null;
|
|
2368
2928
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2369
2929
|
const m = messages[i];
|
|
2370
|
-
if (
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2930
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2931
|
+
lastCorrelated = m;
|
|
2932
|
+
break;
|
|
2933
|
+
}
|
|
2934
|
+
if (lastCorrelated) {
|
|
2935
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2936
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2937
|
+
const m = messages[i];
|
|
2938
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2939
|
+
return m;
|
|
2940
|
+
}
|
|
2375
2941
|
}
|
|
2942
|
+
return lastCorrelated;
|
|
2376
2943
|
}
|
|
2377
|
-
|
|
2378
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2944
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2379
2945
|
if (userIndex === -1) return null;
|
|
2380
2946
|
let last = null;
|
|
2381
|
-
let
|
|
2947
|
+
let lastAssistantIndex = -1;
|
|
2382
2948
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2383
|
-
const role =
|
|
2949
|
+
const role = roleOf2(messages[i]);
|
|
2384
2950
|
if (role === "user") break;
|
|
2385
2951
|
if (role === "assistant") {
|
|
2386
2952
|
last = messages[i];
|
|
2387
|
-
|
|
2953
|
+
lastAssistantIndex = i;
|
|
2388
2954
|
}
|
|
2389
2955
|
}
|
|
2390
|
-
|
|
2956
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2957
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2958
|
+
const m = messages[i];
|
|
2959
|
+
if (roleOf2(m) === "user") break;
|
|
2960
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2961
|
+
}
|
|
2962
|
+
return last;
|
|
2391
2963
|
}
|
|
2392
2964
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2393
2965
|
if (!messages || messages.length === 0) return [];
|
|
2394
2966
|
const byParent = messages.filter(
|
|
2395
|
-
(message) =>
|
|
2967
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2396
2968
|
);
|
|
2397
2969
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2398
2970
|
if (assistants.length === 0) {
|
|
2399
|
-
const userIndex = messages.findIndex((message) =>
|
|
2971
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2400
2972
|
if (userIndex === -1) return [];
|
|
2401
2973
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2402
2974
|
const message = messages[i];
|
|
2403
|
-
if (
|
|
2404
|
-
if (
|
|
2975
|
+
if (roleOf2(message) === "user") break;
|
|
2976
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2405
2977
|
}
|
|
2406
2978
|
}
|
|
2407
2979
|
const refs = [];
|
|
@@ -2441,7 +3013,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2441
3013
|
if (!messages || messages.length === 0) return [];
|
|
2442
3014
|
const calls = [];
|
|
2443
3015
|
for (const message of messages) {
|
|
2444
|
-
if (
|
|
3016
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2445
3017
|
for (const part of message.parts ?? []) {
|
|
2446
3018
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2447
3019
|
continue;
|
|
@@ -2471,7 +3043,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2471
3043
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2472
3044
|
const unattributed = [];
|
|
2473
3045
|
for (const message of messages ?? []) {
|
|
2474
|
-
if (
|
|
3046
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2475
3047
|
const created = finiteNumber(createdOf(message));
|
|
2476
3048
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2477
3049
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2493,8 +3065,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2493
3065
|
}
|
|
2494
3066
|
function sumAssistantUsage(messages) {
|
|
2495
3067
|
if (!messages || messages.length === 0) return null;
|
|
2496
|
-
const
|
|
2497
|
-
const
|
|
3068
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3069
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3070
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2498
3071
|
let sawAnyUsage = false;
|
|
2499
3072
|
let inputSum = 0;
|
|
2500
3073
|
let outputSum = 0;
|
|
@@ -2503,9 +3076,11 @@ function sumAssistantUsage(messages) {
|
|
|
2503
3076
|
let cacheWriteSum = 0;
|
|
2504
3077
|
let costSum = 0;
|
|
2505
3078
|
let sawCost = false;
|
|
3079
|
+
let toolCallCount = 0;
|
|
2506
3080
|
let modelId = null;
|
|
2507
3081
|
let providerId = null;
|
|
2508
3082
|
for (const m of selected) {
|
|
3083
|
+
toolCallCount += m.parts?.filter((part) => part.type === "tool").length ?? 0;
|
|
2509
3084
|
const info = m.info;
|
|
2510
3085
|
if (!info) continue;
|
|
2511
3086
|
const tokens = info.tokens;
|
|
@@ -2542,19 +3117,18 @@ function sumAssistantUsage(messages) {
|
|
|
2542
3117
|
usage_tokens_cache_write: cacheWriteSum,
|
|
2543
3118
|
// NULL means OpenCode never reported a cost; it is distinct from a genuine
|
|
2544
3119
|
// zero-cost message, which sets `sawCost` with `costSum === 0`.
|
|
2545
|
-
usage_cost_usd: sawCost ? costSum : null
|
|
3120
|
+
usage_cost_usd: sawCost ? costSum : null,
|
|
3121
|
+
turn_tool_calls: toolCallCount
|
|
2546
3122
|
};
|
|
2547
3123
|
}
|
|
2548
3124
|
function messageUsage(messages, userMessageId) {
|
|
2549
3125
|
if (!messages || messages.length === 0) return null;
|
|
2550
3126
|
const byParentAll = messages.filter(
|
|
2551
|
-
(m) =>
|
|
3127
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2552
3128
|
);
|
|
2553
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2554
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2555
3129
|
let correlated;
|
|
2556
|
-
if (
|
|
2557
|
-
correlated =
|
|
3130
|
+
if (byParentAll.length > 0) {
|
|
3131
|
+
correlated = byParentAll;
|
|
2558
3132
|
} else {
|
|
2559
3133
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2560
3134
|
correlated = reply ? [reply] : [];
|
|
@@ -2564,7 +3138,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2564
3138
|
}
|
|
2565
3139
|
function messageRunState(messages, userMessageId) {
|
|
2566
3140
|
if (!messages || messages.length === 0) return "unknown";
|
|
2567
|
-
const hasUser = messages.some((m) =>
|
|
3141
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2568
3142
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2569
3143
|
if (!hasUser) {
|
|
2570
3144
|
if (!reply) return "unknown";
|
|
@@ -2600,14 +3174,38 @@ function messageError(messages, userMessageId) {
|
|
|
2600
3174
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2601
3175
|
const error2 = errorOf(reply);
|
|
2602
3176
|
if (error2 == null) return null;
|
|
2603
|
-
|
|
2604
|
-
if (typeof error2 === "
|
|
3177
|
+
let rendered;
|
|
3178
|
+
if (typeof error2 === "string") {
|
|
3179
|
+
rendered = error2;
|
|
3180
|
+
} else if (typeof error2 === "object") {
|
|
2605
3181
|
const e = error2;
|
|
2606
3182
|
const dataMessage = e.data?.message;
|
|
2607
|
-
if (typeof dataMessage === "string")
|
|
2608
|
-
|
|
3183
|
+
if (typeof dataMessage === "string") {
|
|
3184
|
+
rendered = dataMessage;
|
|
3185
|
+
} else if (typeof e.message === "string") {
|
|
3186
|
+
rendered = e.message;
|
|
3187
|
+
} else {
|
|
3188
|
+
rendered = "The agent run failed.";
|
|
3189
|
+
}
|
|
3190
|
+
} else {
|
|
3191
|
+
rendered = "The agent run failed.";
|
|
2609
3192
|
}
|
|
2610
|
-
|
|
3193
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3194
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3195
|
+
) : void 0;
|
|
3196
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3197
|
+
}
|
|
3198
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3199
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3200
|
+
if (!reply) return null;
|
|
3201
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3202
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3203
|
+
return {
|
|
3204
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3205
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3206
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3207
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3208
|
+
};
|
|
2611
3209
|
}
|
|
2612
3210
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2613
3211
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2656,7 +3254,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2656
3254
|
if (!messages || messages.length === 0) return null;
|
|
2657
3255
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2658
3256
|
const message = messages[i];
|
|
2659
|
-
if (
|
|
3257
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2660
3258
|
const created = createdOf(message);
|
|
2661
3259
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2662
3260
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2688,25 +3286,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2688
3286
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2689
3287
|
if (!messages || messages.length === 0) return false;
|
|
2690
3288
|
return messages.some(
|
|
2691
|
-
(m) =>
|
|
3289
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2692
3290
|
);
|
|
2693
3291
|
}
|
|
2694
3292
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2695
3293
|
if (!messages || messages.length === 0) return false;
|
|
2696
|
-
const userIndex = messages.findIndex((message) =>
|
|
3294
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2697
3295
|
if (userIndex === -1) return false;
|
|
2698
3296
|
let hasLaterUser = false;
|
|
2699
3297
|
let hasStartedLaterUser = false;
|
|
2700
3298
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2701
3299
|
const message = messages[i];
|
|
2702
|
-
if (
|
|
3300
|
+
if (roleOf2(message) !== "user") continue;
|
|
2703
3301
|
hasLaterUser = true;
|
|
2704
|
-
const laterUserMessageId =
|
|
3302
|
+
const laterUserMessageId = idOf2(message);
|
|
2705
3303
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2706
3304
|
return false;
|
|
2707
3305
|
}
|
|
2708
3306
|
if (messages.some(
|
|
2709
|
-
(candidate) =>
|
|
3307
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2710
3308
|
)) {
|
|
2711
3309
|
hasStartedLaterUser = true;
|
|
2712
3310
|
}
|
|
@@ -3177,13 +3775,28 @@ async function listV2Sessions(client) {
|
|
|
3177
3775
|
}
|
|
3178
3776
|
async function deleteV2Session(client, sessionId) {
|
|
3179
3777
|
try {
|
|
3180
|
-
await client.request(
|
|
3181
|
-
|
|
3778
|
+
const response = await client.request(
|
|
3779
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3780
|
+
{ method: "DELETE" },
|
|
3781
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3782
|
+
);
|
|
3783
|
+
if (response.status >= 200 && response.status < 300) {
|
|
3784
|
+
return { ok: true, status: response.status };
|
|
3785
|
+
}
|
|
3786
|
+
try {
|
|
3787
|
+
const body = await response.text();
|
|
3788
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
3789
|
+
} catch (error2) {
|
|
3790
|
+
console.error(
|
|
3791
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3792
|
+
);
|
|
3793
|
+
return { ok: false, status: response.status };
|
|
3794
|
+
}
|
|
3182
3795
|
} catch (error2) {
|
|
3183
3796
|
console.error(
|
|
3184
3797
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3185
3798
|
);
|
|
3186
|
-
return false;
|
|
3799
|
+
return { ok: false, status: "threw" };
|
|
3187
3800
|
}
|
|
3188
3801
|
}
|
|
3189
3802
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3805,9 +4418,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3805
4418
|
|
|
3806
4419
|
// src/lib/opencode/session-db-boot.ts
|
|
3807
4420
|
import { spawn as spawn2 } from "node:child_process";
|
|
3808
|
-
import { mkdirSync, statSync as
|
|
4421
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3809
4422
|
import { homedir as homedir4 } from "node:os";
|
|
3810
|
-
import { dirname as
|
|
4423
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3811
4424
|
|
|
3812
4425
|
// src/lib/runner-synchroniser.ts
|
|
3813
4426
|
import { spawn } from "node:child_process";
|
|
@@ -3887,9 +4500,12 @@ function runSynchroniser(args, opts) {
|
|
|
3887
4500
|
|
|
3888
4501
|
// src/lib/opencode/session-db-boot.ts
|
|
3889
4502
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3890
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4503
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3891
4504
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3892
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4505
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4506
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4507
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4508
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3893
4509
|
function commandError(result) {
|
|
3894
4510
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3895
4511
|
}
|
|
@@ -3932,7 +4548,7 @@ function clearMarker(options) {
|
|
|
3932
4548
|
function markNoReplicate(options, message) {
|
|
3933
4549
|
if (options.noReplicateMarker) {
|
|
3934
4550
|
try {
|
|
3935
|
-
mkdirSync(
|
|
4551
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3936
4552
|
writeFileSync(options.noReplicateMarker, "");
|
|
3937
4553
|
} catch (error2) {
|
|
3938
4554
|
options.log(
|
|
@@ -4057,7 +4673,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4057
4673
|
return null;
|
|
4058
4674
|
}
|
|
4059
4675
|
try {
|
|
4060
|
-
if (
|
|
4676
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4061
4677
|
} catch (error2) {
|
|
4062
4678
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4063
4679
|
options.log(
|
|
@@ -4087,7 +4703,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4087
4703
|
return null;
|
|
4088
4704
|
}
|
|
4089
4705
|
try {
|
|
4090
|
-
mkdirSync(
|
|
4706
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
4091
4707
|
writeFileSync(configPath, rendered.stdout);
|
|
4092
4708
|
} catch (error2) {
|
|
4093
4709
|
options.log(
|
|
@@ -4200,20 +4816,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4200
4816
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4201
4817
|
env: {
|
|
4202
4818
|
...env,
|
|
4203
|
-
// The synchroniser reads
|
|
4204
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4819
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4205
4820
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4206
4821
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4822
|
+
),
|
|
4823
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4824
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4825
|
+
),
|
|
4826
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4827
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4828
|
+
),
|
|
4829
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4830
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4207
4831
|
)
|
|
4208
4832
|
}
|
|
4209
4833
|
});
|
|
4210
4834
|
logSynchroniserDiagnostics(result, options);
|
|
4211
4835
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4836
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4837
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4838
|
+
options.log(
|
|
4839
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4840
|
+
"error"
|
|
4841
|
+
);
|
|
4842
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4843
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4844
|
+
}
|
|
4212
4845
|
options.log(
|
|
4213
4846
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4214
4847
|
"warn"
|
|
4215
4848
|
);
|
|
4216
|
-
return false;
|
|
4849
|
+
return { verifyFatal: false };
|
|
4217
4850
|
}
|
|
4218
4851
|
if (result.code === 34) {
|
|
4219
4852
|
reportRecord(
|
|
@@ -4223,14 +4856,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4223
4856
|
null,
|
|
4224
4857
|
options
|
|
4225
4858
|
);
|
|
4226
|
-
return true;
|
|
4859
|
+
return { verifyFatal: true };
|
|
4227
4860
|
}
|
|
4228
4861
|
if (result.code === 33) {
|
|
4229
4862
|
options.log(
|
|
4230
4863
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4231
4864
|
"warn"
|
|
4232
4865
|
);
|
|
4233
|
-
return false;
|
|
4866
|
+
return { verifyFatal: false };
|
|
4234
4867
|
}
|
|
4235
4868
|
if (result.code !== 0) {
|
|
4236
4869
|
options.log(
|
|
@@ -4238,17 +4871,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4238
4871
|
"warn"
|
|
4239
4872
|
);
|
|
4240
4873
|
}
|
|
4241
|
-
return false;
|
|
4874
|
+
return { verifyFatal: false };
|
|
4242
4875
|
}
|
|
4243
4876
|
options.log(
|
|
4244
4877
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4245
4878
|
"debug"
|
|
4246
4879
|
);
|
|
4247
|
-
return false;
|
|
4880
|
+
return { verifyFatal: false };
|
|
4248
4881
|
}
|
|
4249
4882
|
function fileExists(path) {
|
|
4250
4883
|
try {
|
|
4251
|
-
|
|
4884
|
+
statSync3(path);
|
|
4252
4885
|
return true;
|
|
4253
4886
|
} catch (error2) {
|
|
4254
4887
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4331,13 +4964,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4331
4964
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4332
4965
|
return { verifyFatal: false };
|
|
4333
4966
|
}
|
|
4334
|
-
return
|
|
4967
|
+
return verifySessionDb(options, configPath, env);
|
|
4335
4968
|
}
|
|
4336
4969
|
|
|
4337
4970
|
// src/lib/opencode/session-db-provenance.ts
|
|
4338
4971
|
import { createRequire } from "node:module";
|
|
4339
4972
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4340
|
-
import { dirname as
|
|
4973
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4341
4974
|
var require2 = createRequire(import.meta.url);
|
|
4342
4975
|
function readSessionDbMigrationIds(dbPath) {
|
|
4343
4976
|
let db;
|
|
@@ -4403,7 +5036,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4403
5036
|
}
|
|
4404
5037
|
function saveSessionDbProvenanceState(path, state) {
|
|
4405
5038
|
try {
|
|
4406
|
-
mkdirSync2(
|
|
5039
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4407
5040
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4408
5041
|
`, "utf8");
|
|
4409
5042
|
} catch (error2) {
|
|
@@ -4984,7 +5617,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4984
5617
|
toDelete.push(s.id);
|
|
4985
5618
|
}
|
|
4986
5619
|
}
|
|
4987
|
-
|
|
5620
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5621
|
+
return toDelete.sort(
|
|
5622
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5623
|
+
);
|
|
4988
5624
|
}
|
|
4989
5625
|
var DEFAULT_INTERVAL = "1h";
|
|
4990
5626
|
function resolve(flag, envValue, fallback) {
|
|
@@ -5044,13 +5680,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5044
5680
|
}
|
|
5045
5681
|
|
|
5046
5682
|
// src/lib/opencode/session-db-size.ts
|
|
5047
|
-
import { statSync as
|
|
5683
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5048
5684
|
import { join as join6 } from "node:path";
|
|
5049
5685
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5050
5686
|
function statSessionDbBytes(homeDir) {
|
|
5051
5687
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5052
5688
|
try {
|
|
5053
|
-
return
|
|
5689
|
+
return statSync4(dbPath).size;
|
|
5054
5690
|
} catch (err) {
|
|
5055
5691
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5056
5692
|
if (!isMissingFile) {
|
|
@@ -5076,7 +5712,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5076
5712
|
}
|
|
5077
5713
|
|
|
5078
5714
|
// src/lib/opencode/log-tail.ts
|
|
5079
|
-
import { statSync as
|
|
5715
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5080
5716
|
import { homedir as homedir5 } from "node:os";
|
|
5081
5717
|
import { join as join7 } from "node:path";
|
|
5082
5718
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -5099,186 +5735,63 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5099
5735
|
let inode = null;
|
|
5100
5736
|
let baselineReady = true;
|
|
5101
5737
|
try {
|
|
5102
|
-
const initial =
|
|
5738
|
+
const initial = statSync5(logPath);
|
|
5103
5739
|
offset = initial.size;
|
|
5104
5740
|
inode = initial.ino;
|
|
5105
5741
|
} catch (error2) {
|
|
5106
5742
|
if (!isEnoent(error2)) {
|
|
5107
5743
|
reportFailure("initial stat", logPath, error2);
|
|
5108
5744
|
baselineReady = false;
|
|
5109
|
-
}
|
|
5110
|
-
}
|
|
5111
|
-
let polling = false;
|
|
5112
|
-
let stopped = false;
|
|
5113
|
-
const poll = async () => {
|
|
5114
|
-
if (polling || stopped) return;
|
|
5115
|
-
polling = true;
|
|
5116
|
-
try {
|
|
5117
|
-
let current;
|
|
5118
|
-
try {
|
|
5119
|
-
current = await stat(logPath);
|
|
5120
|
-
} catch (error2) {
|
|
5121
|
-
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5122
|
-
return;
|
|
5123
|
-
}
|
|
5124
|
-
if (!baselineReady) {
|
|
5125
|
-
offset = current.size;
|
|
5126
|
-
inode = current.ino;
|
|
5127
|
-
baselineReady = true;
|
|
5128
|
-
return;
|
|
5129
|
-
}
|
|
5130
|
-
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5131
|
-
offset = 0;
|
|
5132
|
-
}
|
|
5133
|
-
inode = current.ino;
|
|
5134
|
-
if (current.size === offset) return;
|
|
5135
|
-
const length = current.size - offset;
|
|
5136
|
-
const fh = await open2(logPath, "r");
|
|
5137
|
-
try {
|
|
5138
|
-
const buf = Buffer.alloc(length);
|
|
5139
|
-
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5140
|
-
offset += bytesRead;
|
|
5141
|
-
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5142
|
-
} finally {
|
|
5143
|
-
await fh.close();
|
|
5144
|
-
}
|
|
5145
|
-
} catch (error2) {
|
|
5146
|
-
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5147
|
-
} finally {
|
|
5148
|
-
polling = false;
|
|
5149
|
-
}
|
|
5150
|
-
};
|
|
5151
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5152
|
-
void poll();
|
|
5153
|
-
return {
|
|
5154
|
-
stop: () => {
|
|
5155
|
-
stopped = true;
|
|
5156
|
-
clearInterval(interval);
|
|
5157
|
-
}
|
|
5158
|
-
};
|
|
5159
|
-
}
|
|
5160
|
-
|
|
5161
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5162
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5163
|
-
import { dirname as dirname4 } from "node:path";
|
|
5164
|
-
function errorMessage(error2) {
|
|
5165
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5166
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5167
|
-
}
|
|
5168
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5169
|
-
try {
|
|
5170
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5171
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5172
|
-
if (availableBytes < requiredBytes) {
|
|
5173
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5174
|
-
}
|
|
5175
|
-
return null;
|
|
5176
|
-
} catch (err) {
|
|
5177
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5178
|
-
}
|
|
5179
|
-
}
|
|
5180
|
-
function readLogicalBytes(db) {
|
|
5181
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5182
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5183
|
-
return pageCount * pageSize;
|
|
5184
|
-
}
|
|
5185
|
-
function readCheckpointResult(db) {
|
|
5186
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5187
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5188
|
-
}
|
|
5189
|
-
async function probeReclaimAvailability(input) {
|
|
5190
|
-
const { dbPath, requiredBytes } = input;
|
|
5191
|
-
let sqlite;
|
|
5192
|
-
try {
|
|
5193
|
-
sqlite = await import("node:sqlite");
|
|
5194
|
-
} catch (err) {
|
|
5195
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5196
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5197
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5198
|
-
}
|
|
5199
|
-
let autoVacuum = null;
|
|
5200
|
-
try {
|
|
5201
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5202
|
-
try {
|
|
5203
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5204
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5205
|
-
} finally {
|
|
5206
|
-
db.close();
|
|
5207
|
-
}
|
|
5208
|
-
} catch (err) {
|
|
5209
|
-
console.warn(
|
|
5210
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5211
|
-
);
|
|
5212
|
-
}
|
|
5213
|
-
if (autoVacuum !== 0) return null;
|
|
5214
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5215
|
-
}
|
|
5216
|
-
async function reclaimSessionDbSpace(input) {
|
|
5217
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5218
|
-
let sqlite;
|
|
5219
|
-
try {
|
|
5220
|
-
sqlite = await import("node:sqlite");
|
|
5221
|
-
} catch (err) {
|
|
5222
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5223
|
-
console.warn(
|
|
5224
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5225
|
-
);
|
|
5226
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5227
|
-
}
|
|
5228
|
-
const { DatabaseSync } = sqlite;
|
|
5229
|
-
let db;
|
|
5230
|
-
try {
|
|
5231
|
-
db = new DatabaseSync(dbPath);
|
|
5232
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5233
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5234
|
-
if (autoVacuum === 0) {
|
|
5235
|
-
if (!allowFullVacuum) {
|
|
5236
|
-
console.warn(
|
|
5237
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5238
|
-
);
|
|
5239
|
-
return {
|
|
5240
|
-
ok: false,
|
|
5241
|
-
skipped: "full-vacuum-blocked",
|
|
5242
|
-
detail: "a session turn is live"
|
|
5243
|
-
};
|
|
5745
|
+
}
|
|
5746
|
+
}
|
|
5747
|
+
let polling = false;
|
|
5748
|
+
let stopped = false;
|
|
5749
|
+
const poll = async () => {
|
|
5750
|
+
if (polling || stopped) return;
|
|
5751
|
+
polling = true;
|
|
5752
|
+
try {
|
|
5753
|
+
let current;
|
|
5754
|
+
try {
|
|
5755
|
+
current = await stat(logPath);
|
|
5756
|
+
} catch (error2) {
|
|
5757
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5758
|
+
return;
|
|
5244
5759
|
}
|
|
5245
|
-
|
|
5246
|
-
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
);
|
|
5251
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5760
|
+
if (!baselineReady) {
|
|
5761
|
+
offset = current.size;
|
|
5762
|
+
inode = current.ino;
|
|
5763
|
+
baselineReady = true;
|
|
5764
|
+
return;
|
|
5252
5765
|
}
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5766
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5767
|
+
offset = 0;
|
|
5768
|
+
}
|
|
5769
|
+
inode = current.ino;
|
|
5770
|
+
if (current.size === offset) return;
|
|
5771
|
+
const length = current.size - offset;
|
|
5772
|
+
const fh = await open2(logPath, "r");
|
|
5773
|
+
try {
|
|
5774
|
+
const buf = Buffer.alloc(length);
|
|
5775
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5776
|
+
offset += bytesRead;
|
|
5777
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5778
|
+
} finally {
|
|
5779
|
+
await fh.close();
|
|
5780
|
+
}
|
|
5781
|
+
} catch (error2) {
|
|
5782
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5783
|
+
} finally {
|
|
5784
|
+
polling = false;
|
|
5259
5785
|
}
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5786
|
+
};
|
|
5787
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5788
|
+
void poll();
|
|
5789
|
+
return {
|
|
5790
|
+
stop: () => {
|
|
5791
|
+
stopped = true;
|
|
5792
|
+
clearInterval(interval);
|
|
5267
5793
|
}
|
|
5268
|
-
|
|
5269
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5270
|
-
);
|
|
5271
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5272
|
-
} catch (err) {
|
|
5273
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5274
|
-
return {
|
|
5275
|
-
ok: false,
|
|
5276
|
-
skipped: "reclaim-error",
|
|
5277
|
-
detail: errorMessage(err)
|
|
5278
|
-
};
|
|
5279
|
-
} finally {
|
|
5280
|
-
db?.close();
|
|
5281
|
-
}
|
|
5794
|
+
};
|
|
5282
5795
|
}
|
|
5283
5796
|
|
|
5284
5797
|
// src/lib/tunnel/connection.ts
|
|
@@ -5510,6 +6023,8 @@ var StreamForwarder = class {
|
|
|
5510
6023
|
if (done) break;
|
|
5511
6024
|
const chunk = Buffer.from(value);
|
|
5512
6025
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
6026
|
+
await stream.pausePromise;
|
|
6027
|
+
if (ac.signal.aborted) break;
|
|
5513
6028
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5514
6029
|
if (this.binaryFramesSupported) {
|
|
5515
6030
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5552,6 +6067,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5552
6067
|
}
|
|
5553
6068
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5554
6069
|
var BASE_RECONNECT_DELAY = 500;
|
|
6070
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5555
6071
|
function getReconnectDelay(attempt) {
|
|
5556
6072
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5557
6073
|
const jitter = Math.random() * 1e3;
|
|
@@ -5623,10 +6139,27 @@ function connectTunnel(options) {
|
|
|
5623
6139
|
ws.close();
|
|
5624
6140
|
reject(new Error("Connection timeout"));
|
|
5625
6141
|
}, 3e4);
|
|
6142
|
+
let relaySilenceTimeout;
|
|
6143
|
+
let relaySilenceTimedOut = false;
|
|
6144
|
+
const clearRelaySilenceTimeout = () => {
|
|
6145
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6146
|
+
clearTimeout(relaySilenceTimeout);
|
|
6147
|
+
relaySilenceTimeout = void 0;
|
|
6148
|
+
};
|
|
6149
|
+
const armRelaySilenceTimeout = () => {
|
|
6150
|
+
clearRelaySilenceTimeout();
|
|
6151
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6152
|
+
relaySilenceTimeout = void 0;
|
|
6153
|
+
relaySilenceTimedOut = true;
|
|
6154
|
+
ws.terminate();
|
|
6155
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6156
|
+
relaySilenceTimeout.unref?.();
|
|
6157
|
+
};
|
|
5626
6158
|
let upgradeRejection = null;
|
|
5627
6159
|
let upgradeRejectionReason = null;
|
|
5628
6160
|
ws.on("unexpected-response", (_req, res) => {
|
|
5629
6161
|
clearTimeout(connectionTimeout);
|
|
6162
|
+
clearRelaySilenceTimeout();
|
|
5630
6163
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5631
6164
|
upgradeRejectionReason = reason;
|
|
5632
6165
|
const chunks = [];
|
|
@@ -5656,6 +6189,7 @@ function connectTunnel(options) {
|
|
|
5656
6189
|
onInfo?.("WebSocket connection established");
|
|
5657
6190
|
});
|
|
5658
6191
|
ws.on("message", (data, isBinary) => {
|
|
6192
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5659
6193
|
if (isBinary) {
|
|
5660
6194
|
try {
|
|
5661
6195
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5689,6 +6223,7 @@ function connectTunnel(options) {
|
|
|
5689
6223
|
switch (message.type) {
|
|
5690
6224
|
case "connected": {
|
|
5691
6225
|
clearTimeout(connectionTimeout);
|
|
6226
|
+
armRelaySilenceTimeout();
|
|
5692
6227
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5693
6228
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5694
6229
|
onConnected?.(connectedAgentId);
|
|
@@ -5713,6 +6248,7 @@ function connectTunnel(options) {
|
|
|
5713
6248
|
});
|
|
5714
6249
|
ws.on("error", (error2) => {
|
|
5715
6250
|
clearTimeout(connectionTimeout);
|
|
6251
|
+
clearRelaySilenceTimeout();
|
|
5716
6252
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5717
6253
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5718
6254
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5724,7 +6260,8 @@ function connectTunnel(options) {
|
|
|
5724
6260
|
);
|
|
5725
6261
|
});
|
|
5726
6262
|
ws.on("close", (code, reason) => {
|
|
5727
|
-
|
|
6263
|
+
clearRelaySilenceTimeout();
|
|
6264
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5728
6265
|
forwarder.abortAll();
|
|
5729
6266
|
onDisconnected?.(code, reasonStr);
|
|
5730
6267
|
});
|
|
@@ -5732,7 +6269,7 @@ function connectTunnel(options) {
|
|
|
5732
6269
|
}
|
|
5733
6270
|
|
|
5734
6271
|
// src/lib/tunnel/runner-connection.ts
|
|
5735
|
-
var RunnerConnection = class {
|
|
6272
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5736
6273
|
opts;
|
|
5737
6274
|
sleep;
|
|
5738
6275
|
now;
|
|
@@ -5746,6 +6283,10 @@ var RunnerConnection = class {
|
|
|
5746
6283
|
reconnectAttempt = 0;
|
|
5747
6284
|
generation = 0;
|
|
5748
6285
|
disconnectedAtMs = null;
|
|
6286
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6287
|
+
"No pong received",
|
|
6288
|
+
"Orphaned socket"
|
|
6289
|
+
]);
|
|
5749
6290
|
constructor(opts) {
|
|
5750
6291
|
this.opts = opts;
|
|
5751
6292
|
this.resolvedAgentId = opts.agentId;
|
|
@@ -5802,7 +6343,8 @@ var RunnerConnection = class {
|
|
|
5802
6343
|
onDisconnected: (code, reason) => {
|
|
5803
6344
|
this.disconnectedAtMs = this.now();
|
|
5804
6345
|
events.onDisconnected(code, reason, { generation: this.generation });
|
|
5805
|
-
|
|
6346
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6347
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5806
6348
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5807
6349
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5808
6350
|
});
|
|
@@ -6141,6 +6683,101 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6141
6683
|
};
|
|
6142
6684
|
}
|
|
6143
6685
|
|
|
6686
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6687
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6688
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6689
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6690
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6691
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6692
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6693
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6694
|
+
var BAND_RANK = {
|
|
6695
|
+
low: 1,
|
|
6696
|
+
very_low: 2,
|
|
6697
|
+
critical: 3
|
|
6698
|
+
};
|
|
6699
|
+
function bandFor(freeBytes) {
|
|
6700
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6701
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6702
|
+
return "low";
|
|
6703
|
+
}
|
|
6704
|
+
function formatBytes(bytes) {
|
|
6705
|
+
if (bytes === null) return "unavailable";
|
|
6706
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6707
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6708
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6709
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6710
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6711
|
+
}
|
|
6712
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6713
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6714
|
+
}
|
|
6715
|
+
function evaluateLowDiskWarning(input) {
|
|
6716
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6717
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6718
|
+
return { warning: null, state };
|
|
6719
|
+
}
|
|
6720
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6721
|
+
if (recovered) {
|
|
6722
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6723
|
+
}
|
|
6724
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6725
|
+
if (!low) return { warning: null, state };
|
|
6726
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6727
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6728
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6729
|
+
const worse = currentRank > previousRank;
|
|
6730
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6731
|
+
const nextState = {
|
|
6732
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6733
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6734
|
+
};
|
|
6735
|
+
return {
|
|
6736
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6737
|
+
state: nextState
|
|
6738
|
+
};
|
|
6739
|
+
}
|
|
6740
|
+
|
|
6741
|
+
// src/lib/opencode/session-store-health.ts
|
|
6742
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6743
|
+
function createSessionStoreHealth(options = {}) {
|
|
6744
|
+
const now = options.now ?? (() => Date.now());
|
|
6745
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6746
|
+
let lastUnwritableAt = null;
|
|
6747
|
+
let lastWritableAt = null;
|
|
6748
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6749
|
+
return {
|
|
6750
|
+
observeUnwritable: () => {
|
|
6751
|
+
lastUnwritableAt = now();
|
|
6752
|
+
},
|
|
6753
|
+
observeWritable: () => {
|
|
6754
|
+
lastWritableAt = now();
|
|
6755
|
+
},
|
|
6756
|
+
isUnwritable: () => {
|
|
6757
|
+
if (lastUnwritableAt === null) return false;
|
|
6758
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6759
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6760
|
+
},
|
|
6761
|
+
observeCause: (sessionId, cause) => {
|
|
6762
|
+
if (sessionId === null) return;
|
|
6763
|
+
causes.delete(sessionId);
|
|
6764
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6765
|
+
while (causes.size > 256) {
|
|
6766
|
+
const oldest = causes.keys().next().value;
|
|
6767
|
+
if (typeof oldest !== "string") break;
|
|
6768
|
+
causes.delete(oldest);
|
|
6769
|
+
}
|
|
6770
|
+
},
|
|
6771
|
+
readCause: (sessionId) => {
|
|
6772
|
+
const observed = causes.get(sessionId);
|
|
6773
|
+
if (!observed) return null;
|
|
6774
|
+
causes.delete(sessionId);
|
|
6775
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6776
|
+
return observed.cause;
|
|
6777
|
+
}
|
|
6778
|
+
};
|
|
6779
|
+
}
|
|
6780
|
+
|
|
6144
6781
|
// src/lib/channels/driver.ts
|
|
6145
6782
|
import { homedir as homedir6 } from "node:os";
|
|
6146
6783
|
|
|
@@ -6577,6 +7214,11 @@ function messageIdOf(m) {
|
|
|
6577
7214
|
const infoId = m.info?.id;
|
|
6578
7215
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6579
7216
|
}
|
|
7217
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7218
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7219
|
+
if (replyShape === null) return base;
|
|
7220
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7221
|
+
}
|
|
6580
7222
|
function cleanImageMime(contentType) {
|
|
6581
7223
|
if (!contentType) return null;
|
|
6582
7224
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6597,8 +7239,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6597
7239
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6598
7240
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6599
7241
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7242
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7243
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7244
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7245
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6600
7246
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6601
7247
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7248
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6602
7249
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6603
7250
|
var HEARTBEAT_MS = 6e4;
|
|
6604
7251
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6660,6 +7307,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6660
7307
|
maxActiveSessions;
|
|
6661
7308
|
watcherStallMs;
|
|
6662
7309
|
wedgeWarningIntervalMs;
|
|
7310
|
+
sessionStoreHealth;
|
|
6663
7311
|
openCodeClient;
|
|
6664
7312
|
/** Cache of conversationId → opencode sessionId. */
|
|
6665
7313
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6752,6 +7400,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6752
7400
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6753
7401
|
*/
|
|
6754
7402
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7403
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6755
7404
|
/**
|
|
6756
7405
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6757
7406
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6784,10 +7433,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6784
7433
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6785
7434
|
*/
|
|
6786
7435
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7436
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7437
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6787
7438
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6788
7439
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6789
7440
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6790
7441
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7442
|
+
/**
|
|
7443
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7444
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7445
|
+
* identical-text turn, so it never produces a correlation.
|
|
7446
|
+
*/
|
|
7447
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6791
7448
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6792
7449
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6793
7450
|
/**
|
|
@@ -6873,21 +7530,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6873
7530
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6874
7531
|
*/
|
|
6875
7532
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6876
|
-
/**
|
|
6877
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6878
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6879
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6880
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6881
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6882
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6883
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6884
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6885
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6886
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6887
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6888
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6889
|
-
*/
|
|
6890
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6891
7533
|
/**
|
|
6892
7534
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6893
7535
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6905,8 +7547,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6905
7547
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6906
7548
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6907
7549
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6908
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6909
|
-
*
|
|
7550
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7551
|
+
* the terminal-report fence and is never sent again).
|
|
6910
7552
|
*/
|
|
6911
7553
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6912
7554
|
/**
|
|
@@ -7029,6 +7671,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7029
7671
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7030
7672
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7031
7673
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7674
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7032
7675
|
}
|
|
7033
7676
|
get isV2() {
|
|
7034
7677
|
return this.openCodeClient.version === "v2";
|
|
@@ -7044,7 +7687,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7044
7687
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7045
7688
|
}
|
|
7046
7689
|
async listSessions() {
|
|
7047
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7690
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7691
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7692
|
+
});
|
|
7048
7693
|
}
|
|
7049
7694
|
async sessionExists(sessionId) {
|
|
7050
7695
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7221,6 +7866,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7221
7866
|
}
|
|
7222
7867
|
return ids;
|
|
7223
7868
|
}
|
|
7869
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7870
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7871
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7872
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7873
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7874
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7875
|
+
}
|
|
7876
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7877
|
+
}
|
|
7878
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7879
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7880
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7881
|
+
this.awaitingReadopt.delete(messageId);
|
|
7882
|
+
this.sessions.delete(conversationId);
|
|
7883
|
+
this.supersede(conversationId, sessionId);
|
|
7884
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7885
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7886
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7887
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7888
|
+
this.log({
|
|
7889
|
+
level: "error",
|
|
7890
|
+
message: errorMessage3,
|
|
7891
|
+
conversation_id: conversationId,
|
|
7892
|
+
message_id: messageId
|
|
7893
|
+
});
|
|
7894
|
+
const reportError = await this.reportUntrackableAck(
|
|
7895
|
+
conversationId,
|
|
7896
|
+
messageId,
|
|
7897
|
+
null,
|
|
7898
|
+
errorMessage3
|
|
7899
|
+
);
|
|
7900
|
+
if (reportError !== null) {
|
|
7901
|
+
this.log({
|
|
7902
|
+
level: "warn",
|
|
7903
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7904
|
+
conversation_id: conversationId,
|
|
7905
|
+
message_id: messageId
|
|
7906
|
+
});
|
|
7907
|
+
}
|
|
7908
|
+
if (source === "readopt") {
|
|
7909
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7910
|
+
}
|
|
7911
|
+
}
|
|
7224
7912
|
/**
|
|
7225
7913
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7226
7914
|
*
|
|
@@ -7400,6 +8088,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7400
8088
|
}
|
|
7401
8089
|
break;
|
|
7402
8090
|
}
|
|
8091
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8092
|
+
if (correlation) {
|
|
8093
|
+
correlation.attempts += 1;
|
|
8094
|
+
let correlatedId = null;
|
|
8095
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8096
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8097
|
+
conv,
|
|
8098
|
+
message,
|
|
8099
|
+
correlation.sessionId
|
|
8100
|
+
);
|
|
8101
|
+
if (polled.ok)
|
|
8102
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8103
|
+
} else if (correlation.knownUserIds === null) {
|
|
8104
|
+
this.log({
|
|
8105
|
+
level: "debug",
|
|
8106
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8107
|
+
conversation_id: conv.id,
|
|
8108
|
+
message_id: message.id
|
|
8109
|
+
});
|
|
8110
|
+
}
|
|
8111
|
+
if (correlatedId !== null) {
|
|
8112
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8113
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8114
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8115
|
+
this.dispatched.add(message.id);
|
|
8116
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8117
|
+
dispatched += 1;
|
|
8118
|
+
this.log({
|
|
8119
|
+
level: "info",
|
|
8120
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8121
|
+
conversation_id: conv.id,
|
|
8122
|
+
message_id: message.id
|
|
8123
|
+
});
|
|
8124
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8125
|
+
opencode_message_id: correlatedId
|
|
8126
|
+
});
|
|
8127
|
+
continue;
|
|
8128
|
+
}
|
|
8129
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8130
|
+
await this.handoffUntrackableCorrelation(
|
|
8131
|
+
conv.id,
|
|
8132
|
+
message.id,
|
|
8133
|
+
correlation.sessionId,
|
|
8134
|
+
correlation.attempts,
|
|
8135
|
+
"fresh"
|
|
8136
|
+
);
|
|
8137
|
+
break;
|
|
8138
|
+
}
|
|
8139
|
+
break;
|
|
8140
|
+
}
|
|
7403
8141
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7404
8142
|
if (effectiveOpencodeMessageId) {
|
|
7405
8143
|
const outcome = await this.resolveRedrive(
|
|
@@ -7421,6 +8159,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7421
8159
|
model: message.opencode_model ?? void 0
|
|
7422
8160
|
};
|
|
7423
8161
|
let opencodeMessageId;
|
|
8162
|
+
let preDispatchUserIds = null;
|
|
7424
8163
|
try {
|
|
7425
8164
|
this.log({
|
|
7426
8165
|
level: "info",
|
|
@@ -7450,7 +8189,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7450
8189
|
message.content,
|
|
7451
8190
|
options,
|
|
7452
8191
|
sendAttachments,
|
|
7453
|
-
this.openCodeClient
|
|
8192
|
+
this.openCodeClient,
|
|
8193
|
+
{
|
|
8194
|
+
onKnownUserIds: (ids) => {
|
|
8195
|
+
preDispatchUserIds = ids;
|
|
8196
|
+
},
|
|
8197
|
+
onKnownUserIdsError: (error2) => {
|
|
8198
|
+
this.log({
|
|
8199
|
+
level: "warn",
|
|
8200
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8201
|
+
conversation_id: conv.id,
|
|
8202
|
+
message_id: message.id
|
|
8203
|
+
});
|
|
8204
|
+
}
|
|
8205
|
+
}
|
|
7454
8206
|
)
|
|
7455
8207
|
);
|
|
7456
8208
|
} catch (err) {
|
|
@@ -7458,6 +8210,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7458
8210
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7459
8211
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7460
8212
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8213
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7461
8214
|
this.log({
|
|
7462
8215
|
level: "error",
|
|
7463
8216
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7533,39 +8286,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7533
8286
|
if (this.isV2) {
|
|
7534
8287
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7535
8288
|
}
|
|
7536
|
-
|
|
7537
|
-
|
|
7538
|
-
|
|
7539
|
-
|
|
7540
|
-
|
|
7541
|
-
|
|
7542
|
-
message_id: message.id
|
|
7543
|
-
});
|
|
7544
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7545
|
-
continue;
|
|
7546
|
-
}
|
|
7547
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7548
|
-
this.sessions.delete(conv.id);
|
|
7549
|
-
this.supersede(conv.id, sessionId);
|
|
7550
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8289
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8290
|
+
sessionId,
|
|
8291
|
+
content: message.content,
|
|
8292
|
+
attempts: 0,
|
|
8293
|
+
knownUserIds: preDispatchUserIds
|
|
8294
|
+
});
|
|
7551
8295
|
this.log({
|
|
7552
|
-
level: "
|
|
7553
|
-
message:
|
|
8296
|
+
level: "warn",
|
|
8297
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7554
8298
|
conversation_id: conv.id,
|
|
7555
8299
|
message_id: message.id
|
|
7556
8300
|
});
|
|
7557
|
-
|
|
7558
|
-
this.log({
|
|
7559
|
-
level: "warn",
|
|
7560
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7561
|
-
conversation_id: conv.id,
|
|
7562
|
-
message_id: message.id
|
|
7563
|
-
});
|
|
7564
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7565
|
-
});
|
|
8301
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7566
8302
|
break;
|
|
7567
8303
|
}
|
|
7568
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7569
8304
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7570
8305
|
this.releasedOpencodeIds.delete(message.id);
|
|
7571
8306
|
this.dispatched.add(message.id);
|
|
@@ -8170,23 +8905,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8170
8905
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8171
8906
|
return 1;
|
|
8172
8907
|
}
|
|
8173
|
-
/**
|
|
8174
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8175
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8176
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8177
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8178
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8179
|
-
* count, since a new session is a genuinely different attempt.
|
|
8180
|
-
*/
|
|
8181
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8182
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8183
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8184
|
-
existing.count += 1;
|
|
8185
|
-
return existing.count;
|
|
8186
|
-
}
|
|
8187
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8188
|
-
return 1;
|
|
8189
|
-
}
|
|
8190
8908
|
/** Record one queued give-up toward the never-started bound. */
|
|
8191
8909
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8192
8910
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8498,8 +9216,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8498
9216
|
b2LastDescendantCheckMs: 0,
|
|
8499
9217
|
b2RootOngoingHeldLogged: false,
|
|
8500
9218
|
b2AbandonedSignalled: false,
|
|
9219
|
+
b2AbandonedWithoutAnswer: false,
|
|
8501
9220
|
ambiguousPinnedSinceMs: 0,
|
|
8502
9221
|
ambiguousResolved: false,
|
|
9222
|
+
sessionStoreFailurePending: false,
|
|
8503
9223
|
neverStartedResolved: false
|
|
8504
9224
|
});
|
|
8505
9225
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8521,6 +9241,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8521
9241
|
this.bufferedSessionErrors.delete(oldest);
|
|
8522
9242
|
}
|
|
8523
9243
|
}
|
|
9244
|
+
retainSessionError(event) {
|
|
9245
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9246
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9247
|
+
reason: event.reason,
|
|
9248
|
+
observedAt: this.now()
|
|
9249
|
+
});
|
|
9250
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9251
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9252
|
+
if (typeof oldest !== "string") break;
|
|
9253
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9254
|
+
}
|
|
9255
|
+
}
|
|
9256
|
+
readRetainedSessionError(sessionId) {
|
|
9257
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9258
|
+
if (!retained) return null;
|
|
9259
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9260
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9261
|
+
return null;
|
|
9262
|
+
}
|
|
9263
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9264
|
+
return retained.reason;
|
|
9265
|
+
}
|
|
8524
9266
|
/**
|
|
8525
9267
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8526
9268
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8616,8 +9358,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8616
9358
|
b2LastDescendantCheckMs: 0,
|
|
8617
9359
|
b2RootOngoingHeldLogged: false,
|
|
8618
9360
|
b2AbandonedSignalled: false,
|
|
9361
|
+
b2AbandonedWithoutAnswer: false,
|
|
8619
9362
|
ambiguousPinnedSinceMs: 0,
|
|
8620
9363
|
ambiguousResolved: false,
|
|
9364
|
+
sessionStoreFailurePending: false,
|
|
8621
9365
|
neverStartedResolved: false
|
|
8622
9366
|
});
|
|
8623
9367
|
}
|
|
@@ -8845,9 +9589,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8845
9589
|
return;
|
|
8846
9590
|
}
|
|
8847
9591
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9592
|
+
this.retainSessionError(event);
|
|
8848
9593
|
this.log({
|
|
8849
|
-
level: "
|
|
8850
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9594
|
+
level: "warn",
|
|
9595
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8851
9596
|
conversation_id: watcher.conv.id
|
|
8852
9597
|
});
|
|
8853
9598
|
return;
|
|
@@ -8877,9 +9622,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8877
9622
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8878
9623
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8879
9624
|
if (state !== "queued") {
|
|
9625
|
+
this.retainSessionError(event);
|
|
8880
9626
|
this.log({
|
|
8881
|
-
level: "
|
|
8882
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9627
|
+
level: "warn",
|
|
9628
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8883
9629
|
conversation_id: watcher.conv.id,
|
|
8884
9630
|
message_id: inFlight.evidentMessageId
|
|
8885
9631
|
});
|
|
@@ -9031,6 +9777,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9031
9777
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9032
9778
|
this.neverStartedAttempts.delete(id);
|
|
9033
9779
|
}
|
|
9780
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9781
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9782
|
+
return;
|
|
9783
|
+
}
|
|
9034
9784
|
if (inFlight.neverStartedResolved) {
|
|
9035
9785
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9036
9786
|
inFlight.neverStartedResolved = false;
|
|
@@ -9104,7 +9854,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9104
9854
|
if (state === "failed") {
|
|
9105
9855
|
this.anchorDeliveryDeadline(inFlight);
|
|
9106
9856
|
if (!inFlight.done) {
|
|
9107
|
-
const
|
|
9857
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9858
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9859
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9860
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9861
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9862
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9863
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9864
|
+
(detail) => detail !== null
|
|
9865
|
+
);
|
|
9866
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9108
9867
|
this.log({
|
|
9109
9868
|
level: "error",
|
|
9110
9869
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9190,10 +9949,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9190
9949
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9191
9950
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9192
9951
|
inFlight.b2AbandonedSignalled = false;
|
|
9952
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9193
9953
|
}
|
|
9194
9954
|
} else {
|
|
9195
9955
|
if (inFlight.b2AbandonedSignalled) {
|
|
9196
|
-
|
|
9956
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9957
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9958
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9959
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9960
|
+
sessionId,
|
|
9961
|
+
watcher,
|
|
9962
|
+
inFlight,
|
|
9963
|
+
messages,
|
|
9964
|
+
finish
|
|
9965
|
+
);
|
|
9966
|
+
} else {
|
|
9967
|
+
await this.settleMessageDone(
|
|
9968
|
+
sessionId,
|
|
9969
|
+
watcher,
|
|
9970
|
+
inFlight,
|
|
9971
|
+
messages,
|
|
9972
|
+
composeAbandonmentNote(
|
|
9973
|
+
finish,
|
|
9974
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
9975
|
+
"with-answer"
|
|
9976
|
+
)
|
|
9977
|
+
);
|
|
9978
|
+
}
|
|
9197
9979
|
return;
|
|
9198
9980
|
}
|
|
9199
9981
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9210,22 +9992,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9210
9992
|
descendantOngoing,
|
|
9211
9993
|
rootOngoing
|
|
9212
9994
|
})) {
|
|
9995
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9996
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9997
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9998
|
+
watched_for_ms: pinnedForMs,
|
|
9999
|
+
run_state: "failed",
|
|
10000
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10001
|
+
});
|
|
10002
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10003
|
+
return;
|
|
10004
|
+
}
|
|
9213
10005
|
inFlight.b2AbandonedSignalled = true;
|
|
10006
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10007
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10008
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
10009
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
10010
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10011
|
+
watched_for_ms: pinnedForMs,
|
|
10012
|
+
run_state: "failed",
|
|
10013
|
+
finish,
|
|
10014
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
10015
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
10016
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10017
|
+
});
|
|
10018
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10019
|
+
sessionId,
|
|
10020
|
+
watcher,
|
|
10021
|
+
inFlight,
|
|
10022
|
+
messages,
|
|
10023
|
+
finish
|
|
10024
|
+
);
|
|
10025
|
+
return;
|
|
10026
|
+
}
|
|
9214
10027
|
this.log({
|
|
9215
10028
|
level: "warn",
|
|
9216
10029
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9217
10030
|
conversation_id: conv.id,
|
|
9218
10031
|
message_id: id
|
|
9219
10032
|
});
|
|
9220
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9221
10033
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9222
10034
|
watched_for_ms: pinnedForMs,
|
|
9223
|
-
finish
|
|
10035
|
+
finish,
|
|
9224
10036
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9225
10037
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9226
10038
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9227
10039
|
});
|
|
9228
|
-
await this.settleMessageDone(
|
|
10040
|
+
await this.settleMessageDone(
|
|
10041
|
+
sessionId,
|
|
10042
|
+
watcher,
|
|
10043
|
+
inFlight,
|
|
10044
|
+
messages,
|
|
10045
|
+
composeAbandonmentNote(
|
|
10046
|
+
finish,
|
|
10047
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10048
|
+
"with-answer"
|
|
10049
|
+
)
|
|
10050
|
+
);
|
|
9229
10051
|
return;
|
|
9230
10052
|
}
|
|
9231
10053
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9268,6 +10090,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9268
10090
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9269
10091
|
sessionOngoing: ongoing
|
|
9270
10092
|
})) {
|
|
10093
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10094
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10095
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10096
|
+
watched_for_ms: pinnedForMs,
|
|
10097
|
+
run_state: "failed",
|
|
10098
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10099
|
+
});
|
|
10100
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10101
|
+
return;
|
|
10102
|
+
}
|
|
9271
10103
|
inFlight.ambiguousResolved = true;
|
|
9272
10104
|
this.log({
|
|
9273
10105
|
level: "warn",
|
|
@@ -9415,15 +10247,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9415
10247
|
}
|
|
9416
10248
|
}
|
|
9417
10249
|
/**
|
|
9418
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9419
|
-
*
|
|
9420
|
-
*
|
|
9421
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9422
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9423
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9424
|
-
* and risking the two copies silently drifting apart.
|
|
10250
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10251
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10252
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9425
10253
|
*/
|
|
9426
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10254
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9427
10255
|
const conv = watcher.conv;
|
|
9428
10256
|
this.anchorDeliveryDeadline(inFlight);
|
|
9429
10257
|
if (!inFlight.done) {
|
|
@@ -9450,7 +10278,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9450
10278
|
title,
|
|
9451
10279
|
usage,
|
|
9452
10280
|
usageAgentName,
|
|
9453
|
-
subagentInvocations
|
|
10281
|
+
subagentInvocations,
|
|
10282
|
+
note
|
|
9454
10283
|
);
|
|
9455
10284
|
} catch (err) {
|
|
9456
10285
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9492,6 +10321,78 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9492
10321
|
}
|
|
9493
10322
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9494
10323
|
}
|
|
10324
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10325
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10326
|
+
const conv = watcher.conv;
|
|
10327
|
+
const error2 = composeAbandonmentNote(
|
|
10328
|
+
finish,
|
|
10329
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10330
|
+
"without-answer"
|
|
10331
|
+
);
|
|
10332
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10333
|
+
this.log({
|
|
10334
|
+
level: "warn",
|
|
10335
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10336
|
+
conversation_id: conv.id,
|
|
10337
|
+
message_id: inFlight.evidentMessageId
|
|
10338
|
+
});
|
|
10339
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10340
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10341
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10342
|
+
messages,
|
|
10343
|
+
inFlight.opencodeMessageId,
|
|
10344
|
+
inFlight.evidentMessageId
|
|
10345
|
+
);
|
|
10346
|
+
try {
|
|
10347
|
+
await this.markFailed(
|
|
10348
|
+
conv.id,
|
|
10349
|
+
inFlight.evidentMessageId,
|
|
10350
|
+
sessionId,
|
|
10351
|
+
error2,
|
|
10352
|
+
usage,
|
|
10353
|
+
void 0,
|
|
10354
|
+
usageAgentName,
|
|
10355
|
+
subagentInvocations
|
|
10356
|
+
);
|
|
10357
|
+
} catch (err) {
|
|
10358
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10359
|
+
if (err instanceof ChannelTerminalError) {
|
|
10360
|
+
this.log({
|
|
10361
|
+
level: "warn",
|
|
10362
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10363
|
+
conversation_id: conv.id,
|
|
10364
|
+
message_id: inFlight.evidentMessageId
|
|
10365
|
+
});
|
|
10366
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10367
|
+
return;
|
|
10368
|
+
}
|
|
10369
|
+
if (this.now() >= inFlight.deadline) {
|
|
10370
|
+
this.log({
|
|
10371
|
+
level: "warn",
|
|
10372
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10373
|
+
conversation_id: conv.id,
|
|
10374
|
+
message_id: inFlight.evidentMessageId
|
|
10375
|
+
});
|
|
10376
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10377
|
+
return;
|
|
10378
|
+
}
|
|
10379
|
+
this.log({
|
|
10380
|
+
level: "warn",
|
|
10381
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10382
|
+
conversation_id: conv.id,
|
|
10383
|
+
message_id: inFlight.evidentMessageId
|
|
10384
|
+
});
|
|
10385
|
+
return;
|
|
10386
|
+
}
|
|
10387
|
+
inFlight.done = true;
|
|
10388
|
+
await this.reportSubagentAuthFailures(
|
|
10389
|
+
watcher.conv.id,
|
|
10390
|
+
inFlight.opencodeMessageId,
|
|
10391
|
+
inFlight.evidentMessageId,
|
|
10392
|
+
messages
|
|
10393
|
+
);
|
|
10394
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10395
|
+
}
|
|
9495
10396
|
/**
|
|
9496
10397
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9497
10398
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9547,6 +10448,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9547
10448
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9548
10449
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9549
10450
|
}
|
|
10451
|
+
/**
|
|
10452
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10453
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10454
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10455
|
+
*/
|
|
10456
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10457
|
+
const conv = watcher.conv;
|
|
10458
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10459
|
+
this.log({
|
|
10460
|
+
level: "error",
|
|
10461
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10462
|
+
conversation_id: conv.id,
|
|
10463
|
+
message_id: inFlight.evidentMessageId
|
|
10464
|
+
});
|
|
10465
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10466
|
+
try {
|
|
10467
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10468
|
+
} catch (err) {
|
|
10469
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10470
|
+
if (err instanceof ChannelTerminalError) {
|
|
10471
|
+
this.log({
|
|
10472
|
+
level: "warn",
|
|
10473
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10474
|
+
conversation_id: conv.id,
|
|
10475
|
+
message_id: inFlight.evidentMessageId
|
|
10476
|
+
});
|
|
10477
|
+
this.recordReleasedOpencodeId(
|
|
10478
|
+
inFlight.evidentMessageId,
|
|
10479
|
+
sessionId,
|
|
10480
|
+
inFlight.opencodeMessageId
|
|
10481
|
+
);
|
|
10482
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10483
|
+
return;
|
|
10484
|
+
}
|
|
10485
|
+
if (this.now() >= inFlight.deadline) {
|
|
10486
|
+
this.log({
|
|
10487
|
+
level: "warn",
|
|
10488
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10489
|
+
conversation_id: conv.id,
|
|
10490
|
+
message_id: inFlight.evidentMessageId
|
|
10491
|
+
});
|
|
10492
|
+
this.recordReleasedOpencodeId(
|
|
10493
|
+
inFlight.evidentMessageId,
|
|
10494
|
+
sessionId,
|
|
10495
|
+
inFlight.opencodeMessageId
|
|
10496
|
+
);
|
|
10497
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10498
|
+
return;
|
|
10499
|
+
}
|
|
10500
|
+
this.log({
|
|
10501
|
+
level: "warn",
|
|
10502
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10503
|
+
conversation_id: conv.id,
|
|
10504
|
+
message_id: inFlight.evidentMessageId
|
|
10505
|
+
});
|
|
10506
|
+
return;
|
|
10507
|
+
}
|
|
10508
|
+
inFlight.done = true;
|
|
10509
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10510
|
+
}
|
|
9550
10511
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9551
10512
|
/**
|
|
9552
10513
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9585,11 +10546,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9585
10546
|
}
|
|
9586
10547
|
}
|
|
9587
10548
|
}
|
|
9588
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10549
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9589
10550
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10551
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10552
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10553
|
+
this.awaitingCorrelation.delete(id);
|
|
10554
|
+
this.log({
|
|
10555
|
+
level: "debug",
|
|
10556
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10557
|
+
message_id: id
|
|
10558
|
+
});
|
|
10559
|
+
}
|
|
10560
|
+
}
|
|
9590
10561
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9591
10562
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9592
10563
|
this.untrackableAck.delete(id);
|
|
10564
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9593
10565
|
this.untrackableAckSignalled.delete(id);
|
|
9594
10566
|
this.untrackableAckWarned.delete(id);
|
|
9595
10567
|
this.log({
|
|
@@ -9629,7 +10601,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9629
10601
|
bySession.set(row.opencode_session_id, list);
|
|
9630
10602
|
}
|
|
9631
10603
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9632
|
-
let messages;
|
|
10604
|
+
let messages = null;
|
|
9633
10605
|
try {
|
|
9634
10606
|
if (this.isV2) {
|
|
9635
10607
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9638,9 +10610,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9638
10610
|
level: "warn",
|
|
9639
10611
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9640
10612
|
});
|
|
9641
|
-
|
|
10613
|
+
} else {
|
|
10614
|
+
messages = snapshot;
|
|
9642
10615
|
}
|
|
9643
|
-
messages = snapshot;
|
|
9644
10616
|
} else {
|
|
9645
10617
|
const polled = await pollSessionMessagesForRedrive(
|
|
9646
10618
|
this.port,
|
|
@@ -9653,15 +10625,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9653
10625
|
level: "warn",
|
|
9654
10626
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9655
10627
|
});
|
|
9656
|
-
|
|
10628
|
+
} else {
|
|
10629
|
+
messages = polled.messages;
|
|
9657
10630
|
}
|
|
9658
|
-
messages = polled.messages;
|
|
9659
10631
|
}
|
|
9660
10632
|
} catch (err) {
|
|
9661
10633
|
this.log({
|
|
9662
10634
|
level: "warn",
|
|
9663
10635
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9664
10636
|
});
|
|
10637
|
+
}
|
|
10638
|
+
if (messages === null) {
|
|
10639
|
+
for (const row of sessionRows) {
|
|
10640
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10641
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10642
|
+
}
|
|
10643
|
+
}
|
|
9665
10644
|
continue;
|
|
9666
10645
|
}
|
|
9667
10646
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9671,6 +10650,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9671
10650
|
}
|
|
9672
10651
|
}
|
|
9673
10652
|
}
|
|
10653
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10654
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10655
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10656
|
+
if (!correlation) return false;
|
|
10657
|
+
correlation.attempts += 1;
|
|
10658
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10659
|
+
if (correlatedId !== null) {
|
|
10660
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10661
|
+
const conv = this.convForRow(sessionId, row);
|
|
10662
|
+
const message = this.queuedMessageForRow(row);
|
|
10663
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10664
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10665
|
+
this.dispatched.add(row.id);
|
|
10666
|
+
this.readopted.add(row.id);
|
|
10667
|
+
this.ensureWatcherRunning(sessionId);
|
|
10668
|
+
this.log({
|
|
10669
|
+
level: "info",
|
|
10670
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10671
|
+
conversation_id: row.conversation_id,
|
|
10672
|
+
message_id: row.id
|
|
10673
|
+
});
|
|
10674
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10675
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10676
|
+
opencode_message_id: correlatedId
|
|
10677
|
+
});
|
|
10678
|
+
return true;
|
|
10679
|
+
}
|
|
10680
|
+
if (correlation.knownUserIds === null) {
|
|
10681
|
+
this.log({
|
|
10682
|
+
level: "debug",
|
|
10683
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10684
|
+
conversation_id: row.conversation_id,
|
|
10685
|
+
message_id: row.id
|
|
10686
|
+
});
|
|
10687
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10688
|
+
this.log({
|
|
10689
|
+
level: "warn",
|
|
10690
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10691
|
+
conversation_id: row.conversation_id,
|
|
10692
|
+
message_id: row.id
|
|
10693
|
+
});
|
|
10694
|
+
}
|
|
10695
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10696
|
+
await this.handoffUntrackableCorrelation(
|
|
10697
|
+
row.conversation_id,
|
|
10698
|
+
row.id,
|
|
10699
|
+
correlation.sessionId,
|
|
10700
|
+
correlation.attempts,
|
|
10701
|
+
"readopt"
|
|
10702
|
+
);
|
|
10703
|
+
}
|
|
10704
|
+
return true;
|
|
10705
|
+
}
|
|
9674
10706
|
/**
|
|
9675
10707
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9676
10708
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9678,8 +10710,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9678
10710
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9679
10711
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9680
10712
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9681
|
-
* before the restart)
|
|
9682
|
-
*
|
|
10713
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10714
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10715
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10716
|
+
* `forceReadoptRun`):
|
|
9683
10717
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9684
10718
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9685
10719
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9688,7 +10722,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9688
10722
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9689
10723
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9690
10724
|
* tracking the stored id so the reply correlates by it;
|
|
9691
|
-
* - `unknown`/null id →
|
|
10725
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10726
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9692
10727
|
*
|
|
9693
10728
|
* Only `ChannelAuthError` propagates.
|
|
9694
10729
|
*/
|
|
@@ -9702,6 +10737,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9702
10737
|
});
|
|
9703
10738
|
return;
|
|
9704
10739
|
}
|
|
10740
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9705
10741
|
const ocId = row.opencode_message_id;
|
|
9706
10742
|
const state = messageRunState(messages, ocId ?? "");
|
|
9707
10743
|
if (state === "done") {
|
|
@@ -9768,12 +10804,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9768
10804
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9769
10805
|
this.dontRedispatch.delete(row.id);
|
|
9770
10806
|
this.untrackableAck.delete(row.id);
|
|
10807
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9771
10808
|
this.untrackableAckSignalled.delete(row.id);
|
|
9772
10809
|
this.untrackableAckWarned.delete(row.id);
|
|
9773
10810
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9774
10811
|
return;
|
|
9775
10812
|
}
|
|
9776
|
-
if (this.
|
|
10813
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10814
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10815
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10816
|
+
this.untrackableAckWarned.add(row.id);
|
|
10817
|
+
this.log({
|
|
10818
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10819
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10820
|
+
conversation_id: row.conversation_id,
|
|
10821
|
+
message_id: row.id
|
|
10822
|
+
});
|
|
10823
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10824
|
+
const reportError = await this.reportUntrackableAck(
|
|
10825
|
+
row.conversation_id,
|
|
10826
|
+
row.id,
|
|
10827
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10828
|
+
errorMessage3
|
|
10829
|
+
);
|
|
10830
|
+
if (reportError !== null) {
|
|
10831
|
+
this.log({
|
|
10832
|
+
level: "warn",
|
|
10833
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10834
|
+
conversation_id: row.conversation_id,
|
|
10835
|
+
message_id: row.id
|
|
10836
|
+
});
|
|
10837
|
+
}
|
|
10838
|
+
} else {
|
|
10839
|
+
this.log({
|
|
10840
|
+
level: "debug",
|
|
10841
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10842
|
+
conversation_id: row.conversation_id,
|
|
10843
|
+
message_id: row.id
|
|
10844
|
+
});
|
|
10845
|
+
}
|
|
10846
|
+
return;
|
|
10847
|
+
}
|
|
10848
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9777
10849
|
this.log({
|
|
9778
10850
|
level: "debug",
|
|
9779
10851
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9951,6 +11023,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9951
11023
|
}
|
|
9952
11024
|
this.dontRedispatch.delete(row.id);
|
|
9953
11025
|
this.untrackableAck.delete(row.id);
|
|
11026
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9954
11027
|
this.untrackableAckSignalled.delete(row.id);
|
|
9955
11028
|
this.untrackableAckWarned.delete(row.id);
|
|
9956
11029
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9970,8 +11043,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9970
11043
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9971
11044
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9972
11045
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9973
|
-
* re-entry)
|
|
9974
|
-
*
|
|
11046
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11047
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11048
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9975
11049
|
*
|
|
9976
11050
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9977
11051
|
* `processed_at` (Invariant 1).
|
|
@@ -10034,6 +11108,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10034
11108
|
);
|
|
10035
11109
|
}
|
|
10036
11110
|
let ocId;
|
|
11111
|
+
let preDispatchUserIds = null;
|
|
10037
11112
|
try {
|
|
10038
11113
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10039
11114
|
sessionId,
|
|
@@ -10043,7 +11118,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10043
11118
|
row.content,
|
|
10044
11119
|
options,
|
|
10045
11120
|
sendAttachments,
|
|
10046
|
-
this.openCodeClient
|
|
11121
|
+
this.openCodeClient,
|
|
11122
|
+
{
|
|
11123
|
+
onKnownUserIds: (ids) => {
|
|
11124
|
+
preDispatchUserIds = ids;
|
|
11125
|
+
},
|
|
11126
|
+
onKnownUserIdsError: (error2) => {
|
|
11127
|
+
this.log({
|
|
11128
|
+
level: "warn",
|
|
11129
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11130
|
+
conversation_id: row.conversation_id,
|
|
11131
|
+
message_id: row.id
|
|
11132
|
+
});
|
|
11133
|
+
}
|
|
11134
|
+
}
|
|
10047
11135
|
)
|
|
10048
11136
|
);
|
|
10049
11137
|
} catch (err) {
|
|
@@ -10076,6 +11164,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10076
11164
|
}
|
|
10077
11165
|
} else {
|
|
10078
11166
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11167
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10079
11168
|
}
|
|
10080
11169
|
this.log({
|
|
10081
11170
|
level: "error",
|
|
@@ -10124,39 +11213,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10124
11213
|
}
|
|
10125
11214
|
if (ocId === null) {
|
|
10126
11215
|
this.awaitingReadopt.delete(row.id);
|
|
10127
|
-
|
|
10128
|
-
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
|
|
10132
|
-
|
|
10133
|
-
this.log({
|
|
10134
|
-
level: "error",
|
|
10135
|
-
message: errorMessage3,
|
|
10136
|
-
conversation_id: row.conversation_id,
|
|
10137
|
-
message_id: row.id
|
|
10138
|
-
});
|
|
10139
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10140
|
-
this.log({
|
|
10141
|
-
level: "warn",
|
|
10142
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10143
|
-
conversation_id: row.conversation_id,
|
|
10144
|
-
message_id: row.id
|
|
10145
|
-
});
|
|
10146
|
-
});
|
|
10147
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10148
|
-
return;
|
|
10149
|
-
}
|
|
11216
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11217
|
+
sessionId,
|
|
11218
|
+
content: row.content,
|
|
11219
|
+
attempts: 0,
|
|
11220
|
+
knownUserIds: preDispatchUserIds
|
|
11221
|
+
});
|
|
10150
11222
|
this.log({
|
|
10151
11223
|
level: "warn",
|
|
10152
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11224
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10153
11225
|
conversation_id: row.conversation_id,
|
|
10154
11226
|
message_id: row.id
|
|
10155
11227
|
});
|
|
10156
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10157
11228
|
return;
|
|
10158
11229
|
}
|
|
10159
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10160
11230
|
const processedAtMs = this.processedAtMs(row);
|
|
10161
11231
|
this.registerReadopted(
|
|
10162
11232
|
readoptConv,
|
|
@@ -11062,7 +12132,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11062
12132
|
* watcher retries next tick within the
|
|
11063
12133
|
* deadline, Finding 4).
|
|
11064
12134
|
*/
|
|
11065
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12135
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11066
12136
|
const res = await this.fetchImpl(
|
|
11067
12137
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11068
12138
|
{
|
|
@@ -11080,7 +12150,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11080
12150
|
...title ? { title } : {},
|
|
11081
12151
|
...usage ? usage : {},
|
|
11082
12152
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11083
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12153
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12154
|
+
...error2 ? { error: error2 } : {}
|
|
11084
12155
|
})
|
|
11085
12156
|
}
|
|
11086
12157
|
);
|
|
@@ -11808,7 +12879,9 @@ async function installRunnerSecret({
|
|
|
11808
12879
|
}) {
|
|
11809
12880
|
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
11810
12881
|
if (!arn) {
|
|
11811
|
-
|
|
12882
|
+
if (!env.GH_TOKEN) {
|
|
12883
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
12884
|
+
}
|
|
11812
12885
|
return false;
|
|
11813
12886
|
}
|
|
11814
12887
|
const result = await (commandRunner ?? runCommand2)(
|
|
@@ -12751,9 +13824,12 @@ async function driveChannels(state, driver) {
|
|
|
12751
13824
|
}
|
|
12752
13825
|
}
|
|
12753
13826
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12754
|
-
var
|
|
13827
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
13828
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
13829
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
13830
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
12755
13831
|
function shouldWarnForReclaimSkip(reason) {
|
|
12756
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13832
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12757
13833
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12758
13834
|
if (!version2) return false;
|
|
12759
13835
|
const major = Number(version2[1]);
|
|
@@ -12810,61 +13886,146 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12810
13886
|
async function runSweep(state, driver, config) {
|
|
12811
13887
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12812
13888
|
try {
|
|
12813
|
-
const
|
|
12814
|
-
|
|
12815
|
-
|
|
13889
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13890
|
+
const sessions = enumeration.sessions;
|
|
13891
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13892
|
+
if (enumerationReason !== null) {
|
|
13893
|
+
logActivity(state, {
|
|
12816
13894
|
type: "info",
|
|
12817
|
-
|
|
12818
|
-
|
|
12819
|
-
outcome: "sweep-skipped",
|
|
12820
|
-
reason: "opencode-unreachable"
|
|
12821
|
-
}
|
|
13895
|
+
level: "warn",
|
|
13896
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12822
13897
|
});
|
|
12823
|
-
return;
|
|
12824
13898
|
}
|
|
12825
|
-
const toDelete = selectSessionsToDelete(
|
|
12826
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12827
|
-
{
|
|
12828
|
-
maxAgeMs: config.maxAgeMs,
|
|
12829
|
-
maxCount: config.maxCount,
|
|
12830
|
-
nowMs: Date.now(),
|
|
12831
|
-
protectedIds: driver.protectedSessionIds()
|
|
12832
|
-
}
|
|
12833
|
-
);
|
|
12834
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12835
13899
|
let deleted = 0;
|
|
12836
13900
|
let failed = 0;
|
|
13901
|
+
let failed4xx = 0;
|
|
13902
|
+
let failed5xx = 0;
|
|
13903
|
+
let failedOtherStatus = 0;
|
|
13904
|
+
let failedThrew = 0;
|
|
12837
13905
|
let skippedNewlyActive = 0;
|
|
12838
|
-
|
|
12839
|
-
|
|
12840
|
-
|
|
12841
|
-
|
|
13906
|
+
let remaining = 0;
|
|
13907
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13908
|
+
if (sessions !== null) {
|
|
13909
|
+
const toDelete = selectSessionsToDelete(
|
|
13910
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13911
|
+
{
|
|
13912
|
+
maxAgeMs: config.maxAgeMs,
|
|
13913
|
+
maxCount: config.maxCount,
|
|
13914
|
+
nowMs: Date.now(),
|
|
13915
|
+
protectedIds: protectedNow
|
|
13916
|
+
}
|
|
13917
|
+
);
|
|
13918
|
+
const eligibleToDelete = toDelete.filter(
|
|
13919
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
13920
|
+
);
|
|
13921
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13922
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
13923
|
+
const answeredFailures = [];
|
|
13924
|
+
protectedNow = driver.protectedSessionIds();
|
|
13925
|
+
for (const id of deleteCandidates) {
|
|
13926
|
+
if (protectedNow.has(id)) {
|
|
13927
|
+
skippedNewlyActive++;
|
|
13928
|
+
logActivity(state, {
|
|
13929
|
+
type: "info",
|
|
13930
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13931
|
+
});
|
|
13932
|
+
continue;
|
|
13933
|
+
}
|
|
13934
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13935
|
+
if (outcome.ok) {
|
|
13936
|
+
deleted++;
|
|
13937
|
+
} else if (outcome.status === "threw") {
|
|
13938
|
+
failed++;
|
|
13939
|
+
failedThrew++;
|
|
13940
|
+
} else {
|
|
13941
|
+
failed++;
|
|
13942
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13943
|
+
if (statusClass === 4) failed4xx++;
|
|
13944
|
+
else if (statusClass === 5) failed5xx++;
|
|
13945
|
+
else failedOtherStatus++;
|
|
13946
|
+
answeredFailures.push({
|
|
13947
|
+
id,
|
|
13948
|
+
status: outcome.status,
|
|
13949
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
13950
|
+
});
|
|
13951
|
+
}
|
|
13952
|
+
}
|
|
13953
|
+
const givenUpFailures = [];
|
|
13954
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
13955
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
13956
|
+
} else {
|
|
13957
|
+
for (const failure of answeredFailures) {
|
|
13958
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
13959
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
13960
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
13961
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
13962
|
+
givenUpFailures.push(failure);
|
|
13963
|
+
}
|
|
13964
|
+
}
|
|
13965
|
+
}
|
|
13966
|
+
for (const failure of givenUpFailures) {
|
|
13967
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
13968
|
+
0,
|
|
13969
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
13970
|
+
);
|
|
13971
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12842
13972
|
type: "info",
|
|
12843
|
-
|
|
13973
|
+
level: "warn",
|
|
13974
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
13975
|
+
metadata: {
|
|
13976
|
+
outcome: "session-delete-given-up",
|
|
13977
|
+
session_id: failure.id,
|
|
13978
|
+
status: failure.status,
|
|
13979
|
+
body_excerpt: excerpt,
|
|
13980
|
+
given_up_count: givenUpFailures.length
|
|
13981
|
+
}
|
|
12844
13982
|
});
|
|
12845
|
-
continue;
|
|
12846
13983
|
}
|
|
12847
|
-
|
|
12848
|
-
|
|
13984
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13985
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13986
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13987
|
+
logActivity(state, {
|
|
13988
|
+
type: "info",
|
|
13989
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13990
|
+
});
|
|
12849
13991
|
}
|
|
12850
|
-
const
|
|
12851
|
-
|
|
12852
|
-
|
|
12853
|
-
|
|
12854
|
-
|
|
12855
|
-
|
|
13992
|
+
const sessionMetadata = {
|
|
13993
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13994
|
+
sessions_deleted: deleted,
|
|
13995
|
+
sessions_failed: failed,
|
|
13996
|
+
sessions_failed_4xx: failed4xx,
|
|
13997
|
+
sessions_failed_5xx: failed5xx,
|
|
13998
|
+
sessions_failed_other: failedOtherStatus,
|
|
13999
|
+
sessions_failed_threw: failedThrew,
|
|
14000
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
14001
|
+
sessions_remaining: remaining,
|
|
14002
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
14003
|
+
};
|
|
14004
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
12856
14005
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12857
14006
|
dbPath: sessionDbPath(),
|
|
12858
|
-
|
|
12859
|
-
|
|
14007
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
14008
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
14009
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
14010
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14011
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
12860
14012
|
});
|
|
12861
14013
|
if (reclaimResult.ok) {
|
|
14014
|
+
state.sessionStoreHealth.observeWritable();
|
|
12862
14015
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12863
14016
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12864
14017
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
14018
|
+
const loopElapsedMs = Math.max(
|
|
14019
|
+
0,
|
|
14020
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14021
|
+
);
|
|
14022
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14023
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14024
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
12865
14025
|
logSessionCleanupActivity(state, "sweep", {
|
|
12866
14026
|
type: "info",
|
|
12867
|
-
|
|
14027
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14028
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
12868
14029
|
metadata: {
|
|
12869
14030
|
outcome: "reclaimed",
|
|
12870
14031
|
mode: reclaimResult.mode,
|
|
@@ -12872,25 +14033,28 @@ async function runSweep(state, driver, config) {
|
|
|
12872
14033
|
after_bytes: reclaimResult.afterBytes,
|
|
12873
14034
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12874
14035
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12875
|
-
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
14036
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
14037
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
14038
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
14039
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
14040
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14041
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
14042
|
+
...sessionMetadata
|
|
12879
14043
|
}
|
|
12880
14044
|
});
|
|
12881
14045
|
} else {
|
|
14046
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
14047
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
14048
|
+
}
|
|
12882
14049
|
logSessionCleanupActivity(state, "sweep", {
|
|
12883
14050
|
type: "info",
|
|
12884
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
14051
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12885
14052
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12886
14053
|
metadata: {
|
|
12887
14054
|
outcome: "reclaim-skipped",
|
|
12888
14055
|
reason: reclaimResult.skipped,
|
|
12889
14056
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12890
|
-
|
|
12891
|
-
sessions_deleted: deleted,
|
|
12892
|
-
sessions_failed: failed,
|
|
12893
|
-
sessions_skipped_active: skippedNewlyActive
|
|
14057
|
+
...sessionMetadata
|
|
12894
14058
|
}
|
|
12895
14059
|
});
|
|
12896
14060
|
}
|
|
@@ -13147,6 +14311,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13147
14311
|
message: `Resource usage collection: ${warning2}`
|
|
13148
14312
|
});
|
|
13149
14313
|
}
|
|
14314
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14315
|
+
...usage,
|
|
14316
|
+
nowMs: Date.now(),
|
|
14317
|
+
state: state.lowDiskWarningState
|
|
14318
|
+
});
|
|
14319
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14320
|
+
if (lowDiskDecision.warning !== null) {
|
|
14321
|
+
logActivity(state, {
|
|
14322
|
+
type: "info",
|
|
14323
|
+
level: "warn",
|
|
14324
|
+
message: lowDiskDecision.warning
|
|
14325
|
+
});
|
|
14326
|
+
}
|
|
13150
14327
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13151
14328
|
if (result.ok) {
|
|
13152
14329
|
if (consecutiveFailures > 0) {
|
|
@@ -13393,6 +14570,7 @@ async function run(options) {
|
|
|
13393
14570
|
opencodeClient: null,
|
|
13394
14571
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13395
14572
|
sessionDbProvenanceAnomaly: false,
|
|
14573
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13396
14574
|
opencodeProcess: null,
|
|
13397
14575
|
stopOpenCodeLogTail: null,
|
|
13398
14576
|
litestreamProcess: null,
|
|
@@ -13404,12 +14582,14 @@ async function run(options) {
|
|
|
13404
14582
|
messageCount: 0,
|
|
13405
14583
|
lastProxiedActivityAt: null,
|
|
13406
14584
|
sessionCleanupTimers: [],
|
|
14585
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13407
14586
|
claudeUsageTimer: null,
|
|
13408
14587
|
claudeUsageRearm: null,
|
|
13409
14588
|
openaiUsageTimer: null,
|
|
13410
14589
|
openaiUsageRearm: null,
|
|
13411
14590
|
resourceUsageTimer: null,
|
|
13412
14591
|
stopResourceUsageSampling: null,
|
|
14592
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13413
14593
|
credentialSync: null,
|
|
13414
14594
|
authHeader: ""
|
|
13415
14595
|
};
|
|
@@ -13583,27 +14763,62 @@ async function run(options) {
|
|
|
13583
14763
|
} else {
|
|
13584
14764
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13585
14765
|
}
|
|
13586
|
-
if (options.restoreRunnerCredentials) {
|
|
13587
|
-
|
|
13588
|
-
|
|
13589
|
-
|
|
13590
|
-
|
|
13591
|
-
if (level === "error") {
|
|
13592
|
-
logActivity(state, { type: "error", error: message });
|
|
13593
|
-
} else {
|
|
13594
|
-
logActivity(state, { type: "info", level, message });
|
|
13595
|
-
}
|
|
14766
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14767
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14768
|
+
log2(state, message, level);
|
|
14769
|
+
if (state.interactive && !state.json) {
|
|
14770
|
+
logActivity(state, { type: "info", level, message });
|
|
13596
14771
|
}
|
|
13597
14772
|
};
|
|
13598
|
-
const
|
|
14773
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14774
|
+
if (vaultResolution.error) {
|
|
14775
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14776
|
+
logCredentialVaultResolution(
|
|
14777
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14778
|
+
"warn"
|
|
14779
|
+
);
|
|
14780
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14781
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14782
|
+
logCredentialVaultResolution(
|
|
14783
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14784
|
+
"info"
|
|
14785
|
+
);
|
|
14786
|
+
} else {
|
|
14787
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14788
|
+
logCredentialVaultResolution(
|
|
14789
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14790
|
+
"debug"
|
|
14791
|
+
);
|
|
14792
|
+
}
|
|
14793
|
+
}
|
|
14794
|
+
const credentialContext = {
|
|
14795
|
+
env: process.env,
|
|
14796
|
+
log: (message, level = "info") => {
|
|
14797
|
+
if (level === "error") {
|
|
14798
|
+
logActivity(state, { type: "error", error: message });
|
|
14799
|
+
} else {
|
|
14800
|
+
logActivity(state, { type: "info", level, message });
|
|
14801
|
+
}
|
|
14802
|
+
}
|
|
14803
|
+
};
|
|
14804
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
14805
|
+
if (options.restoreRunnerCredentials) {
|
|
13599
14806
|
await restoreCredentialStores(credentialContext);
|
|
13600
|
-
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13601
14807
|
}
|
|
14808
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13602
14809
|
state.credentialSync?.arm();
|
|
13603
|
-
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13604
|
-
|
|
13605
|
-
|
|
13606
|
-
|
|
14810
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14811
|
+
() => ({
|
|
14812
|
+
agentId: state.agentId,
|
|
14813
|
+
authHeader: state.authHeader
|
|
14814
|
+
}),
|
|
14815
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14816
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14817
|
+
if (cause !== null && sessionId !== null) {
|
|
14818
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14819
|
+
}
|
|
14820
|
+
}
|
|
14821
|
+
);
|
|
13607
14822
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13608
14823
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13609
14824
|
openCodeActivityForwarder
|
|
@@ -13613,22 +14828,27 @@ async function run(options) {
|
|
|
13613
14828
|
openCodeLogTail.stop();
|
|
13614
14829
|
};
|
|
13615
14830
|
let sessionDbVerifyFatal = false;
|
|
14831
|
+
let sessionDbVerifyFatalReason;
|
|
13616
14832
|
if (!options.restoreSessionDb) {
|
|
13617
|
-
|
|
14833
|
+
logActivity(state, {
|
|
14834
|
+
type: "info",
|
|
14835
|
+
level: "warn",
|
|
14836
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14837
|
+
});
|
|
13618
14838
|
} else {
|
|
13619
14839
|
const health = await checkOpenCodeHealth(state.port);
|
|
13620
14840
|
if (health.healthy) {
|
|
13621
|
-
|
|
13622
|
-
|
|
13623
|
-
|
|
13624
|
-
"
|
|
13625
|
-
);
|
|
14841
|
+
logActivity(state, {
|
|
14842
|
+
type: "info",
|
|
14843
|
+
level: "warn",
|
|
14844
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14845
|
+
});
|
|
13626
14846
|
} else if (health.authFailed) {
|
|
13627
|
-
|
|
13628
|
-
|
|
13629
|
-
|
|
13630
|
-
"
|
|
13631
|
-
);
|
|
14847
|
+
logActivity(state, {
|
|
14848
|
+
type: "info",
|
|
14849
|
+
level: "warn",
|
|
14850
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14851
|
+
});
|
|
13632
14852
|
} else {
|
|
13633
14853
|
const result = await restoreAndVerifySessionDb({
|
|
13634
14854
|
dbPath: sessionDbPath(),
|
|
@@ -13645,12 +14865,13 @@ async function run(options) {
|
|
|
13645
14865
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13646
14866
|
});
|
|
13647
14867
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14868
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13648
14869
|
}
|
|
13649
14870
|
}
|
|
13650
14871
|
reportSessionDbRecovery(state);
|
|
13651
14872
|
if (sessionDbVerifyFatal) {
|
|
13652
14873
|
throw new Error(
|
|
13653
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14874
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13654
14875
|
);
|
|
13655
14876
|
}
|
|
13656
14877
|
applyRunnerOpenCodeConfig({
|
|
@@ -13895,6 +15116,7 @@ async function run(options) {
|
|
|
13895
15116
|
getAuthHeader: () => state.authHeader,
|
|
13896
15117
|
conversationFilter: state.conversationFilter,
|
|
13897
15118
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15119
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13898
15120
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13899
15121
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13900
15122
|
fileSyncDirectories,
|
|
@@ -13976,14 +15198,14 @@ async function run(options) {
|
|
|
13976
15198
|
},
|
|
13977
15199
|
onDisconnected: (code, reason, transition) => {
|
|
13978
15200
|
state.connected = false;
|
|
13979
|
-
const
|
|
15201
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13980
15202
|
logActivity(state, {
|
|
13981
15203
|
type: "info",
|
|
13982
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15204
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
13983
15205
|
});
|
|
13984
15206
|
emitAgentDisconnected(state.agentId, {
|
|
13985
15207
|
code,
|
|
13986
|
-
reason:
|
|
15208
|
+
reason: safeReason2,
|
|
13987
15209
|
connection_generation: transition.generation
|
|
13988
15210
|
});
|
|
13989
15211
|
if (state.interactive) displayStatus(state);
|
|
@@ -14146,6 +15368,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14146
15368
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14147
15369
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14148
15370
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15371
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14149
15372
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14150
15373
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14151
15374
|
"-a, --agent [id]",
|