@evident-ai/cli 3.5.2-dev.3431f9f → 3.5.2-dev.35de651
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/index.js +1696 -505
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -703,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
703
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
704
704
|
});
|
|
705
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
706
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
707
732
|
const apiUrl = getApiUrlConfig();
|
|
708
733
|
try {
|
|
@@ -877,6 +902,76 @@ async function status(options = {}) {
|
|
|
877
902
|
process.exit(result.exitCode);
|
|
878
903
|
}
|
|
879
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// src/lib/claude-usage.ts
|
|
881
976
|
import { execFileSync } from "node:child_process";
|
|
882
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1069,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1069
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1070
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1071
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1072
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1073
1238
|
var TelemetryEventTypes = {
|
|
1074
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1157,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1157
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1158
1323
|
|
|
1159
1324
|
// src/lib/telemetry.ts
|
|
1160
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.35de651" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1161
1326
|
function getCliVersion() {
|
|
1162
1327
|
return CLI_VERSION;
|
|
1163
1328
|
}
|
|
@@ -1318,8 +1483,334 @@ var EventTypes = {
|
|
|
1318
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1319
1484
|
};
|
|
1320
1485
|
|
|
1321
|
-
// src/lib/
|
|
1486
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1487
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1488
|
+
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1322
1491
|
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1519
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1520
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1522
|
+
function errorMessage(error2) {
|
|
1523
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1524
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1525
|
+
}
|
|
1526
|
+
function errorMessages(error2) {
|
|
1527
|
+
const messages = [];
|
|
1528
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1529
|
+
let current = error2;
|
|
1530
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1531
|
+
visited.add(current);
|
|
1532
|
+
if (typeof current === "string") {
|
|
1533
|
+
messages.push(current);
|
|
1534
|
+
break;
|
|
1535
|
+
}
|
|
1536
|
+
if (current instanceof Error) {
|
|
1537
|
+
messages.push(current.name, current.message);
|
|
1538
|
+
current = current.cause;
|
|
1539
|
+
continue;
|
|
1540
|
+
}
|
|
1541
|
+
if (typeof current === "object") {
|
|
1542
|
+
const record = current;
|
|
1543
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1544
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1545
|
+
current = record.cause;
|
|
1546
|
+
continue;
|
|
1547
|
+
}
|
|
1548
|
+
break;
|
|
1549
|
+
}
|
|
1550
|
+
return messages;
|
|
1551
|
+
}
|
|
1552
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1553
|
+
return errorMessages(error2).some(
|
|
1554
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1555
|
+
message
|
|
1556
|
+
)
|
|
1557
|
+
);
|
|
1558
|
+
}
|
|
1559
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1560
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1561
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1562
|
+
function isCauseBearingText(text) {
|
|
1563
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1564
|
+
}
|
|
1565
|
+
function structuredLogFields(line) {
|
|
1566
|
+
const fields = [];
|
|
1567
|
+
for (let index = 0; index < line.length; index++) {
|
|
1568
|
+
if (line[index] === '"') {
|
|
1569
|
+
index++;
|
|
1570
|
+
while (index < line.length) {
|
|
1571
|
+
if (line[index] === "\\") {
|
|
1572
|
+
index++;
|
|
1573
|
+
} else if (line[index] === '"') {
|
|
1574
|
+
break;
|
|
1575
|
+
}
|
|
1576
|
+
index++;
|
|
1577
|
+
}
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1581
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1582
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1583
|
+
const valueStart = index + key.length;
|
|
1584
|
+
if (line[valueStart] !== '"') {
|
|
1585
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1586
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1587
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
let valueEnd = valueStart + 1;
|
|
1591
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1592
|
+
if (line[valueEnd] === "\\") {
|
|
1593
|
+
valueEnd++;
|
|
1594
|
+
} else if (line[valueEnd] === '"') {
|
|
1595
|
+
break;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1599
|
+
index = valueEnd;
|
|
1600
|
+
}
|
|
1601
|
+
return fields;
|
|
1602
|
+
}
|
|
1603
|
+
function structuredCauseMessages(text) {
|
|
1604
|
+
const fields = structuredLogFields(text);
|
|
1605
|
+
if (fields.length > 0) return fields;
|
|
1606
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1607
|
+
}
|
|
1608
|
+
function causeTokenFromText(text) {
|
|
1609
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1610
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1611
|
+
const match = pattern.exec(text);
|
|
1612
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1613
|
+
return match[1];
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
return null;
|
|
1617
|
+
}
|
|
1618
|
+
function sessionStoreCauseToken(value) {
|
|
1619
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1620
|
+
const token = causeTokenFromText(message);
|
|
1621
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1622
|
+
}
|
|
1623
|
+
return null;
|
|
1624
|
+
}
|
|
1625
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1626
|
+
try {
|
|
1627
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1628
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1629
|
+
if (availableBytes < requiredBytes) {
|
|
1630
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1631
|
+
}
|
|
1632
|
+
return null;
|
|
1633
|
+
} catch (err) {
|
|
1634
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
function readLogicalBytes(db) {
|
|
1638
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1639
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1640
|
+
return pageCount * pageSize;
|
|
1641
|
+
}
|
|
1642
|
+
function readFreelistCount(db) {
|
|
1643
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1644
|
+
}
|
|
1645
|
+
function readCheckpointResult(db) {
|
|
1646
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1647
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1648
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1649
|
+
}
|
|
1650
|
+
async function probeReclaimAvailability(input) {
|
|
1651
|
+
const { dbPath, requiredBytes } = input;
|
|
1652
|
+
let sqlite;
|
|
1653
|
+
try {
|
|
1654
|
+
sqlite = await import("node:sqlite");
|
|
1655
|
+
} catch (err) {
|
|
1656
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1657
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1658
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1659
|
+
}
|
|
1660
|
+
let autoVacuum = null;
|
|
1661
|
+
try {
|
|
1662
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1663
|
+
try {
|
|
1664
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1665
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1666
|
+
} finally {
|
|
1667
|
+
db.close();
|
|
1668
|
+
}
|
|
1669
|
+
} catch (err) {
|
|
1670
|
+
console.warn(
|
|
1671
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1672
|
+
);
|
|
1673
|
+
}
|
|
1674
|
+
if (autoVacuum !== 0) return null;
|
|
1675
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1676
|
+
}
|
|
1677
|
+
async function reclaimSessionDbSpace(input) {
|
|
1678
|
+
const {
|
|
1679
|
+
dbPath,
|
|
1680
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1681
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1682
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1683
|
+
now = () => performance.now(),
|
|
1684
|
+
hasLiveTurn = false,
|
|
1685
|
+
hasProvenanceAnomaly = false
|
|
1686
|
+
} = input;
|
|
1687
|
+
let sqlite;
|
|
1688
|
+
try {
|
|
1689
|
+
sqlite = await import("node:sqlite");
|
|
1690
|
+
} catch (err) {
|
|
1691
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1692
|
+
console.warn(
|
|
1693
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1694
|
+
);
|
|
1695
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1696
|
+
}
|
|
1697
|
+
const { DatabaseSync } = sqlite;
|
|
1698
|
+
let db;
|
|
1699
|
+
try {
|
|
1700
|
+
db = new DatabaseSync(dbPath);
|
|
1701
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1702
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1703
|
+
if (autoVacuum === 0) {
|
|
1704
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1705
|
+
const detail = [
|
|
1706
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1707
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1708
|
+
].join(" and ");
|
|
1709
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1710
|
+
return {
|
|
1711
|
+
ok: false,
|
|
1712
|
+
skipped: "full-vacuum-blocked",
|
|
1713
|
+
detail
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1717
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1718
|
+
if (skipReason !== null) {
|
|
1719
|
+
console.warn(
|
|
1720
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1721
|
+
);
|
|
1722
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1723
|
+
}
|
|
1724
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1725
|
+
const freelistBefore = readFreelistCount(db);
|
|
1726
|
+
const startedAt = now();
|
|
1727
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1728
|
+
db.exec("VACUUM");
|
|
1729
|
+
const afterBytes = readLogicalBytes(db);
|
|
1730
|
+
const freelistAfter = readFreelistCount(db);
|
|
1731
|
+
const checkpointStartedAt = now();
|
|
1732
|
+
const checkpoint = readCheckpointResult(db);
|
|
1733
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1734
|
+
return {
|
|
1735
|
+
ok: true,
|
|
1736
|
+
mode: "convert",
|
|
1737
|
+
beforeBytes,
|
|
1738
|
+
afterBytes,
|
|
1739
|
+
freelistBefore,
|
|
1740
|
+
freelistAfter,
|
|
1741
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1742
|
+
stoppedBy: "freelist-drained",
|
|
1743
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1744
|
+
checkpointElapsedMs,
|
|
1745
|
+
checkpoint
|
|
1746
|
+
};
|
|
1747
|
+
}
|
|
1748
|
+
if (autoVacuum === 2) {
|
|
1749
|
+
if (hasLiveTurn) {
|
|
1750
|
+
const detail = "a session turn is live";
|
|
1751
|
+
console.warn(
|
|
1752
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1753
|
+
);
|
|
1754
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1755
|
+
}
|
|
1756
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1757
|
+
const freelistBefore = readFreelistCount(db);
|
|
1758
|
+
const startedAt = now();
|
|
1759
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1760
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1761
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1762
|
+
let pagesAttempted = 0;
|
|
1763
|
+
let freelistAfter = freelistBefore;
|
|
1764
|
+
let stoppedBy = "freelist-drained";
|
|
1765
|
+
while (freelistAfter > 0) {
|
|
1766
|
+
if (pagesAttempted >= pageLimit) {
|
|
1767
|
+
stoppedBy = "page-ceiling";
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
if (now() - startedAt >= budget) {
|
|
1771
|
+
stoppedBy = "time-budget";
|
|
1772
|
+
break;
|
|
1773
|
+
}
|
|
1774
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1775
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1776
|
+
pagesAttempted += pages;
|
|
1777
|
+
freelistAfter = readFreelistCount(db);
|
|
1778
|
+
}
|
|
1779
|
+
const afterBytes = readLogicalBytes(db);
|
|
1780
|
+
const checkpointStartedAt = now();
|
|
1781
|
+
const checkpoint = readCheckpointResult(db);
|
|
1782
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1783
|
+
return {
|
|
1784
|
+
ok: true,
|
|
1785
|
+
mode: "incremental",
|
|
1786
|
+
beforeBytes,
|
|
1787
|
+
afterBytes,
|
|
1788
|
+
freelistBefore,
|
|
1789
|
+
freelistAfter,
|
|
1790
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1791
|
+
stoppedBy,
|
|
1792
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1793
|
+
checkpointElapsedMs,
|
|
1794
|
+
checkpoint
|
|
1795
|
+
};
|
|
1796
|
+
}
|
|
1797
|
+
console.warn(
|
|
1798
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1799
|
+
);
|
|
1800
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1801
|
+
} catch (err) {
|
|
1802
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1803
|
+
return {
|
|
1804
|
+
ok: false,
|
|
1805
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1806
|
+
detail: errorMessage(err)
|
|
1807
|
+
};
|
|
1808
|
+
} finally {
|
|
1809
|
+
db?.close();
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1323
1814
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1324
1815
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1325
1816
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1336,11 +1827,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1336
1827
|
if (!levelMatch) return null;
|
|
1337
1828
|
const level = levelMatch[1].toUpperCase();
|
|
1338
1829
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1339
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1830
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1831
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1340
1832
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1341
1833
|
const parsed = {
|
|
1342
1834
|
level: level === "WARN" ? "warn" : "error",
|
|
1343
|
-
sessionID: sessionMatch?.[1]
|
|
1835
|
+
sessionID: sessionMatch?.[1],
|
|
1836
|
+
messageID: messageIDMatch?.[1]
|
|
1344
1837
|
};
|
|
1345
1838
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1346
1839
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1348,12 +1841,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1348
1841
|
}
|
|
1349
1842
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1350
1843
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1351
|
-
function createOpenCodeActivityForwarder(getContext) {
|
|
1844
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1352
1845
|
let buffer = Buffer.alloc(0);
|
|
1353
1846
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1354
1847
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1355
1848
|
const metadata = {
|
|
1356
1849
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1850
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1357
1851
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1358
1852
|
};
|
|
1359
1853
|
forwardRunnerActivity(
|
|
@@ -1379,9 +1873,24 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1379
1873
|
};
|
|
1380
1874
|
const flushLine = (line) => {
|
|
1381
1875
|
const parsed = parseOpenCodeLogLine(line);
|
|
1876
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1877
|
+
const cause = sessionStoreCauseToken(line);
|
|
1878
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1879
|
+
try {
|
|
1880
|
+
onSessionStoreWriteFailure({
|
|
1881
|
+
isDiskFailure,
|
|
1882
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1883
|
+
cause
|
|
1884
|
+
});
|
|
1885
|
+
} catch (err) {
|
|
1886
|
+
reportTelemetryDiagnostic(
|
|
1887
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1888
|
+
);
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1382
1891
|
if (!parsed) return;
|
|
1383
1892
|
const now = Date.now();
|
|
1384
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1893
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1385
1894
|
expireRepeatStates(now, key);
|
|
1386
1895
|
if (key === void 0) {
|
|
1387
1896
|
forwardLine(line, parsed);
|
|
@@ -1427,34 +1936,21 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1427
1936
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1428
1937
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1429
1938
|
var MAX_METADATA_ENTRIES = 20;
|
|
1430
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1431
|
-
function redactHomeDirectory(message) {
|
|
1432
|
-
const home = homedir2();
|
|
1433
|
-
if (!home || home === "/") {
|
|
1434
|
-
return message;
|
|
1435
|
-
}
|
|
1436
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1437
|
-
let result = "";
|
|
1438
|
-
let searchFrom = 0;
|
|
1439
|
-
let homeIndex;
|
|
1440
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1441
|
-
const homeEnd = homeIndex + home.length;
|
|
1442
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1443
|
-
const nextCharacter = message[homeEnd];
|
|
1444
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1445
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1446
|
-
result += atPathBoundary ? "~" : home;
|
|
1447
|
-
searchFrom = homeEnd;
|
|
1448
|
-
}
|
|
1449
|
-
return result + message.slice(searchFrom);
|
|
1450
|
-
}
|
|
1451
|
-
function redactDiagnosticText(message) {
|
|
1452
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1453
|
-
return redactHomeDirectory(redacted);
|
|
1454
|
-
}
|
|
1455
1939
|
function truncate(message) {
|
|
1456
1940
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1457
|
-
|
|
1941
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1942
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1943
|
+
let previousDropped = -1;
|
|
1944
|
+
while (dropped !== previousDropped) {
|
|
1945
|
+
previousDropped = dropped;
|
|
1946
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1947
|
+
dropped = nextDropped;
|
|
1948
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1949
|
+
}
|
|
1950
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1951
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1952
|
+
const tailLength = remaining - headLength;
|
|
1953
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1458
1954
|
}
|
|
1459
1955
|
function sanitiseMetadata(metadata) {
|
|
1460
1956
|
if (!metadata) return {};
|
|
@@ -1664,12 +2160,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1664
2160
|
metadata: withoutContractFields(record),
|
|
1665
2161
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1666
2162
|
};
|
|
1667
|
-
case "session_db_boot_refused":
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
2163
|
+
case "session_db_boot_refused": {
|
|
2164
|
+
const message = (() => {
|
|
2165
|
+
switch (record.reason) {
|
|
2166
|
+
case "local_discard_failed":
|
|
2167
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2168
|
+
case "replica_separation_unproven":
|
|
2169
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2170
|
+
case "verification_timeout_before_separation":
|
|
2171
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2172
|
+
case "verification_timeout_during_separation":
|
|
2173
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2174
|
+
default:
|
|
2175
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2176
|
+
}
|
|
2177
|
+
})();
|
|
2178
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2179
|
+
}
|
|
1673
2180
|
case "schema_provenance_mismatch":
|
|
1674
2181
|
return {
|
|
1675
2182
|
level,
|
|
@@ -1919,7 +2426,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1919
2426
|
return null;
|
|
1920
2427
|
}
|
|
1921
2428
|
}
|
|
1922
|
-
function
|
|
2429
|
+
function roleOf2(m) {
|
|
1923
2430
|
if (!m || typeof m !== "object") return void 0;
|
|
1924
2431
|
if (typeof m.role === "string") return m.role;
|
|
1925
2432
|
const infoRole = m.info?.role;
|
|
@@ -1933,13 +2440,13 @@ function createdOf(m) {
|
|
|
1933
2440
|
if (!m || typeof m !== "object") return void 0;
|
|
1934
2441
|
return m.info?.time?.created ?? m.time?.created;
|
|
1935
2442
|
}
|
|
1936
|
-
function
|
|
2443
|
+
function idOf2(m) {
|
|
1937
2444
|
if (!m || typeof m !== "object") return void 0;
|
|
1938
2445
|
if (typeof m.id === "string") return m.id;
|
|
1939
2446
|
const infoId = m.info?.id;
|
|
1940
2447
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1941
2448
|
}
|
|
1942
|
-
function
|
|
2449
|
+
function parentIdOf2(m) {
|
|
1943
2450
|
if (!m || typeof m !== "object") return void 0;
|
|
1944
2451
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1945
2452
|
const infoParent = m.info?.parentID;
|
|
@@ -1955,8 +2462,12 @@ function errorOf(m) {
|
|
|
1955
2462
|
if (!m || typeof m !== "object") return void 0;
|
|
1956
2463
|
return m.info?.error ?? m.error;
|
|
1957
2464
|
}
|
|
2465
|
+
function isTerminalStopAnswer(m) {
|
|
2466
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2467
|
+
}
|
|
1958
2468
|
function isAssistantInFlight(m) {
|
|
1959
2469
|
if (completedOf(m) == null) return true;
|
|
2470
|
+
if (errorOf(m) != null) return false;
|
|
1960
2471
|
return finishOf(m) === "tool-calls";
|
|
1961
2472
|
}
|
|
1962
2473
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2004,9 +2515,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2004
2515
|
function isSessionActivelyGenerating(messages) {
|
|
2005
2516
|
if (!messages || messages.length === 0) return false;
|
|
2006
2517
|
const last = messages[messages.length - 1];
|
|
2007
|
-
if (
|
|
2518
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2008
2519
|
return completedOf(last) == null;
|
|
2009
2520
|
}
|
|
2521
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2010
2522
|
function sessionLastActivityMs(session) {
|
|
2011
2523
|
const candidates = [
|
|
2012
2524
|
session.time?.updated,
|
|
@@ -2021,30 +2533,71 @@ function sessionLastActivityMs(session) {
|
|
|
2021
2533
|
}
|
|
2022
2534
|
return null;
|
|
2023
2535
|
}
|
|
2024
|
-
|
|
2025
|
-
|
|
2536
|
+
function sessionListPath(options) {
|
|
2537
|
+
const params = new URLSearchParams();
|
|
2538
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2539
|
+
const query = params.toString();
|
|
2540
|
+
return query ? `/session?${query}` : "/session";
|
|
2541
|
+
}
|
|
2542
|
+
async function listSessionPage(port, client, options) {
|
|
2543
|
+
const path = sessionListPath(options);
|
|
2026
2544
|
try {
|
|
2027
|
-
const res = await requestWithClient(port, client,
|
|
2545
|
+
const res = await requestWithClient(port, client, path);
|
|
2028
2546
|
if (!res.ok) return null;
|
|
2029
2547
|
const body = await res.json();
|
|
2030
2548
|
return Array.isArray(body) ? body : null;
|
|
2031
2549
|
} catch (error2) {
|
|
2032
2550
|
console.error(
|
|
2033
|
-
`[
|
|
2551
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2034
2552
|
);
|
|
2035
2553
|
return null;
|
|
2036
2554
|
}
|
|
2037
2555
|
}
|
|
2556
|
+
async function listSessions(port, client, options) {
|
|
2557
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2558
|
+
return listSessionPage(port, client, options);
|
|
2559
|
+
}
|
|
2560
|
+
async function listAllSessions(port, client) {
|
|
2561
|
+
if (client?.version === "v2") {
|
|
2562
|
+
const sessions2 = await listV2Sessions(client);
|
|
2563
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2564
|
+
}
|
|
2565
|
+
const sessions = await listSessionPage(port, client, {
|
|
2566
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2567
|
+
});
|
|
2568
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2569
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2570
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2571
|
+
}
|
|
2572
|
+
return { sessions };
|
|
2573
|
+
}
|
|
2038
2574
|
async function deleteSession(port, id, client) {
|
|
2039
2575
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2040
2576
|
try {
|
|
2041
|
-
const res = await requestWithClient(
|
|
2042
|
-
|
|
2577
|
+
const res = await requestWithClient(
|
|
2578
|
+
port,
|
|
2579
|
+
client,
|
|
2580
|
+
`/session/${id}`,
|
|
2581
|
+
{ method: "DELETE" },
|
|
2582
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2583
|
+
);
|
|
2584
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2585
|
+
return { ok: true, status: res.status };
|
|
2586
|
+
}
|
|
2587
|
+
try {
|
|
2588
|
+
const body = await res.text();
|
|
2589
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2590
|
+
} catch (error2) {
|
|
2591
|
+
console.error(
|
|
2592
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2593
|
+
);
|
|
2594
|
+
return { ok: false, status: res.status };
|
|
2595
|
+
}
|
|
2043
2596
|
} catch (error2) {
|
|
2044
2597
|
console.error(
|
|
2045
2598
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2046
2599
|
);
|
|
2047
|
-
return false;
|
|
2600
|
+
return { ok: false, status: "threw" };
|
|
2048
2601
|
}
|
|
2049
2602
|
}
|
|
2050
2603
|
async function sessionExists(port, id, client) {
|
|
@@ -2282,11 +2835,31 @@ function messageText(m) {
|
|
|
2282
2835
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2283
2836
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2284
2837
|
}
|
|
2285
|
-
|
|
2838
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2839
|
+
if (!messages || messages.length === 0) return null;
|
|
2840
|
+
let best = null;
|
|
2841
|
+
for (const message of messages) {
|
|
2842
|
+
if (roleOf2(message) !== "user") continue;
|
|
2843
|
+
const id = idOf2(message);
|
|
2844
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2845
|
+
if (messageText(message) !== content) continue;
|
|
2846
|
+
const created = createdOf(message) ?? 0;
|
|
2847
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2848
|
+
}
|
|
2849
|
+
return best?.id ?? null;
|
|
2850
|
+
}
|
|
2851
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2286
2852
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2287
2853
|
const knownUserIds = new Set(
|
|
2288
|
-
(before ?? []).filter((m) =>
|
|
2854
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2289
2855
|
);
|
|
2856
|
+
if (callbacks) {
|
|
2857
|
+
try {
|
|
2858
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2859
|
+
} catch (error2) {
|
|
2860
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2290
2863
|
const parts = [{ type: "text", text: content }];
|
|
2291
2864
|
let pendingOutcomes = null;
|
|
2292
2865
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2325,22 +2898,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2325
2898
|
const READ_BACK_DELAY_MS = 150;
|
|
2326
2899
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2327
2900
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
const id = idOf(m);
|
|
2333
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2334
|
-
if (messageText(m) !== content) continue;
|
|
2335
|
-
const created = createdOf(m) ?? 0;
|
|
2336
|
-
if (best === null || created > best.created) {
|
|
2337
|
-
best = { id, created };
|
|
2338
|
-
}
|
|
2339
|
-
}
|
|
2340
|
-
if (best) {
|
|
2341
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2342
|
-
return best.id;
|
|
2343
|
-
}
|
|
2901
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2902
|
+
if (dispatchedUserMessageId !== null) {
|
|
2903
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2904
|
+
return dispatchedUserMessageId;
|
|
2344
2905
|
}
|
|
2345
2906
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2346
2907
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2351,57 +2912,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2351
2912
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2352
2913
|
if (!messages || messages.length === 0) return null;
|
|
2353
2914
|
const byParent = messages.find(
|
|
2354
|
-
(m) =>
|
|
2915
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2355
2916
|
);
|
|
2356
2917
|
if (byParent) return byParent;
|
|
2357
|
-
const userIndex = messages.findIndex((m) =>
|
|
2918
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2358
2919
|
if (userIndex === -1) return null;
|
|
2359
2920
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2360
|
-
if (
|
|
2921
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2361
2922
|
}
|
|
2362
2923
|
return null;
|
|
2363
2924
|
}
|
|
2364
2925
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2365
2926
|
if (!messages || messages.length === 0) return null;
|
|
2366
2927
|
let lastCorrelated = null;
|
|
2367
|
-
let lastNonErrored = null;
|
|
2368
2928
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2369
2929
|
const m = messages[i];
|
|
2370
|
-
if (
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2930
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2931
|
+
lastCorrelated = m;
|
|
2932
|
+
break;
|
|
2933
|
+
}
|
|
2934
|
+
if (lastCorrelated) {
|
|
2935
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2936
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2937
|
+
const m = messages[i];
|
|
2938
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2939
|
+
return m;
|
|
2940
|
+
}
|
|
2375
2941
|
}
|
|
2942
|
+
return lastCorrelated;
|
|
2376
2943
|
}
|
|
2377
|
-
|
|
2378
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2944
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2379
2945
|
if (userIndex === -1) return null;
|
|
2380
2946
|
let last = null;
|
|
2381
|
-
let
|
|
2947
|
+
let lastAssistantIndex = -1;
|
|
2382
2948
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2383
|
-
const role =
|
|
2949
|
+
const role = roleOf2(messages[i]);
|
|
2384
2950
|
if (role === "user") break;
|
|
2385
2951
|
if (role === "assistant") {
|
|
2386
2952
|
last = messages[i];
|
|
2387
|
-
|
|
2953
|
+
lastAssistantIndex = i;
|
|
2388
2954
|
}
|
|
2389
2955
|
}
|
|
2390
|
-
|
|
2956
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2957
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2958
|
+
const m = messages[i];
|
|
2959
|
+
if (roleOf2(m) === "user") break;
|
|
2960
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2961
|
+
}
|
|
2962
|
+
return last;
|
|
2391
2963
|
}
|
|
2392
2964
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2393
2965
|
if (!messages || messages.length === 0) return [];
|
|
2394
2966
|
const byParent = messages.filter(
|
|
2395
|
-
(message) =>
|
|
2967
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2396
2968
|
);
|
|
2397
2969
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2398
2970
|
if (assistants.length === 0) {
|
|
2399
|
-
const userIndex = messages.findIndex((message) =>
|
|
2971
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2400
2972
|
if (userIndex === -1) return [];
|
|
2401
2973
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2402
2974
|
const message = messages[i];
|
|
2403
|
-
if (
|
|
2404
|
-
if (
|
|
2975
|
+
if (roleOf2(message) === "user") break;
|
|
2976
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2405
2977
|
}
|
|
2406
2978
|
}
|
|
2407
2979
|
const refs = [];
|
|
@@ -2441,7 +3013,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2441
3013
|
if (!messages || messages.length === 0) return [];
|
|
2442
3014
|
const calls = [];
|
|
2443
3015
|
for (const message of messages) {
|
|
2444
|
-
if (
|
|
3016
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2445
3017
|
for (const part of message.parts ?? []) {
|
|
2446
3018
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2447
3019
|
continue;
|
|
@@ -2471,7 +3043,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2471
3043
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2472
3044
|
const unattributed = [];
|
|
2473
3045
|
for (const message of messages ?? []) {
|
|
2474
|
-
if (
|
|
3046
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2475
3047
|
const created = finiteNumber(createdOf(message));
|
|
2476
3048
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2477
3049
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2493,8 +3065,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2493
3065
|
}
|
|
2494
3066
|
function sumAssistantUsage(messages) {
|
|
2495
3067
|
if (!messages || messages.length === 0) return null;
|
|
2496
|
-
const
|
|
2497
|
-
const
|
|
3068
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3069
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3070
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2498
3071
|
let sawAnyUsage = false;
|
|
2499
3072
|
let inputSum = 0;
|
|
2500
3073
|
let outputSum = 0;
|
|
@@ -2548,13 +3121,11 @@ function sumAssistantUsage(messages) {
|
|
|
2548
3121
|
function messageUsage(messages, userMessageId) {
|
|
2549
3122
|
if (!messages || messages.length === 0) return null;
|
|
2550
3123
|
const byParentAll = messages.filter(
|
|
2551
|
-
(m) =>
|
|
3124
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2552
3125
|
);
|
|
2553
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2554
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2555
3126
|
let correlated;
|
|
2556
|
-
if (
|
|
2557
|
-
correlated =
|
|
3127
|
+
if (byParentAll.length > 0) {
|
|
3128
|
+
correlated = byParentAll;
|
|
2558
3129
|
} else {
|
|
2559
3130
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2560
3131
|
correlated = reply ? [reply] : [];
|
|
@@ -2564,7 +3135,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2564
3135
|
}
|
|
2565
3136
|
function messageRunState(messages, userMessageId) {
|
|
2566
3137
|
if (!messages || messages.length === 0) return "unknown";
|
|
2567
|
-
const hasUser = messages.some((m) =>
|
|
3138
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2568
3139
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2569
3140
|
if (!hasUser) {
|
|
2570
3141
|
if (!reply) return "unknown";
|
|
@@ -2600,14 +3171,38 @@ function messageError(messages, userMessageId) {
|
|
|
2600
3171
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2601
3172
|
const error2 = errorOf(reply);
|
|
2602
3173
|
if (error2 == null) return null;
|
|
2603
|
-
|
|
2604
|
-
if (typeof error2 === "
|
|
3174
|
+
let rendered;
|
|
3175
|
+
if (typeof error2 === "string") {
|
|
3176
|
+
rendered = error2;
|
|
3177
|
+
} else if (typeof error2 === "object") {
|
|
2605
3178
|
const e = error2;
|
|
2606
3179
|
const dataMessage = e.data?.message;
|
|
2607
|
-
if (typeof dataMessage === "string")
|
|
2608
|
-
|
|
3180
|
+
if (typeof dataMessage === "string") {
|
|
3181
|
+
rendered = dataMessage;
|
|
3182
|
+
} else if (typeof e.message === "string") {
|
|
3183
|
+
rendered = e.message;
|
|
3184
|
+
} else {
|
|
3185
|
+
rendered = "The agent run failed.";
|
|
3186
|
+
}
|
|
3187
|
+
} else {
|
|
3188
|
+
rendered = "The agent run failed.";
|
|
2609
3189
|
}
|
|
2610
|
-
|
|
3190
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3191
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3192
|
+
) : void 0;
|
|
3193
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3194
|
+
}
|
|
3195
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3196
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3197
|
+
if (!reply) return null;
|
|
3198
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3199
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3200
|
+
return {
|
|
3201
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3202
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3203
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3204
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3205
|
+
};
|
|
2611
3206
|
}
|
|
2612
3207
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2613
3208
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2656,7 +3251,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2656
3251
|
if (!messages || messages.length === 0) return null;
|
|
2657
3252
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2658
3253
|
const message = messages[i];
|
|
2659
|
-
if (
|
|
3254
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2660
3255
|
const created = createdOf(message);
|
|
2661
3256
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2662
3257
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2688,25 +3283,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2688
3283
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2689
3284
|
if (!messages || messages.length === 0) return false;
|
|
2690
3285
|
return messages.some(
|
|
2691
|
-
(m) =>
|
|
3286
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2692
3287
|
);
|
|
2693
3288
|
}
|
|
2694
3289
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2695
3290
|
if (!messages || messages.length === 0) return false;
|
|
2696
|
-
const userIndex = messages.findIndex((message) =>
|
|
3291
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2697
3292
|
if (userIndex === -1) return false;
|
|
2698
3293
|
let hasLaterUser = false;
|
|
2699
3294
|
let hasStartedLaterUser = false;
|
|
2700
3295
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2701
3296
|
const message = messages[i];
|
|
2702
|
-
if (
|
|
3297
|
+
if (roleOf2(message) !== "user") continue;
|
|
2703
3298
|
hasLaterUser = true;
|
|
2704
|
-
const laterUserMessageId =
|
|
3299
|
+
const laterUserMessageId = idOf2(message);
|
|
2705
3300
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2706
3301
|
return false;
|
|
2707
3302
|
}
|
|
2708
3303
|
if (messages.some(
|
|
2709
|
-
(candidate) =>
|
|
3304
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2710
3305
|
)) {
|
|
2711
3306
|
hasStartedLaterUser = true;
|
|
2712
3307
|
}
|
|
@@ -3177,13 +3772,28 @@ async function listV2Sessions(client) {
|
|
|
3177
3772
|
}
|
|
3178
3773
|
async function deleteV2Session(client, sessionId) {
|
|
3179
3774
|
try {
|
|
3180
|
-
await client.request(
|
|
3181
|
-
|
|
3775
|
+
const response = await client.request(
|
|
3776
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3777
|
+
{ method: "DELETE" },
|
|
3778
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3779
|
+
);
|
|
3780
|
+
if (response.status >= 200 && response.status < 300) {
|
|
3781
|
+
return { ok: true, status: response.status };
|
|
3782
|
+
}
|
|
3783
|
+
try {
|
|
3784
|
+
const body = await response.text();
|
|
3785
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
3786
|
+
} catch (error2) {
|
|
3787
|
+
console.error(
|
|
3788
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3789
|
+
);
|
|
3790
|
+
return { ok: false, status: response.status };
|
|
3791
|
+
}
|
|
3182
3792
|
} catch (error2) {
|
|
3183
3793
|
console.error(
|
|
3184
3794
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3185
3795
|
);
|
|
3186
|
-
return false;
|
|
3796
|
+
return { ok: false, status: "threw" };
|
|
3187
3797
|
}
|
|
3188
3798
|
}
|
|
3189
3799
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3805,9 +4415,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3805
4415
|
|
|
3806
4416
|
// src/lib/opencode/session-db-boot.ts
|
|
3807
4417
|
import { spawn as spawn2 } from "node:child_process";
|
|
3808
|
-
import { mkdirSync, statSync as
|
|
4418
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3809
4419
|
import { homedir as homedir4 } from "node:os";
|
|
3810
|
-
import { dirname as
|
|
4420
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3811
4421
|
|
|
3812
4422
|
// src/lib/runner-synchroniser.ts
|
|
3813
4423
|
import { spawn } from "node:child_process";
|
|
@@ -3887,9 +4497,12 @@ function runSynchroniser(args, opts) {
|
|
|
3887
4497
|
|
|
3888
4498
|
// src/lib/opencode/session-db-boot.ts
|
|
3889
4499
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3890
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4500
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3891
4501
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3892
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4502
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4503
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4504
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4505
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3893
4506
|
function commandError(result) {
|
|
3894
4507
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3895
4508
|
}
|
|
@@ -3932,7 +4545,7 @@ function clearMarker(options) {
|
|
|
3932
4545
|
function markNoReplicate(options, message) {
|
|
3933
4546
|
if (options.noReplicateMarker) {
|
|
3934
4547
|
try {
|
|
3935
|
-
mkdirSync(
|
|
4548
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3936
4549
|
writeFileSync(options.noReplicateMarker, "");
|
|
3937
4550
|
} catch (error2) {
|
|
3938
4551
|
options.log(
|
|
@@ -4057,7 +4670,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4057
4670
|
return null;
|
|
4058
4671
|
}
|
|
4059
4672
|
try {
|
|
4060
|
-
if (
|
|
4673
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4061
4674
|
} catch (error2) {
|
|
4062
4675
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4063
4676
|
options.log(
|
|
@@ -4087,7 +4700,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4087
4700
|
return null;
|
|
4088
4701
|
}
|
|
4089
4702
|
try {
|
|
4090
|
-
mkdirSync(
|
|
4703
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
4091
4704
|
writeFileSync(configPath, rendered.stdout);
|
|
4092
4705
|
} catch (error2) {
|
|
4093
4706
|
options.log(
|
|
@@ -4200,20 +4813,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4200
4813
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4201
4814
|
env: {
|
|
4202
4815
|
...env,
|
|
4203
|
-
// The synchroniser reads
|
|
4204
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4816
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4205
4817
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4206
4818
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4819
|
+
),
|
|
4820
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4821
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4822
|
+
),
|
|
4823
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4824
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4825
|
+
),
|
|
4826
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4827
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4207
4828
|
)
|
|
4208
4829
|
}
|
|
4209
4830
|
});
|
|
4210
4831
|
logSynchroniserDiagnostics(result, options);
|
|
4211
4832
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4833
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4834
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4835
|
+
options.log(
|
|
4836
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4837
|
+
"error"
|
|
4838
|
+
);
|
|
4839
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4840
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4841
|
+
}
|
|
4212
4842
|
options.log(
|
|
4213
4843
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4214
4844
|
"warn"
|
|
4215
4845
|
);
|
|
4216
|
-
return false;
|
|
4846
|
+
return { verifyFatal: false };
|
|
4217
4847
|
}
|
|
4218
4848
|
if (result.code === 34) {
|
|
4219
4849
|
reportRecord(
|
|
@@ -4223,14 +4853,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4223
4853
|
null,
|
|
4224
4854
|
options
|
|
4225
4855
|
);
|
|
4226
|
-
return true;
|
|
4856
|
+
return { verifyFatal: true };
|
|
4227
4857
|
}
|
|
4228
4858
|
if (result.code === 33) {
|
|
4229
4859
|
options.log(
|
|
4230
4860
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4231
4861
|
"warn"
|
|
4232
4862
|
);
|
|
4233
|
-
return false;
|
|
4863
|
+
return { verifyFatal: false };
|
|
4234
4864
|
}
|
|
4235
4865
|
if (result.code !== 0) {
|
|
4236
4866
|
options.log(
|
|
@@ -4238,17 +4868,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4238
4868
|
"warn"
|
|
4239
4869
|
);
|
|
4240
4870
|
}
|
|
4241
|
-
return false;
|
|
4871
|
+
return { verifyFatal: false };
|
|
4242
4872
|
}
|
|
4243
4873
|
options.log(
|
|
4244
4874
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4245
4875
|
"debug"
|
|
4246
4876
|
);
|
|
4247
|
-
return false;
|
|
4877
|
+
return { verifyFatal: false };
|
|
4248
4878
|
}
|
|
4249
4879
|
function fileExists(path) {
|
|
4250
4880
|
try {
|
|
4251
|
-
|
|
4881
|
+
statSync3(path);
|
|
4252
4882
|
return true;
|
|
4253
4883
|
} catch (error2) {
|
|
4254
4884
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4331,13 +4961,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4331
4961
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4332
4962
|
return { verifyFatal: false };
|
|
4333
4963
|
}
|
|
4334
|
-
return
|
|
4964
|
+
return verifySessionDb(options, configPath, env);
|
|
4335
4965
|
}
|
|
4336
4966
|
|
|
4337
4967
|
// src/lib/opencode/session-db-provenance.ts
|
|
4338
4968
|
import { createRequire } from "node:module";
|
|
4339
4969
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4340
|
-
import { dirname as
|
|
4970
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4341
4971
|
var require2 = createRequire(import.meta.url);
|
|
4342
4972
|
function readSessionDbMigrationIds(dbPath) {
|
|
4343
4973
|
let db;
|
|
@@ -4403,7 +5033,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4403
5033
|
}
|
|
4404
5034
|
function saveSessionDbProvenanceState(path, state) {
|
|
4405
5035
|
try {
|
|
4406
|
-
mkdirSync2(
|
|
5036
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4407
5037
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4408
5038
|
`, "utf8");
|
|
4409
5039
|
} catch (error2) {
|
|
@@ -4984,7 +5614,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4984
5614
|
toDelete.push(s.id);
|
|
4985
5615
|
}
|
|
4986
5616
|
}
|
|
4987
|
-
|
|
5617
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5618
|
+
return toDelete.sort(
|
|
5619
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5620
|
+
);
|
|
4988
5621
|
}
|
|
4989
5622
|
var DEFAULT_INTERVAL = "1h";
|
|
4990
5623
|
function resolve(flag, envValue, fallback) {
|
|
@@ -5044,13 +5677,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5044
5677
|
}
|
|
5045
5678
|
|
|
5046
5679
|
// src/lib/opencode/session-db-size.ts
|
|
5047
|
-
import { statSync as
|
|
5680
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5048
5681
|
import { join as join6 } from "node:path";
|
|
5049
5682
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5050
5683
|
function statSessionDbBytes(homeDir) {
|
|
5051
5684
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5052
5685
|
try {
|
|
5053
|
-
return
|
|
5686
|
+
return statSync4(dbPath).size;
|
|
5054
5687
|
} catch (err) {
|
|
5055
5688
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5056
5689
|
if (!isMissingFile) {
|
|
@@ -5076,7 +5709,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5076
5709
|
}
|
|
5077
5710
|
|
|
5078
5711
|
// src/lib/opencode/log-tail.ts
|
|
5079
|
-
import { statSync as
|
|
5712
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5080
5713
|
import { homedir as homedir5 } from "node:os";
|
|
5081
5714
|
import { join as join7 } from "node:path";
|
|
5082
5715
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -5099,186 +5732,63 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5099
5732
|
let inode = null;
|
|
5100
5733
|
let baselineReady = true;
|
|
5101
5734
|
try {
|
|
5102
|
-
const initial =
|
|
5735
|
+
const initial = statSync5(logPath);
|
|
5103
5736
|
offset = initial.size;
|
|
5104
5737
|
inode = initial.ino;
|
|
5105
5738
|
} catch (error2) {
|
|
5106
5739
|
if (!isEnoent(error2)) {
|
|
5107
|
-
reportFailure("initial stat", logPath, error2);
|
|
5108
|
-
baselineReady = false;
|
|
5109
|
-
}
|
|
5110
|
-
}
|
|
5111
|
-
let polling = false;
|
|
5112
|
-
let stopped = false;
|
|
5113
|
-
const poll = async () => {
|
|
5114
|
-
if (polling || stopped) return;
|
|
5115
|
-
polling = true;
|
|
5116
|
-
try {
|
|
5117
|
-
let current;
|
|
5118
|
-
try {
|
|
5119
|
-
current = await stat(logPath);
|
|
5120
|
-
} catch (error2) {
|
|
5121
|
-
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5122
|
-
return;
|
|
5123
|
-
}
|
|
5124
|
-
if (!baselineReady) {
|
|
5125
|
-
offset = current.size;
|
|
5126
|
-
inode = current.ino;
|
|
5127
|
-
baselineReady = true;
|
|
5128
|
-
return;
|
|
5129
|
-
}
|
|
5130
|
-
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5131
|
-
offset = 0;
|
|
5132
|
-
}
|
|
5133
|
-
inode = current.ino;
|
|
5134
|
-
if (current.size === offset) return;
|
|
5135
|
-
const length = current.size - offset;
|
|
5136
|
-
const fh = await open2(logPath, "r");
|
|
5137
|
-
try {
|
|
5138
|
-
const buf = Buffer.alloc(length);
|
|
5139
|
-
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5140
|
-
offset += bytesRead;
|
|
5141
|
-
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5142
|
-
} finally {
|
|
5143
|
-
await fh.close();
|
|
5144
|
-
}
|
|
5145
|
-
} catch (error2) {
|
|
5146
|
-
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5147
|
-
} finally {
|
|
5148
|
-
polling = false;
|
|
5149
|
-
}
|
|
5150
|
-
};
|
|
5151
|
-
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5152
|
-
void poll();
|
|
5153
|
-
return {
|
|
5154
|
-
stop: () => {
|
|
5155
|
-
stopped = true;
|
|
5156
|
-
clearInterval(interval);
|
|
5157
|
-
}
|
|
5158
|
-
};
|
|
5159
|
-
}
|
|
5160
|
-
|
|
5161
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5162
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5163
|
-
import { dirname as dirname4 } from "node:path";
|
|
5164
|
-
function errorMessage(error2) {
|
|
5165
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5166
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5167
|
-
}
|
|
5168
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5169
|
-
try {
|
|
5170
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5171
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5172
|
-
if (availableBytes < requiredBytes) {
|
|
5173
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5174
|
-
}
|
|
5175
|
-
return null;
|
|
5176
|
-
} catch (err) {
|
|
5177
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5178
|
-
}
|
|
5179
|
-
}
|
|
5180
|
-
function readLogicalBytes(db) {
|
|
5181
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5182
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5183
|
-
return pageCount * pageSize;
|
|
5184
|
-
}
|
|
5185
|
-
function readCheckpointResult(db) {
|
|
5186
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5187
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5188
|
-
}
|
|
5189
|
-
async function probeReclaimAvailability(input) {
|
|
5190
|
-
const { dbPath, requiredBytes } = input;
|
|
5191
|
-
let sqlite;
|
|
5192
|
-
try {
|
|
5193
|
-
sqlite = await import("node:sqlite");
|
|
5194
|
-
} catch (err) {
|
|
5195
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5196
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5197
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5198
|
-
}
|
|
5199
|
-
let autoVacuum = null;
|
|
5200
|
-
try {
|
|
5201
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5202
|
-
try {
|
|
5203
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5204
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5205
|
-
} finally {
|
|
5206
|
-
db.close();
|
|
5207
|
-
}
|
|
5208
|
-
} catch (err) {
|
|
5209
|
-
console.warn(
|
|
5210
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5211
|
-
);
|
|
5212
|
-
}
|
|
5213
|
-
if (autoVacuum !== 0) return null;
|
|
5214
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5215
|
-
}
|
|
5216
|
-
async function reclaimSessionDbSpace(input) {
|
|
5217
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5218
|
-
let sqlite;
|
|
5219
|
-
try {
|
|
5220
|
-
sqlite = await import("node:sqlite");
|
|
5221
|
-
} catch (err) {
|
|
5222
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5223
|
-
console.warn(
|
|
5224
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5225
|
-
);
|
|
5226
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5227
|
-
}
|
|
5228
|
-
const { DatabaseSync } = sqlite;
|
|
5229
|
-
let db;
|
|
5230
|
-
try {
|
|
5231
|
-
db = new DatabaseSync(dbPath);
|
|
5232
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5233
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5234
|
-
if (autoVacuum === 0) {
|
|
5235
|
-
if (!allowFullVacuum) {
|
|
5236
|
-
console.warn(
|
|
5237
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5238
|
-
);
|
|
5239
|
-
return {
|
|
5240
|
-
ok: false,
|
|
5241
|
-
skipped: "full-vacuum-blocked",
|
|
5242
|
-
detail: "a session turn is live"
|
|
5243
|
-
};
|
|
5740
|
+
reportFailure("initial stat", logPath, error2);
|
|
5741
|
+
baselineReady = false;
|
|
5742
|
+
}
|
|
5743
|
+
}
|
|
5744
|
+
let polling = false;
|
|
5745
|
+
let stopped = false;
|
|
5746
|
+
const poll = async () => {
|
|
5747
|
+
if (polling || stopped) return;
|
|
5748
|
+
polling = true;
|
|
5749
|
+
try {
|
|
5750
|
+
let current;
|
|
5751
|
+
try {
|
|
5752
|
+
current = await stat(logPath);
|
|
5753
|
+
} catch (error2) {
|
|
5754
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
5755
|
+
return;
|
|
5244
5756
|
}
|
|
5245
|
-
|
|
5246
|
-
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
);
|
|
5251
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5757
|
+
if (!baselineReady) {
|
|
5758
|
+
offset = current.size;
|
|
5759
|
+
inode = current.ino;
|
|
5760
|
+
baselineReady = true;
|
|
5761
|
+
return;
|
|
5252
5762
|
}
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5763
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
5764
|
+
offset = 0;
|
|
5765
|
+
}
|
|
5766
|
+
inode = current.ino;
|
|
5767
|
+
if (current.size === offset) return;
|
|
5768
|
+
const length = current.size - offset;
|
|
5769
|
+
const fh = await open2(logPath, "r");
|
|
5770
|
+
try {
|
|
5771
|
+
const buf = Buffer.alloc(length);
|
|
5772
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
5773
|
+
offset += bytesRead;
|
|
5774
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
5775
|
+
} finally {
|
|
5776
|
+
await fh.close();
|
|
5777
|
+
}
|
|
5778
|
+
} catch (error2) {
|
|
5779
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
5780
|
+
} finally {
|
|
5781
|
+
polling = false;
|
|
5259
5782
|
}
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5783
|
+
};
|
|
5784
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
5785
|
+
void poll();
|
|
5786
|
+
return {
|
|
5787
|
+
stop: () => {
|
|
5788
|
+
stopped = true;
|
|
5789
|
+
clearInterval(interval);
|
|
5267
5790
|
}
|
|
5268
|
-
|
|
5269
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5270
|
-
);
|
|
5271
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5272
|
-
} catch (err) {
|
|
5273
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5274
|
-
return {
|
|
5275
|
-
ok: false,
|
|
5276
|
-
skipped: "reclaim-error",
|
|
5277
|
-
detail: errorMessage(err)
|
|
5278
|
-
};
|
|
5279
|
-
} finally {
|
|
5280
|
-
db?.close();
|
|
5281
|
-
}
|
|
5791
|
+
};
|
|
5282
5792
|
}
|
|
5283
5793
|
|
|
5284
5794
|
// src/lib/tunnel/connection.ts
|
|
@@ -6170,6 +6680,101 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6170
6680
|
};
|
|
6171
6681
|
}
|
|
6172
6682
|
|
|
6683
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6684
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6685
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6686
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6687
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6688
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6689
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6690
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6691
|
+
var BAND_RANK = {
|
|
6692
|
+
low: 1,
|
|
6693
|
+
very_low: 2,
|
|
6694
|
+
critical: 3
|
|
6695
|
+
};
|
|
6696
|
+
function bandFor(freeBytes) {
|
|
6697
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6698
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6699
|
+
return "low";
|
|
6700
|
+
}
|
|
6701
|
+
function formatBytes(bytes) {
|
|
6702
|
+
if (bytes === null) return "unavailable";
|
|
6703
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6704
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6705
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6706
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6707
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6708
|
+
}
|
|
6709
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6710
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6711
|
+
}
|
|
6712
|
+
function evaluateLowDiskWarning(input) {
|
|
6713
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6714
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6715
|
+
return { warning: null, state };
|
|
6716
|
+
}
|
|
6717
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6718
|
+
if (recovered) {
|
|
6719
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6720
|
+
}
|
|
6721
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6722
|
+
if (!low) return { warning: null, state };
|
|
6723
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6724
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6725
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6726
|
+
const worse = currentRank > previousRank;
|
|
6727
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6728
|
+
const nextState = {
|
|
6729
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6730
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6731
|
+
};
|
|
6732
|
+
return {
|
|
6733
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6734
|
+
state: nextState
|
|
6735
|
+
};
|
|
6736
|
+
}
|
|
6737
|
+
|
|
6738
|
+
// src/lib/opencode/session-store-health.ts
|
|
6739
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6740
|
+
function createSessionStoreHealth(options = {}) {
|
|
6741
|
+
const now = options.now ?? (() => Date.now());
|
|
6742
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6743
|
+
let lastUnwritableAt = null;
|
|
6744
|
+
let lastWritableAt = null;
|
|
6745
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6746
|
+
return {
|
|
6747
|
+
observeUnwritable: () => {
|
|
6748
|
+
lastUnwritableAt = now();
|
|
6749
|
+
},
|
|
6750
|
+
observeWritable: () => {
|
|
6751
|
+
lastWritableAt = now();
|
|
6752
|
+
},
|
|
6753
|
+
isUnwritable: () => {
|
|
6754
|
+
if (lastUnwritableAt === null) return false;
|
|
6755
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6756
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6757
|
+
},
|
|
6758
|
+
observeCause: (sessionId, cause) => {
|
|
6759
|
+
if (sessionId === null) return;
|
|
6760
|
+
causes.delete(sessionId);
|
|
6761
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6762
|
+
while (causes.size > 256) {
|
|
6763
|
+
const oldest = causes.keys().next().value;
|
|
6764
|
+
if (typeof oldest !== "string") break;
|
|
6765
|
+
causes.delete(oldest);
|
|
6766
|
+
}
|
|
6767
|
+
},
|
|
6768
|
+
readCause: (sessionId) => {
|
|
6769
|
+
const observed = causes.get(sessionId);
|
|
6770
|
+
if (!observed) return null;
|
|
6771
|
+
causes.delete(sessionId);
|
|
6772
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6773
|
+
return observed.cause;
|
|
6774
|
+
}
|
|
6775
|
+
};
|
|
6776
|
+
}
|
|
6777
|
+
|
|
6173
6778
|
// src/lib/channels/driver.ts
|
|
6174
6779
|
import { homedir as homedir6 } from "node:os";
|
|
6175
6780
|
|
|
@@ -6606,6 +7211,11 @@ function messageIdOf(m) {
|
|
|
6606
7211
|
const infoId = m.info?.id;
|
|
6607
7212
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6608
7213
|
}
|
|
7214
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7215
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7216
|
+
if (replyShape === null) return base;
|
|
7217
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7218
|
+
}
|
|
6609
7219
|
function cleanImageMime(contentType) {
|
|
6610
7220
|
if (!contentType) return null;
|
|
6611
7221
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6626,8 +7236,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6626
7236
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6627
7237
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6628
7238
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7239
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7240
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7241
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7242
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6629
7243
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6630
7244
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7245
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6631
7246
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6632
7247
|
var HEARTBEAT_MS = 6e4;
|
|
6633
7248
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6689,6 +7304,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6689
7304
|
maxActiveSessions;
|
|
6690
7305
|
watcherStallMs;
|
|
6691
7306
|
wedgeWarningIntervalMs;
|
|
7307
|
+
sessionStoreHealth;
|
|
6692
7308
|
openCodeClient;
|
|
6693
7309
|
/** Cache of conversationId → opencode sessionId. */
|
|
6694
7310
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6781,6 +7397,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6781
7397
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6782
7398
|
*/
|
|
6783
7399
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7400
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6784
7401
|
/**
|
|
6785
7402
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6786
7403
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6813,10 +7430,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6813
7430
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6814
7431
|
*/
|
|
6815
7432
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7433
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7434
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6816
7435
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6817
7436
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6818
7437
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6819
7438
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7439
|
+
/**
|
|
7440
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7441
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7442
|
+
* identical-text turn, so it never produces a correlation.
|
|
7443
|
+
*/
|
|
7444
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6820
7445
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6821
7446
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6822
7447
|
/**
|
|
@@ -6902,21 +7527,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6902
7527
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6903
7528
|
*/
|
|
6904
7529
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6905
|
-
/**
|
|
6906
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6907
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6908
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6909
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6910
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6911
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6912
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6913
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6914
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6915
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6916
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6917
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6918
|
-
*/
|
|
6919
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6920
7530
|
/**
|
|
6921
7531
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6922
7532
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6934,8 +7544,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6934
7544
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6935
7545
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6936
7546
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6937
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6938
|
-
*
|
|
7547
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7548
|
+
* the terminal-report fence and is never sent again).
|
|
6939
7549
|
*/
|
|
6940
7550
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6941
7551
|
/**
|
|
@@ -7058,6 +7668,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7058
7668
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7059
7669
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7060
7670
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7671
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7061
7672
|
}
|
|
7062
7673
|
get isV2() {
|
|
7063
7674
|
return this.openCodeClient.version === "v2";
|
|
@@ -7073,7 +7684,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7073
7684
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7074
7685
|
}
|
|
7075
7686
|
async listSessions() {
|
|
7076
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7687
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7688
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7689
|
+
});
|
|
7077
7690
|
}
|
|
7078
7691
|
async sessionExists(sessionId) {
|
|
7079
7692
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7250,6 +7863,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7250
7863
|
}
|
|
7251
7864
|
return ids;
|
|
7252
7865
|
}
|
|
7866
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7867
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7868
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7869
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7870
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7871
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7872
|
+
}
|
|
7873
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7874
|
+
}
|
|
7875
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7876
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7877
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7878
|
+
this.awaitingReadopt.delete(messageId);
|
|
7879
|
+
this.sessions.delete(conversationId);
|
|
7880
|
+
this.supersede(conversationId, sessionId);
|
|
7881
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7882
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7883
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7884
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7885
|
+
this.log({
|
|
7886
|
+
level: "error",
|
|
7887
|
+
message: errorMessage3,
|
|
7888
|
+
conversation_id: conversationId,
|
|
7889
|
+
message_id: messageId
|
|
7890
|
+
});
|
|
7891
|
+
const reportError = await this.reportUntrackableAck(
|
|
7892
|
+
conversationId,
|
|
7893
|
+
messageId,
|
|
7894
|
+
null,
|
|
7895
|
+
errorMessage3
|
|
7896
|
+
);
|
|
7897
|
+
if (reportError !== null) {
|
|
7898
|
+
this.log({
|
|
7899
|
+
level: "warn",
|
|
7900
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7901
|
+
conversation_id: conversationId,
|
|
7902
|
+
message_id: messageId
|
|
7903
|
+
});
|
|
7904
|
+
}
|
|
7905
|
+
if (source === "readopt") {
|
|
7906
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7907
|
+
}
|
|
7908
|
+
}
|
|
7253
7909
|
/**
|
|
7254
7910
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7255
7911
|
*
|
|
@@ -7429,6 +8085,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7429
8085
|
}
|
|
7430
8086
|
break;
|
|
7431
8087
|
}
|
|
8088
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8089
|
+
if (correlation) {
|
|
8090
|
+
correlation.attempts += 1;
|
|
8091
|
+
let correlatedId = null;
|
|
8092
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8093
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8094
|
+
conv,
|
|
8095
|
+
message,
|
|
8096
|
+
correlation.sessionId
|
|
8097
|
+
);
|
|
8098
|
+
if (polled.ok)
|
|
8099
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8100
|
+
} else if (correlation.knownUserIds === null) {
|
|
8101
|
+
this.log({
|
|
8102
|
+
level: "debug",
|
|
8103
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8104
|
+
conversation_id: conv.id,
|
|
8105
|
+
message_id: message.id
|
|
8106
|
+
});
|
|
8107
|
+
}
|
|
8108
|
+
if (correlatedId !== null) {
|
|
8109
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8110
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8111
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8112
|
+
this.dispatched.add(message.id);
|
|
8113
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8114
|
+
dispatched += 1;
|
|
8115
|
+
this.log({
|
|
8116
|
+
level: "info",
|
|
8117
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8118
|
+
conversation_id: conv.id,
|
|
8119
|
+
message_id: message.id
|
|
8120
|
+
});
|
|
8121
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8122
|
+
opencode_message_id: correlatedId
|
|
8123
|
+
});
|
|
8124
|
+
continue;
|
|
8125
|
+
}
|
|
8126
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8127
|
+
await this.handoffUntrackableCorrelation(
|
|
8128
|
+
conv.id,
|
|
8129
|
+
message.id,
|
|
8130
|
+
correlation.sessionId,
|
|
8131
|
+
correlation.attempts,
|
|
8132
|
+
"fresh"
|
|
8133
|
+
);
|
|
8134
|
+
break;
|
|
8135
|
+
}
|
|
8136
|
+
break;
|
|
8137
|
+
}
|
|
7432
8138
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7433
8139
|
if (effectiveOpencodeMessageId) {
|
|
7434
8140
|
const outcome = await this.resolveRedrive(
|
|
@@ -7450,6 +8156,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7450
8156
|
model: message.opencode_model ?? void 0
|
|
7451
8157
|
};
|
|
7452
8158
|
let opencodeMessageId;
|
|
8159
|
+
let preDispatchUserIds = null;
|
|
7453
8160
|
try {
|
|
7454
8161
|
this.log({
|
|
7455
8162
|
level: "info",
|
|
@@ -7479,7 +8186,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7479
8186
|
message.content,
|
|
7480
8187
|
options,
|
|
7481
8188
|
sendAttachments,
|
|
7482
|
-
this.openCodeClient
|
|
8189
|
+
this.openCodeClient,
|
|
8190
|
+
{
|
|
8191
|
+
onKnownUserIds: (ids) => {
|
|
8192
|
+
preDispatchUserIds = ids;
|
|
8193
|
+
},
|
|
8194
|
+
onKnownUserIdsError: (error2) => {
|
|
8195
|
+
this.log({
|
|
8196
|
+
level: "warn",
|
|
8197
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8198
|
+
conversation_id: conv.id,
|
|
8199
|
+
message_id: message.id
|
|
8200
|
+
});
|
|
8201
|
+
}
|
|
8202
|
+
}
|
|
7483
8203
|
)
|
|
7484
8204
|
);
|
|
7485
8205
|
} catch (err) {
|
|
@@ -7487,6 +8207,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7487
8207
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7488
8208
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7489
8209
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8210
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7490
8211
|
this.log({
|
|
7491
8212
|
level: "error",
|
|
7492
8213
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7562,39 +8283,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7562
8283
|
if (this.isV2) {
|
|
7563
8284
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7564
8285
|
}
|
|
7565
|
-
|
|
7566
|
-
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
7570
|
-
|
|
7571
|
-
message_id: message.id
|
|
7572
|
-
});
|
|
7573
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7574
|
-
continue;
|
|
7575
|
-
}
|
|
7576
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7577
|
-
this.sessions.delete(conv.id);
|
|
7578
|
-
this.supersede(conv.id, sessionId);
|
|
7579
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8286
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8287
|
+
sessionId,
|
|
8288
|
+
content: message.content,
|
|
8289
|
+
attempts: 0,
|
|
8290
|
+
knownUserIds: preDispatchUserIds
|
|
8291
|
+
});
|
|
7580
8292
|
this.log({
|
|
7581
|
-
level: "
|
|
7582
|
-
message:
|
|
8293
|
+
level: "warn",
|
|
8294
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7583
8295
|
conversation_id: conv.id,
|
|
7584
8296
|
message_id: message.id
|
|
7585
8297
|
});
|
|
7586
|
-
|
|
7587
|
-
this.log({
|
|
7588
|
-
level: "warn",
|
|
7589
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7590
|
-
conversation_id: conv.id,
|
|
7591
|
-
message_id: message.id
|
|
7592
|
-
});
|
|
7593
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7594
|
-
});
|
|
8298
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7595
8299
|
break;
|
|
7596
8300
|
}
|
|
7597
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7598
8301
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7599
8302
|
this.releasedOpencodeIds.delete(message.id);
|
|
7600
8303
|
this.dispatched.add(message.id);
|
|
@@ -8199,23 +8902,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8199
8902
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8200
8903
|
return 1;
|
|
8201
8904
|
}
|
|
8202
|
-
/**
|
|
8203
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8204
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8205
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8206
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8207
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8208
|
-
* count, since a new session is a genuinely different attempt.
|
|
8209
|
-
*/
|
|
8210
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8211
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8212
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8213
|
-
existing.count += 1;
|
|
8214
|
-
return existing.count;
|
|
8215
|
-
}
|
|
8216
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8217
|
-
return 1;
|
|
8218
|
-
}
|
|
8219
8905
|
/** Record one queued give-up toward the never-started bound. */
|
|
8220
8906
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8221
8907
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8527,8 +9213,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8527
9213
|
b2LastDescendantCheckMs: 0,
|
|
8528
9214
|
b2RootOngoingHeldLogged: false,
|
|
8529
9215
|
b2AbandonedSignalled: false,
|
|
9216
|
+
b2AbandonedWithoutAnswer: false,
|
|
8530
9217
|
ambiguousPinnedSinceMs: 0,
|
|
8531
9218
|
ambiguousResolved: false,
|
|
9219
|
+
sessionStoreFailurePending: false,
|
|
8532
9220
|
neverStartedResolved: false
|
|
8533
9221
|
});
|
|
8534
9222
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8550,6 +9238,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8550
9238
|
this.bufferedSessionErrors.delete(oldest);
|
|
8551
9239
|
}
|
|
8552
9240
|
}
|
|
9241
|
+
retainSessionError(event) {
|
|
9242
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9243
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9244
|
+
reason: event.reason,
|
|
9245
|
+
observedAt: this.now()
|
|
9246
|
+
});
|
|
9247
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9248
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9249
|
+
if (typeof oldest !== "string") break;
|
|
9250
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9251
|
+
}
|
|
9252
|
+
}
|
|
9253
|
+
readRetainedSessionError(sessionId) {
|
|
9254
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9255
|
+
if (!retained) return null;
|
|
9256
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9257
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9258
|
+
return null;
|
|
9259
|
+
}
|
|
9260
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9261
|
+
return retained.reason;
|
|
9262
|
+
}
|
|
8553
9263
|
/**
|
|
8554
9264
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8555
9265
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8645,8 +9355,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8645
9355
|
b2LastDescendantCheckMs: 0,
|
|
8646
9356
|
b2RootOngoingHeldLogged: false,
|
|
8647
9357
|
b2AbandonedSignalled: false,
|
|
9358
|
+
b2AbandonedWithoutAnswer: false,
|
|
8648
9359
|
ambiguousPinnedSinceMs: 0,
|
|
8649
9360
|
ambiguousResolved: false,
|
|
9361
|
+
sessionStoreFailurePending: false,
|
|
8650
9362
|
neverStartedResolved: false
|
|
8651
9363
|
});
|
|
8652
9364
|
}
|
|
@@ -8874,9 +9586,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8874
9586
|
return;
|
|
8875
9587
|
}
|
|
8876
9588
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9589
|
+
this.retainSessionError(event);
|
|
8877
9590
|
this.log({
|
|
8878
|
-
level: "
|
|
8879
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9591
|
+
level: "warn",
|
|
9592
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8880
9593
|
conversation_id: watcher.conv.id
|
|
8881
9594
|
});
|
|
8882
9595
|
return;
|
|
@@ -8906,9 +9619,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8906
9619
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8907
9620
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8908
9621
|
if (state !== "queued") {
|
|
9622
|
+
this.retainSessionError(event);
|
|
8909
9623
|
this.log({
|
|
8910
|
-
level: "
|
|
8911
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9624
|
+
level: "warn",
|
|
9625
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8912
9626
|
conversation_id: watcher.conv.id,
|
|
8913
9627
|
message_id: inFlight.evidentMessageId
|
|
8914
9628
|
});
|
|
@@ -9060,6 +9774,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9060
9774
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9061
9775
|
this.neverStartedAttempts.delete(id);
|
|
9062
9776
|
}
|
|
9777
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9778
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9779
|
+
return;
|
|
9780
|
+
}
|
|
9063
9781
|
if (inFlight.neverStartedResolved) {
|
|
9064
9782
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9065
9783
|
inFlight.neverStartedResolved = false;
|
|
@@ -9133,7 +9851,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9133
9851
|
if (state === "failed") {
|
|
9134
9852
|
this.anchorDeliveryDeadline(inFlight);
|
|
9135
9853
|
if (!inFlight.done) {
|
|
9136
|
-
const
|
|
9854
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9855
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9856
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9857
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9858
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9859
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9860
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9861
|
+
(detail) => detail !== null
|
|
9862
|
+
);
|
|
9863
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9137
9864
|
this.log({
|
|
9138
9865
|
level: "error",
|
|
9139
9866
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9219,10 +9946,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9219
9946
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9220
9947
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9221
9948
|
inFlight.b2AbandonedSignalled = false;
|
|
9949
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9222
9950
|
}
|
|
9223
9951
|
} else {
|
|
9224
9952
|
if (inFlight.b2AbandonedSignalled) {
|
|
9225
|
-
|
|
9953
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9954
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9955
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9956
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9957
|
+
sessionId,
|
|
9958
|
+
watcher,
|
|
9959
|
+
inFlight,
|
|
9960
|
+
messages,
|
|
9961
|
+
finish
|
|
9962
|
+
);
|
|
9963
|
+
} else {
|
|
9964
|
+
await this.settleMessageDone(
|
|
9965
|
+
sessionId,
|
|
9966
|
+
watcher,
|
|
9967
|
+
inFlight,
|
|
9968
|
+
messages,
|
|
9969
|
+
composeAbandonmentNote(
|
|
9970
|
+
finish,
|
|
9971
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
9972
|
+
"with-answer"
|
|
9973
|
+
)
|
|
9974
|
+
);
|
|
9975
|
+
}
|
|
9226
9976
|
return;
|
|
9227
9977
|
}
|
|
9228
9978
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9239,22 +9989,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9239
9989
|
descendantOngoing,
|
|
9240
9990
|
rootOngoing
|
|
9241
9991
|
})) {
|
|
9992
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9993
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9994
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9995
|
+
watched_for_ms: pinnedForMs,
|
|
9996
|
+
run_state: "failed",
|
|
9997
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9998
|
+
});
|
|
9999
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10000
|
+
return;
|
|
10001
|
+
}
|
|
9242
10002
|
inFlight.b2AbandonedSignalled = true;
|
|
10003
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10004
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10005
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
10006
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
10007
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10008
|
+
watched_for_ms: pinnedForMs,
|
|
10009
|
+
run_state: "failed",
|
|
10010
|
+
finish,
|
|
10011
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
10012
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
10013
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10014
|
+
});
|
|
10015
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10016
|
+
sessionId,
|
|
10017
|
+
watcher,
|
|
10018
|
+
inFlight,
|
|
10019
|
+
messages,
|
|
10020
|
+
finish
|
|
10021
|
+
);
|
|
10022
|
+
return;
|
|
10023
|
+
}
|
|
9243
10024
|
this.log({
|
|
9244
10025
|
level: "warn",
|
|
9245
10026
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9246
10027
|
conversation_id: conv.id,
|
|
9247
10028
|
message_id: id
|
|
9248
10029
|
});
|
|
9249
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9250
10030
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9251
10031
|
watched_for_ms: pinnedForMs,
|
|
9252
|
-
finish
|
|
10032
|
+
finish,
|
|
9253
10033
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9254
10034
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9255
10035
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9256
10036
|
});
|
|
9257
|
-
await this.settleMessageDone(
|
|
10037
|
+
await this.settleMessageDone(
|
|
10038
|
+
sessionId,
|
|
10039
|
+
watcher,
|
|
10040
|
+
inFlight,
|
|
10041
|
+
messages,
|
|
10042
|
+
composeAbandonmentNote(
|
|
10043
|
+
finish,
|
|
10044
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10045
|
+
"with-answer"
|
|
10046
|
+
)
|
|
10047
|
+
);
|
|
9258
10048
|
return;
|
|
9259
10049
|
}
|
|
9260
10050
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9297,6 +10087,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9297
10087
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9298
10088
|
sessionOngoing: ongoing
|
|
9299
10089
|
})) {
|
|
10090
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10091
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10092
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10093
|
+
watched_for_ms: pinnedForMs,
|
|
10094
|
+
run_state: "failed",
|
|
10095
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10096
|
+
});
|
|
10097
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10098
|
+
return;
|
|
10099
|
+
}
|
|
9300
10100
|
inFlight.ambiguousResolved = true;
|
|
9301
10101
|
this.log({
|
|
9302
10102
|
level: "warn",
|
|
@@ -9444,15 +10244,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9444
10244
|
}
|
|
9445
10245
|
}
|
|
9446
10246
|
/**
|
|
9447
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9448
|
-
*
|
|
9449
|
-
*
|
|
9450
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9451
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9452
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9453
|
-
* and risking the two copies silently drifting apart.
|
|
10247
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10248
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10249
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9454
10250
|
*/
|
|
9455
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10251
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9456
10252
|
const conv = watcher.conv;
|
|
9457
10253
|
this.anchorDeliveryDeadline(inFlight);
|
|
9458
10254
|
if (!inFlight.done) {
|
|
@@ -9479,7 +10275,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9479
10275
|
title,
|
|
9480
10276
|
usage,
|
|
9481
10277
|
usageAgentName,
|
|
9482
|
-
subagentInvocations
|
|
10278
|
+
subagentInvocations,
|
|
10279
|
+
note
|
|
9483
10280
|
);
|
|
9484
10281
|
} catch (err) {
|
|
9485
10282
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9521,6 +10318,78 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9521
10318
|
}
|
|
9522
10319
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9523
10320
|
}
|
|
10321
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10322
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10323
|
+
const conv = watcher.conv;
|
|
10324
|
+
const error2 = composeAbandonmentNote(
|
|
10325
|
+
finish,
|
|
10326
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10327
|
+
"without-answer"
|
|
10328
|
+
);
|
|
10329
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10330
|
+
this.log({
|
|
10331
|
+
level: "warn",
|
|
10332
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10333
|
+
conversation_id: conv.id,
|
|
10334
|
+
message_id: inFlight.evidentMessageId
|
|
10335
|
+
});
|
|
10336
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10337
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10338
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10339
|
+
messages,
|
|
10340
|
+
inFlight.opencodeMessageId,
|
|
10341
|
+
inFlight.evidentMessageId
|
|
10342
|
+
);
|
|
10343
|
+
try {
|
|
10344
|
+
await this.markFailed(
|
|
10345
|
+
conv.id,
|
|
10346
|
+
inFlight.evidentMessageId,
|
|
10347
|
+
sessionId,
|
|
10348
|
+
error2,
|
|
10349
|
+
usage,
|
|
10350
|
+
void 0,
|
|
10351
|
+
usageAgentName,
|
|
10352
|
+
subagentInvocations
|
|
10353
|
+
);
|
|
10354
|
+
} catch (err) {
|
|
10355
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10356
|
+
if (err instanceof ChannelTerminalError) {
|
|
10357
|
+
this.log({
|
|
10358
|
+
level: "warn",
|
|
10359
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10360
|
+
conversation_id: conv.id,
|
|
10361
|
+
message_id: inFlight.evidentMessageId
|
|
10362
|
+
});
|
|
10363
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10364
|
+
return;
|
|
10365
|
+
}
|
|
10366
|
+
if (this.now() >= inFlight.deadline) {
|
|
10367
|
+
this.log({
|
|
10368
|
+
level: "warn",
|
|
10369
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10370
|
+
conversation_id: conv.id,
|
|
10371
|
+
message_id: inFlight.evidentMessageId
|
|
10372
|
+
});
|
|
10373
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10374
|
+
return;
|
|
10375
|
+
}
|
|
10376
|
+
this.log({
|
|
10377
|
+
level: "warn",
|
|
10378
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10379
|
+
conversation_id: conv.id,
|
|
10380
|
+
message_id: inFlight.evidentMessageId
|
|
10381
|
+
});
|
|
10382
|
+
return;
|
|
10383
|
+
}
|
|
10384
|
+
inFlight.done = true;
|
|
10385
|
+
await this.reportSubagentAuthFailures(
|
|
10386
|
+
watcher.conv.id,
|
|
10387
|
+
inFlight.opencodeMessageId,
|
|
10388
|
+
inFlight.evidentMessageId,
|
|
10389
|
+
messages
|
|
10390
|
+
);
|
|
10391
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10392
|
+
}
|
|
9524
10393
|
/**
|
|
9525
10394
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9526
10395
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9576,6 +10445,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9576
10445
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9577
10446
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9578
10447
|
}
|
|
10448
|
+
/**
|
|
10449
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10450
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10451
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10452
|
+
*/
|
|
10453
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10454
|
+
const conv = watcher.conv;
|
|
10455
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10456
|
+
this.log({
|
|
10457
|
+
level: "error",
|
|
10458
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10459
|
+
conversation_id: conv.id,
|
|
10460
|
+
message_id: inFlight.evidentMessageId
|
|
10461
|
+
});
|
|
10462
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10463
|
+
try {
|
|
10464
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10465
|
+
} catch (err) {
|
|
10466
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10467
|
+
if (err instanceof ChannelTerminalError) {
|
|
10468
|
+
this.log({
|
|
10469
|
+
level: "warn",
|
|
10470
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10471
|
+
conversation_id: conv.id,
|
|
10472
|
+
message_id: inFlight.evidentMessageId
|
|
10473
|
+
});
|
|
10474
|
+
this.recordReleasedOpencodeId(
|
|
10475
|
+
inFlight.evidentMessageId,
|
|
10476
|
+
sessionId,
|
|
10477
|
+
inFlight.opencodeMessageId
|
|
10478
|
+
);
|
|
10479
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10480
|
+
return;
|
|
10481
|
+
}
|
|
10482
|
+
if (this.now() >= inFlight.deadline) {
|
|
10483
|
+
this.log({
|
|
10484
|
+
level: "warn",
|
|
10485
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10486
|
+
conversation_id: conv.id,
|
|
10487
|
+
message_id: inFlight.evidentMessageId
|
|
10488
|
+
});
|
|
10489
|
+
this.recordReleasedOpencodeId(
|
|
10490
|
+
inFlight.evidentMessageId,
|
|
10491
|
+
sessionId,
|
|
10492
|
+
inFlight.opencodeMessageId
|
|
10493
|
+
);
|
|
10494
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10495
|
+
return;
|
|
10496
|
+
}
|
|
10497
|
+
this.log({
|
|
10498
|
+
level: "warn",
|
|
10499
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10500
|
+
conversation_id: conv.id,
|
|
10501
|
+
message_id: inFlight.evidentMessageId
|
|
10502
|
+
});
|
|
10503
|
+
return;
|
|
10504
|
+
}
|
|
10505
|
+
inFlight.done = true;
|
|
10506
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10507
|
+
}
|
|
9579
10508
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9580
10509
|
/**
|
|
9581
10510
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9614,11 +10543,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9614
10543
|
}
|
|
9615
10544
|
}
|
|
9616
10545
|
}
|
|
9617
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10546
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9618
10547
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10548
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10549
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10550
|
+
this.awaitingCorrelation.delete(id);
|
|
10551
|
+
this.log({
|
|
10552
|
+
level: "debug",
|
|
10553
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10554
|
+
message_id: id
|
|
10555
|
+
});
|
|
10556
|
+
}
|
|
10557
|
+
}
|
|
9619
10558
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9620
10559
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9621
10560
|
this.untrackableAck.delete(id);
|
|
10561
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9622
10562
|
this.untrackableAckSignalled.delete(id);
|
|
9623
10563
|
this.untrackableAckWarned.delete(id);
|
|
9624
10564
|
this.log({
|
|
@@ -9658,7 +10598,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9658
10598
|
bySession.set(row.opencode_session_id, list);
|
|
9659
10599
|
}
|
|
9660
10600
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9661
|
-
let messages;
|
|
10601
|
+
let messages = null;
|
|
9662
10602
|
try {
|
|
9663
10603
|
if (this.isV2) {
|
|
9664
10604
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9667,9 +10607,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9667
10607
|
level: "warn",
|
|
9668
10608
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9669
10609
|
});
|
|
9670
|
-
|
|
10610
|
+
} else {
|
|
10611
|
+
messages = snapshot;
|
|
9671
10612
|
}
|
|
9672
|
-
messages = snapshot;
|
|
9673
10613
|
} else {
|
|
9674
10614
|
const polled = await pollSessionMessagesForRedrive(
|
|
9675
10615
|
this.port,
|
|
@@ -9682,15 +10622,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9682
10622
|
level: "warn",
|
|
9683
10623
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9684
10624
|
});
|
|
9685
|
-
|
|
10625
|
+
} else {
|
|
10626
|
+
messages = polled.messages;
|
|
9686
10627
|
}
|
|
9687
|
-
messages = polled.messages;
|
|
9688
10628
|
}
|
|
9689
10629
|
} catch (err) {
|
|
9690
10630
|
this.log({
|
|
9691
10631
|
level: "warn",
|
|
9692
10632
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9693
10633
|
});
|
|
10634
|
+
}
|
|
10635
|
+
if (messages === null) {
|
|
10636
|
+
for (const row of sessionRows) {
|
|
10637
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10638
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10639
|
+
}
|
|
10640
|
+
}
|
|
9694
10641
|
continue;
|
|
9695
10642
|
}
|
|
9696
10643
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9700,6 +10647,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9700
10647
|
}
|
|
9701
10648
|
}
|
|
9702
10649
|
}
|
|
10650
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10651
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10652
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10653
|
+
if (!correlation) return false;
|
|
10654
|
+
correlation.attempts += 1;
|
|
10655
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10656
|
+
if (correlatedId !== null) {
|
|
10657
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10658
|
+
const conv = this.convForRow(sessionId, row);
|
|
10659
|
+
const message = this.queuedMessageForRow(row);
|
|
10660
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10661
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10662
|
+
this.dispatched.add(row.id);
|
|
10663
|
+
this.readopted.add(row.id);
|
|
10664
|
+
this.ensureWatcherRunning(sessionId);
|
|
10665
|
+
this.log({
|
|
10666
|
+
level: "info",
|
|
10667
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10668
|
+
conversation_id: row.conversation_id,
|
|
10669
|
+
message_id: row.id
|
|
10670
|
+
});
|
|
10671
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10672
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10673
|
+
opencode_message_id: correlatedId
|
|
10674
|
+
});
|
|
10675
|
+
return true;
|
|
10676
|
+
}
|
|
10677
|
+
if (correlation.knownUserIds === null) {
|
|
10678
|
+
this.log({
|
|
10679
|
+
level: "debug",
|
|
10680
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10681
|
+
conversation_id: row.conversation_id,
|
|
10682
|
+
message_id: row.id
|
|
10683
|
+
});
|
|
10684
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10685
|
+
this.log({
|
|
10686
|
+
level: "warn",
|
|
10687
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10688
|
+
conversation_id: row.conversation_id,
|
|
10689
|
+
message_id: row.id
|
|
10690
|
+
});
|
|
10691
|
+
}
|
|
10692
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10693
|
+
await this.handoffUntrackableCorrelation(
|
|
10694
|
+
row.conversation_id,
|
|
10695
|
+
row.id,
|
|
10696
|
+
correlation.sessionId,
|
|
10697
|
+
correlation.attempts,
|
|
10698
|
+
"readopt"
|
|
10699
|
+
);
|
|
10700
|
+
}
|
|
10701
|
+
return true;
|
|
10702
|
+
}
|
|
9703
10703
|
/**
|
|
9704
10704
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9705
10705
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9707,8 +10707,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9707
10707
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9708
10708
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9709
10709
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9710
|
-
* before the restart)
|
|
9711
|
-
*
|
|
10710
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10711
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10712
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10713
|
+
* `forceReadoptRun`):
|
|
9712
10714
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9713
10715
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9714
10716
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9717,7 +10719,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9717
10719
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9718
10720
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9719
10721
|
* tracking the stored id so the reply correlates by it;
|
|
9720
|
-
* - `unknown`/null id →
|
|
10722
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10723
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9721
10724
|
*
|
|
9722
10725
|
* Only `ChannelAuthError` propagates.
|
|
9723
10726
|
*/
|
|
@@ -9731,6 +10734,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9731
10734
|
});
|
|
9732
10735
|
return;
|
|
9733
10736
|
}
|
|
10737
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9734
10738
|
const ocId = row.opencode_message_id;
|
|
9735
10739
|
const state = messageRunState(messages, ocId ?? "");
|
|
9736
10740
|
if (state === "done") {
|
|
@@ -9797,12 +10801,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9797
10801
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9798
10802
|
this.dontRedispatch.delete(row.id);
|
|
9799
10803
|
this.untrackableAck.delete(row.id);
|
|
10804
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9800
10805
|
this.untrackableAckSignalled.delete(row.id);
|
|
9801
10806
|
this.untrackableAckWarned.delete(row.id);
|
|
9802
10807
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9803
10808
|
return;
|
|
9804
10809
|
}
|
|
9805
|
-
if (this.
|
|
10810
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10811
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10812
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10813
|
+
this.untrackableAckWarned.add(row.id);
|
|
10814
|
+
this.log({
|
|
10815
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10816
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10817
|
+
conversation_id: row.conversation_id,
|
|
10818
|
+
message_id: row.id
|
|
10819
|
+
});
|
|
10820
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10821
|
+
const reportError = await this.reportUntrackableAck(
|
|
10822
|
+
row.conversation_id,
|
|
10823
|
+
row.id,
|
|
10824
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10825
|
+
errorMessage3
|
|
10826
|
+
);
|
|
10827
|
+
if (reportError !== null) {
|
|
10828
|
+
this.log({
|
|
10829
|
+
level: "warn",
|
|
10830
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10831
|
+
conversation_id: row.conversation_id,
|
|
10832
|
+
message_id: row.id
|
|
10833
|
+
});
|
|
10834
|
+
}
|
|
10835
|
+
} else {
|
|
10836
|
+
this.log({
|
|
10837
|
+
level: "debug",
|
|
10838
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10839
|
+
conversation_id: row.conversation_id,
|
|
10840
|
+
message_id: row.id
|
|
10841
|
+
});
|
|
10842
|
+
}
|
|
10843
|
+
return;
|
|
10844
|
+
}
|
|
10845
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9806
10846
|
this.log({
|
|
9807
10847
|
level: "debug",
|
|
9808
10848
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9980,6 +11020,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9980
11020
|
}
|
|
9981
11021
|
this.dontRedispatch.delete(row.id);
|
|
9982
11022
|
this.untrackableAck.delete(row.id);
|
|
11023
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9983
11024
|
this.untrackableAckSignalled.delete(row.id);
|
|
9984
11025
|
this.untrackableAckWarned.delete(row.id);
|
|
9985
11026
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9999,8 +11040,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9999
11040
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
10000
11041
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
10001
11042
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
10002
|
-
* re-entry)
|
|
10003
|
-
*
|
|
11043
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11044
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11045
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
10004
11046
|
*
|
|
10005
11047
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
10006
11048
|
* `processed_at` (Invariant 1).
|
|
@@ -10063,6 +11105,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10063
11105
|
);
|
|
10064
11106
|
}
|
|
10065
11107
|
let ocId;
|
|
11108
|
+
let preDispatchUserIds = null;
|
|
10066
11109
|
try {
|
|
10067
11110
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10068
11111
|
sessionId,
|
|
@@ -10072,7 +11115,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10072
11115
|
row.content,
|
|
10073
11116
|
options,
|
|
10074
11117
|
sendAttachments,
|
|
10075
|
-
this.openCodeClient
|
|
11118
|
+
this.openCodeClient,
|
|
11119
|
+
{
|
|
11120
|
+
onKnownUserIds: (ids) => {
|
|
11121
|
+
preDispatchUserIds = ids;
|
|
11122
|
+
},
|
|
11123
|
+
onKnownUserIdsError: (error2) => {
|
|
11124
|
+
this.log({
|
|
11125
|
+
level: "warn",
|
|
11126
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11127
|
+
conversation_id: row.conversation_id,
|
|
11128
|
+
message_id: row.id
|
|
11129
|
+
});
|
|
11130
|
+
}
|
|
11131
|
+
}
|
|
10076
11132
|
)
|
|
10077
11133
|
);
|
|
10078
11134
|
} catch (err) {
|
|
@@ -10105,6 +11161,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10105
11161
|
}
|
|
10106
11162
|
} else {
|
|
10107
11163
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11164
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10108
11165
|
}
|
|
10109
11166
|
this.log({
|
|
10110
11167
|
level: "error",
|
|
@@ -10153,39 +11210,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10153
11210
|
}
|
|
10154
11211
|
if (ocId === null) {
|
|
10155
11212
|
this.awaitingReadopt.delete(row.id);
|
|
10156
|
-
|
|
10157
|
-
|
|
10158
|
-
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
this.log({
|
|
10163
|
-
level: "error",
|
|
10164
|
-
message: errorMessage3,
|
|
10165
|
-
conversation_id: row.conversation_id,
|
|
10166
|
-
message_id: row.id
|
|
10167
|
-
});
|
|
10168
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10169
|
-
this.log({
|
|
10170
|
-
level: "warn",
|
|
10171
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10172
|
-
conversation_id: row.conversation_id,
|
|
10173
|
-
message_id: row.id
|
|
10174
|
-
});
|
|
10175
|
-
});
|
|
10176
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10177
|
-
return;
|
|
10178
|
-
}
|
|
11213
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11214
|
+
sessionId,
|
|
11215
|
+
content: row.content,
|
|
11216
|
+
attempts: 0,
|
|
11217
|
+
knownUserIds: preDispatchUserIds
|
|
11218
|
+
});
|
|
10179
11219
|
this.log({
|
|
10180
11220
|
level: "warn",
|
|
10181
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11221
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10182
11222
|
conversation_id: row.conversation_id,
|
|
10183
11223
|
message_id: row.id
|
|
10184
11224
|
});
|
|
10185
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10186
11225
|
return;
|
|
10187
11226
|
}
|
|
10188
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10189
11227
|
const processedAtMs = this.processedAtMs(row);
|
|
10190
11228
|
this.registerReadopted(
|
|
10191
11229
|
readoptConv,
|
|
@@ -11091,7 +12129,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11091
12129
|
* watcher retries next tick within the
|
|
11092
12130
|
* deadline, Finding 4).
|
|
11093
12131
|
*/
|
|
11094
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12132
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11095
12133
|
const res = await this.fetchImpl(
|
|
11096
12134
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11097
12135
|
{
|
|
@@ -11109,7 +12147,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11109
12147
|
...title ? { title } : {},
|
|
11110
12148
|
...usage ? usage : {},
|
|
11111
12149
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11112
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12150
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12151
|
+
...error2 ? { error: error2 } : {}
|
|
11113
12152
|
})
|
|
11114
12153
|
}
|
|
11115
12154
|
);
|
|
@@ -11837,7 +12876,9 @@ async function installRunnerSecret({
|
|
|
11837
12876
|
}) {
|
|
11838
12877
|
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
11839
12878
|
if (!arn) {
|
|
11840
|
-
|
|
12879
|
+
if (!env.GH_TOKEN) {
|
|
12880
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
12881
|
+
}
|
|
11841
12882
|
return false;
|
|
11842
12883
|
}
|
|
11843
12884
|
const result = await (commandRunner ?? runCommand2)(
|
|
@@ -12780,9 +13821,12 @@ async function driveChannels(state, driver) {
|
|
|
12780
13821
|
}
|
|
12781
13822
|
}
|
|
12782
13823
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12783
|
-
var
|
|
13824
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
13825
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
13826
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
13827
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
12784
13828
|
function shouldWarnForReclaimSkip(reason) {
|
|
12785
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13829
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12786
13830
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12787
13831
|
if (!version2) return false;
|
|
12788
13832
|
const major = Number(version2[1]);
|
|
@@ -12839,61 +13883,146 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12839
13883
|
async function runSweep(state, driver, config) {
|
|
12840
13884
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12841
13885
|
try {
|
|
12842
|
-
const
|
|
12843
|
-
|
|
12844
|
-
|
|
13886
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13887
|
+
const sessions = enumeration.sessions;
|
|
13888
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13889
|
+
if (enumerationReason !== null) {
|
|
13890
|
+
logActivity(state, {
|
|
12845
13891
|
type: "info",
|
|
12846
|
-
|
|
12847
|
-
|
|
12848
|
-
outcome: "sweep-skipped",
|
|
12849
|
-
reason: "opencode-unreachable"
|
|
12850
|
-
}
|
|
13892
|
+
level: "warn",
|
|
13893
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12851
13894
|
});
|
|
12852
|
-
return;
|
|
12853
13895
|
}
|
|
12854
|
-
const toDelete = selectSessionsToDelete(
|
|
12855
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12856
|
-
{
|
|
12857
|
-
maxAgeMs: config.maxAgeMs,
|
|
12858
|
-
maxCount: config.maxCount,
|
|
12859
|
-
nowMs: Date.now(),
|
|
12860
|
-
protectedIds: driver.protectedSessionIds()
|
|
12861
|
-
}
|
|
12862
|
-
);
|
|
12863
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12864
13896
|
let deleted = 0;
|
|
12865
13897
|
let failed = 0;
|
|
13898
|
+
let failed4xx = 0;
|
|
13899
|
+
let failed5xx = 0;
|
|
13900
|
+
let failedOtherStatus = 0;
|
|
13901
|
+
let failedThrew = 0;
|
|
12866
13902
|
let skippedNewlyActive = 0;
|
|
12867
|
-
|
|
12868
|
-
|
|
12869
|
-
|
|
12870
|
-
|
|
13903
|
+
let remaining = 0;
|
|
13904
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13905
|
+
if (sessions !== null) {
|
|
13906
|
+
const toDelete = selectSessionsToDelete(
|
|
13907
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13908
|
+
{
|
|
13909
|
+
maxAgeMs: config.maxAgeMs,
|
|
13910
|
+
maxCount: config.maxCount,
|
|
13911
|
+
nowMs: Date.now(),
|
|
13912
|
+
protectedIds: protectedNow
|
|
13913
|
+
}
|
|
13914
|
+
);
|
|
13915
|
+
const eligibleToDelete = toDelete.filter(
|
|
13916
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
13917
|
+
);
|
|
13918
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13919
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
13920
|
+
const answeredFailures = [];
|
|
13921
|
+
protectedNow = driver.protectedSessionIds();
|
|
13922
|
+
for (const id of deleteCandidates) {
|
|
13923
|
+
if (protectedNow.has(id)) {
|
|
13924
|
+
skippedNewlyActive++;
|
|
13925
|
+
logActivity(state, {
|
|
13926
|
+
type: "info",
|
|
13927
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13928
|
+
});
|
|
13929
|
+
continue;
|
|
13930
|
+
}
|
|
13931
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13932
|
+
if (outcome.ok) {
|
|
13933
|
+
deleted++;
|
|
13934
|
+
} else if (outcome.status === "threw") {
|
|
13935
|
+
failed++;
|
|
13936
|
+
failedThrew++;
|
|
13937
|
+
} else {
|
|
13938
|
+
failed++;
|
|
13939
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13940
|
+
if (statusClass === 4) failed4xx++;
|
|
13941
|
+
else if (statusClass === 5) failed5xx++;
|
|
13942
|
+
else failedOtherStatus++;
|
|
13943
|
+
answeredFailures.push({
|
|
13944
|
+
id,
|
|
13945
|
+
status: outcome.status,
|
|
13946
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
13947
|
+
});
|
|
13948
|
+
}
|
|
13949
|
+
}
|
|
13950
|
+
const givenUpFailures = [];
|
|
13951
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
13952
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
13953
|
+
} else {
|
|
13954
|
+
for (const failure of answeredFailures) {
|
|
13955
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
13956
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
13957
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
13958
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
13959
|
+
givenUpFailures.push(failure);
|
|
13960
|
+
}
|
|
13961
|
+
}
|
|
13962
|
+
}
|
|
13963
|
+
for (const failure of givenUpFailures) {
|
|
13964
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
13965
|
+
0,
|
|
13966
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
13967
|
+
);
|
|
13968
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12871
13969
|
type: "info",
|
|
12872
|
-
|
|
13970
|
+
level: "warn",
|
|
13971
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
13972
|
+
metadata: {
|
|
13973
|
+
outcome: "session-delete-given-up",
|
|
13974
|
+
session_id: failure.id,
|
|
13975
|
+
status: failure.status,
|
|
13976
|
+
body_excerpt: excerpt,
|
|
13977
|
+
given_up_count: givenUpFailures.length
|
|
13978
|
+
}
|
|
12873
13979
|
});
|
|
12874
|
-
continue;
|
|
12875
13980
|
}
|
|
12876
|
-
|
|
12877
|
-
|
|
13981
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13982
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13983
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13984
|
+
logActivity(state, {
|
|
13985
|
+
type: "info",
|
|
13986
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13987
|
+
});
|
|
12878
13988
|
}
|
|
12879
|
-
const
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
13989
|
+
const sessionMetadata = {
|
|
13990
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13991
|
+
sessions_deleted: deleted,
|
|
13992
|
+
sessions_failed: failed,
|
|
13993
|
+
sessions_failed_4xx: failed4xx,
|
|
13994
|
+
sessions_failed_5xx: failed5xx,
|
|
13995
|
+
sessions_failed_other: failedOtherStatus,
|
|
13996
|
+
sessions_failed_threw: failedThrew,
|
|
13997
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13998
|
+
sessions_remaining: remaining,
|
|
13999
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
14000
|
+
};
|
|
14001
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
12885
14002
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12886
14003
|
dbPath: sessionDbPath(),
|
|
12887
|
-
|
|
12888
|
-
|
|
14004
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
14005
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
14006
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
14007
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14008
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
12889
14009
|
});
|
|
12890
14010
|
if (reclaimResult.ok) {
|
|
14011
|
+
state.sessionStoreHealth.observeWritable();
|
|
12891
14012
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12892
14013
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12893
14014
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
14015
|
+
const loopElapsedMs = Math.max(
|
|
14016
|
+
0,
|
|
14017
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14018
|
+
);
|
|
14019
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14020
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14021
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
12894
14022
|
logSessionCleanupActivity(state, "sweep", {
|
|
12895
14023
|
type: "info",
|
|
12896
|
-
|
|
14024
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14025
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
12897
14026
|
metadata: {
|
|
12898
14027
|
outcome: "reclaimed",
|
|
12899
14028
|
mode: reclaimResult.mode,
|
|
@@ -12901,25 +14030,28 @@ async function runSweep(state, driver, config) {
|
|
|
12901
14030
|
after_bytes: reclaimResult.afterBytes,
|
|
12902
14031
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12903
14032
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
14033
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
14034
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
14035
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
14036
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
14037
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14038
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
14039
|
+
...sessionMetadata
|
|
12908
14040
|
}
|
|
12909
14041
|
});
|
|
12910
14042
|
} else {
|
|
14043
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
14044
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
14045
|
+
}
|
|
12911
14046
|
logSessionCleanupActivity(state, "sweep", {
|
|
12912
14047
|
type: "info",
|
|
12913
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
14048
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12914
14049
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12915
14050
|
metadata: {
|
|
12916
14051
|
outcome: "reclaim-skipped",
|
|
12917
14052
|
reason: reclaimResult.skipped,
|
|
12918
14053
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12919
|
-
|
|
12920
|
-
sessions_deleted: deleted,
|
|
12921
|
-
sessions_failed: failed,
|
|
12922
|
-
sessions_skipped_active: skippedNewlyActive
|
|
14054
|
+
...sessionMetadata
|
|
12923
14055
|
}
|
|
12924
14056
|
});
|
|
12925
14057
|
}
|
|
@@ -13176,6 +14308,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13176
14308
|
message: `Resource usage collection: ${warning2}`
|
|
13177
14309
|
});
|
|
13178
14310
|
}
|
|
14311
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14312
|
+
...usage,
|
|
14313
|
+
nowMs: Date.now(),
|
|
14314
|
+
state: state.lowDiskWarningState
|
|
14315
|
+
});
|
|
14316
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14317
|
+
if (lowDiskDecision.warning !== null) {
|
|
14318
|
+
logActivity(state, {
|
|
14319
|
+
type: "info",
|
|
14320
|
+
level: "warn",
|
|
14321
|
+
message: lowDiskDecision.warning
|
|
14322
|
+
});
|
|
14323
|
+
}
|
|
13179
14324
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13180
14325
|
if (result.ok) {
|
|
13181
14326
|
if (consecutiveFailures > 0) {
|
|
@@ -13422,6 +14567,7 @@ async function run(options) {
|
|
|
13422
14567
|
opencodeClient: null,
|
|
13423
14568
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13424
14569
|
sessionDbProvenanceAnomaly: false,
|
|
14570
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13425
14571
|
opencodeProcess: null,
|
|
13426
14572
|
stopOpenCodeLogTail: null,
|
|
13427
14573
|
litestreamProcess: null,
|
|
@@ -13433,12 +14579,14 @@ async function run(options) {
|
|
|
13433
14579
|
messageCount: 0,
|
|
13434
14580
|
lastProxiedActivityAt: null,
|
|
13435
14581
|
sessionCleanupTimers: [],
|
|
14582
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13436
14583
|
claudeUsageTimer: null,
|
|
13437
14584
|
claudeUsageRearm: null,
|
|
13438
14585
|
openaiUsageTimer: null,
|
|
13439
14586
|
openaiUsageRearm: null,
|
|
13440
14587
|
resourceUsageTimer: null,
|
|
13441
14588
|
stopResourceUsageSampling: null,
|
|
14589
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13442
14590
|
credentialSync: null,
|
|
13443
14591
|
authHeader: ""
|
|
13444
14592
|
};
|
|
@@ -13612,27 +14760,62 @@ async function run(options) {
|
|
|
13612
14760
|
} else {
|
|
13613
14761
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13614
14762
|
}
|
|
13615
|
-
if (options.restoreRunnerCredentials) {
|
|
13616
|
-
|
|
13617
|
-
|
|
13618
|
-
|
|
13619
|
-
|
|
13620
|
-
if (level === "error") {
|
|
13621
|
-
logActivity(state, { type: "error", error: message });
|
|
13622
|
-
} else {
|
|
13623
|
-
logActivity(state, { type: "info", level, message });
|
|
13624
|
-
}
|
|
14763
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14764
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14765
|
+
log2(state, message, level);
|
|
14766
|
+
if (state.interactive && !state.json) {
|
|
14767
|
+
logActivity(state, { type: "info", level, message });
|
|
13625
14768
|
}
|
|
13626
14769
|
};
|
|
13627
|
-
const
|
|
14770
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14771
|
+
if (vaultResolution.error) {
|
|
14772
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14773
|
+
logCredentialVaultResolution(
|
|
14774
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14775
|
+
"warn"
|
|
14776
|
+
);
|
|
14777
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14778
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14779
|
+
logCredentialVaultResolution(
|
|
14780
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14781
|
+
"info"
|
|
14782
|
+
);
|
|
14783
|
+
} else {
|
|
14784
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14785
|
+
logCredentialVaultResolution(
|
|
14786
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14787
|
+
"debug"
|
|
14788
|
+
);
|
|
14789
|
+
}
|
|
14790
|
+
}
|
|
14791
|
+
const credentialContext = {
|
|
14792
|
+
env: process.env,
|
|
14793
|
+
log: (message, level = "info") => {
|
|
14794
|
+
if (level === "error") {
|
|
14795
|
+
logActivity(state, { type: "error", error: message });
|
|
14796
|
+
} else {
|
|
14797
|
+
logActivity(state, { type: "info", level, message });
|
|
14798
|
+
}
|
|
14799
|
+
}
|
|
14800
|
+
};
|
|
14801
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
14802
|
+
if (options.restoreRunnerCredentials) {
|
|
13628
14803
|
await restoreCredentialStores(credentialContext);
|
|
13629
|
-
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13630
14804
|
}
|
|
14805
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13631
14806
|
state.credentialSync?.arm();
|
|
13632
|
-
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13633
|
-
|
|
13634
|
-
|
|
13635
|
-
|
|
14807
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14808
|
+
() => ({
|
|
14809
|
+
agentId: state.agentId,
|
|
14810
|
+
authHeader: state.authHeader
|
|
14811
|
+
}),
|
|
14812
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14813
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14814
|
+
if (cause !== null && sessionId !== null) {
|
|
14815
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14816
|
+
}
|
|
14817
|
+
}
|
|
14818
|
+
);
|
|
13636
14819
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13637
14820
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13638
14821
|
openCodeActivityForwarder
|
|
@@ -13642,22 +14825,27 @@ async function run(options) {
|
|
|
13642
14825
|
openCodeLogTail.stop();
|
|
13643
14826
|
};
|
|
13644
14827
|
let sessionDbVerifyFatal = false;
|
|
14828
|
+
let sessionDbVerifyFatalReason;
|
|
13645
14829
|
if (!options.restoreSessionDb) {
|
|
13646
|
-
|
|
14830
|
+
logActivity(state, {
|
|
14831
|
+
type: "info",
|
|
14832
|
+
level: "warn",
|
|
14833
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14834
|
+
});
|
|
13647
14835
|
} else {
|
|
13648
14836
|
const health = await checkOpenCodeHealth(state.port);
|
|
13649
14837
|
if (health.healthy) {
|
|
13650
|
-
|
|
13651
|
-
|
|
13652
|
-
|
|
13653
|
-
"
|
|
13654
|
-
);
|
|
14838
|
+
logActivity(state, {
|
|
14839
|
+
type: "info",
|
|
14840
|
+
level: "warn",
|
|
14841
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14842
|
+
});
|
|
13655
14843
|
} else if (health.authFailed) {
|
|
13656
|
-
|
|
13657
|
-
|
|
13658
|
-
|
|
13659
|
-
"
|
|
13660
|
-
);
|
|
14844
|
+
logActivity(state, {
|
|
14845
|
+
type: "info",
|
|
14846
|
+
level: "warn",
|
|
14847
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14848
|
+
});
|
|
13661
14849
|
} else {
|
|
13662
14850
|
const result = await restoreAndVerifySessionDb({
|
|
13663
14851
|
dbPath: sessionDbPath(),
|
|
@@ -13674,12 +14862,13 @@ async function run(options) {
|
|
|
13674
14862
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13675
14863
|
});
|
|
13676
14864
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14865
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13677
14866
|
}
|
|
13678
14867
|
}
|
|
13679
14868
|
reportSessionDbRecovery(state);
|
|
13680
14869
|
if (sessionDbVerifyFatal) {
|
|
13681
14870
|
throw new Error(
|
|
13682
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14871
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13683
14872
|
);
|
|
13684
14873
|
}
|
|
13685
14874
|
applyRunnerOpenCodeConfig({
|
|
@@ -13924,6 +15113,7 @@ async function run(options) {
|
|
|
13924
15113
|
getAuthHeader: () => state.authHeader,
|
|
13925
15114
|
conversationFilter: state.conversationFilter,
|
|
13926
15115
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15116
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13927
15117
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13928
15118
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13929
15119
|
fileSyncDirectories,
|
|
@@ -14005,14 +15195,14 @@ async function run(options) {
|
|
|
14005
15195
|
},
|
|
14006
15196
|
onDisconnected: (code, reason, transition) => {
|
|
14007
15197
|
state.connected = false;
|
|
14008
|
-
const
|
|
15198
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
14009
15199
|
logActivity(state, {
|
|
14010
15200
|
type: "info",
|
|
14011
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15201
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14012
15202
|
});
|
|
14013
15203
|
emitAgentDisconnected(state.agentId, {
|
|
14014
15204
|
code,
|
|
14015
|
-
reason:
|
|
15205
|
+
reason: safeReason2,
|
|
14016
15206
|
connection_generation: transition.generation
|
|
14017
15207
|
});
|
|
14018
15208
|
if (state.interactive) displayStatus(state);
|
|
@@ -14175,6 +15365,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14175
15365
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14176
15366
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14177
15367
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15368
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14178
15369
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14179
15370
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14180
15371
|
"-a, --agent [id]",
|