@evident-ai/cli 3.5.2-dev.3431f9f → 3.5.2-dev.3e69672
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1242 -373
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -703,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
703
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
704
704
|
});
|
|
705
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
706
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
707
732
|
const apiUrl = getApiUrlConfig();
|
|
708
733
|
try {
|
|
@@ -877,6 +902,76 @@ async function status(options = {}) {
|
|
|
877
902
|
process.exit(result.exitCode);
|
|
878
903
|
}
|
|
879
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// src/lib/claude-usage.ts
|
|
881
976
|
import { execFileSync } from "node:child_process";
|
|
882
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1069,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1069
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1070
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1071
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1072
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1073
1238
|
var TelemetryEventTypes = {
|
|
1074
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1157,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1157
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1158
1323
|
|
|
1159
1324
|
// src/lib/telemetry.ts
|
|
1160
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.3e69672" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1161
1326
|
function getCliVersion() {
|
|
1162
1327
|
return CLI_VERSION;
|
|
1163
1328
|
}
|
|
@@ -1320,6 +1485,219 @@ var EventTypes = {
|
|
|
1320
1485
|
|
|
1321
1486
|
// src/lib/runner-activity-telemetry.ts
|
|
1322
1487
|
import { homedir as homedir2 } from "node:os";
|
|
1488
|
+
|
|
1489
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1490
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1491
|
+
import { dirname as dirname2 } from "node:path";
|
|
1492
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1493
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1494
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1495
|
+
function errorMessage(error2) {
|
|
1496
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1497
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1498
|
+
}
|
|
1499
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1500
|
+
const messages = [];
|
|
1501
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1502
|
+
let current = error2;
|
|
1503
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1504
|
+
visited.add(current);
|
|
1505
|
+
if (typeof current === "string") {
|
|
1506
|
+
messages.push(current);
|
|
1507
|
+
break;
|
|
1508
|
+
}
|
|
1509
|
+
if (current instanceof Error) {
|
|
1510
|
+
messages.push(current.name, current.message);
|
|
1511
|
+
current = current.cause;
|
|
1512
|
+
continue;
|
|
1513
|
+
}
|
|
1514
|
+
if (typeof current === "object") {
|
|
1515
|
+
const record = current;
|
|
1516
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1517
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1518
|
+
current = record.cause;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
break;
|
|
1522
|
+
}
|
|
1523
|
+
return messages.some(
|
|
1524
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1525
|
+
message
|
|
1526
|
+
)
|
|
1527
|
+
);
|
|
1528
|
+
}
|
|
1529
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1530
|
+
try {
|
|
1531
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1532
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1533
|
+
if (availableBytes < requiredBytes) {
|
|
1534
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1535
|
+
}
|
|
1536
|
+
return null;
|
|
1537
|
+
} catch (err) {
|
|
1538
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
function readLogicalBytes(db) {
|
|
1542
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1543
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1544
|
+
return pageCount * pageSize;
|
|
1545
|
+
}
|
|
1546
|
+
function readFreelistCount(db) {
|
|
1547
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1548
|
+
}
|
|
1549
|
+
function readCheckpointResult(db) {
|
|
1550
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1551
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1552
|
+
}
|
|
1553
|
+
async function probeReclaimAvailability(input) {
|
|
1554
|
+
const { dbPath, requiredBytes } = input;
|
|
1555
|
+
let sqlite;
|
|
1556
|
+
try {
|
|
1557
|
+
sqlite = await import("node:sqlite");
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1560
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1561
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1562
|
+
}
|
|
1563
|
+
let autoVacuum = null;
|
|
1564
|
+
try {
|
|
1565
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1566
|
+
try {
|
|
1567
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1568
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1569
|
+
} finally {
|
|
1570
|
+
db.close();
|
|
1571
|
+
}
|
|
1572
|
+
} catch (err) {
|
|
1573
|
+
console.warn(
|
|
1574
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1575
|
+
);
|
|
1576
|
+
}
|
|
1577
|
+
if (autoVacuum !== 0) return null;
|
|
1578
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1579
|
+
}
|
|
1580
|
+
async function reclaimSessionDbSpace(input) {
|
|
1581
|
+
const {
|
|
1582
|
+
dbPath,
|
|
1583
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1584
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1585
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1586
|
+
now = () => performance.now(),
|
|
1587
|
+
allowFullVacuum = true
|
|
1588
|
+
} = input;
|
|
1589
|
+
let sqlite;
|
|
1590
|
+
try {
|
|
1591
|
+
sqlite = await import("node:sqlite");
|
|
1592
|
+
} catch (err) {
|
|
1593
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1594
|
+
console.warn(
|
|
1595
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1596
|
+
);
|
|
1597
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1598
|
+
}
|
|
1599
|
+
const { DatabaseSync } = sqlite;
|
|
1600
|
+
let db;
|
|
1601
|
+
try {
|
|
1602
|
+
db = new DatabaseSync(dbPath);
|
|
1603
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1604
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1605
|
+
if (autoVacuum === 0) {
|
|
1606
|
+
if (!allowFullVacuum) {
|
|
1607
|
+
console.warn(
|
|
1608
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1609
|
+
);
|
|
1610
|
+
return {
|
|
1611
|
+
ok: false,
|
|
1612
|
+
skipped: "full-vacuum-blocked",
|
|
1613
|
+
detail: "a session turn is live"
|
|
1614
|
+
};
|
|
1615
|
+
}
|
|
1616
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1617
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1618
|
+
if (skipReason !== null) {
|
|
1619
|
+
console.warn(
|
|
1620
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1621
|
+
);
|
|
1622
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1623
|
+
}
|
|
1624
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1625
|
+
const freelistBefore = readFreelistCount(db);
|
|
1626
|
+
const startedAt = now();
|
|
1627
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1628
|
+
db.exec("VACUUM");
|
|
1629
|
+
const afterBytes = readLogicalBytes(db);
|
|
1630
|
+
const freelistAfter = readFreelistCount(db);
|
|
1631
|
+
const checkpoint = readCheckpointResult(db);
|
|
1632
|
+
return {
|
|
1633
|
+
ok: true,
|
|
1634
|
+
mode: "convert",
|
|
1635
|
+
beforeBytes,
|
|
1636
|
+
afterBytes,
|
|
1637
|
+
freelistBefore,
|
|
1638
|
+
freelistAfter,
|
|
1639
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1640
|
+
stoppedBy: "freelist-drained",
|
|
1641
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1642
|
+
checkpoint
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
if (autoVacuum === 2) {
|
|
1646
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1647
|
+
const freelistBefore = readFreelistCount(db);
|
|
1648
|
+
const startedAt = now();
|
|
1649
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1650
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1651
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1652
|
+
let pagesAttempted = 0;
|
|
1653
|
+
let freelistAfter = freelistBefore;
|
|
1654
|
+
let stoppedBy = "freelist-drained";
|
|
1655
|
+
while (freelistAfter > 0) {
|
|
1656
|
+
if (pagesAttempted >= pageLimit) {
|
|
1657
|
+
stoppedBy = "page-ceiling";
|
|
1658
|
+
break;
|
|
1659
|
+
}
|
|
1660
|
+
if (now() - startedAt >= budget) {
|
|
1661
|
+
stoppedBy = "time-budget";
|
|
1662
|
+
break;
|
|
1663
|
+
}
|
|
1664
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1665
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1666
|
+
pagesAttempted += pages;
|
|
1667
|
+
freelistAfter = readFreelistCount(db);
|
|
1668
|
+
}
|
|
1669
|
+
const afterBytes = readLogicalBytes(db);
|
|
1670
|
+
const checkpoint = readCheckpointResult(db);
|
|
1671
|
+
return {
|
|
1672
|
+
ok: true,
|
|
1673
|
+
mode: "incremental",
|
|
1674
|
+
beforeBytes,
|
|
1675
|
+
afterBytes,
|
|
1676
|
+
freelistBefore,
|
|
1677
|
+
freelistAfter,
|
|
1678
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1679
|
+
stoppedBy,
|
|
1680
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1681
|
+
checkpoint
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1684
|
+
console.warn(
|
|
1685
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1686
|
+
);
|
|
1687
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1688
|
+
} catch (err) {
|
|
1689
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1690
|
+
return {
|
|
1691
|
+
ok: false,
|
|
1692
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1693
|
+
detail: errorMessage(err)
|
|
1694
|
+
};
|
|
1695
|
+
} finally {
|
|
1696
|
+
db?.close();
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1323
1701
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1324
1702
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1325
1703
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1336,11 +1714,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1336
1714
|
if (!levelMatch) return null;
|
|
1337
1715
|
const level = levelMatch[1].toUpperCase();
|
|
1338
1716
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1339
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1717
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1718
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1340
1719
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1341
1720
|
const parsed = {
|
|
1342
1721
|
level: level === "WARN" ? "warn" : "error",
|
|
1343
|
-
sessionID: sessionMatch?.[1]
|
|
1722
|
+
sessionID: sessionMatch?.[1],
|
|
1723
|
+
messageID: messageIDMatch?.[1]
|
|
1344
1724
|
};
|
|
1345
1725
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1346
1726
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1348,12 +1728,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1348
1728
|
}
|
|
1349
1729
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1350
1730
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1351
|
-
function createOpenCodeActivityForwarder(getContext) {
|
|
1731
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1352
1732
|
let buffer = Buffer.alloc(0);
|
|
1353
1733
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1354
1734
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1355
1735
|
const metadata = {
|
|
1356
1736
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1737
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1357
1738
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1358
1739
|
};
|
|
1359
1740
|
forwardRunnerActivity(
|
|
@@ -1378,10 +1759,19 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1378
1759
|
}
|
|
1379
1760
|
};
|
|
1380
1761
|
const flushLine = (line) => {
|
|
1762
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1763
|
+
try {
|
|
1764
|
+
onSessionStoreWriteFailure();
|
|
1765
|
+
} catch (err) {
|
|
1766
|
+
reportTelemetryDiagnostic(
|
|
1767
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1768
|
+
);
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1381
1771
|
const parsed = parseOpenCodeLogLine(line);
|
|
1382
1772
|
if (!parsed) return;
|
|
1383
1773
|
const now = Date.now();
|
|
1384
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1774
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1385
1775
|
expireRepeatStates(now, key);
|
|
1386
1776
|
if (key === void 0) {
|
|
1387
1777
|
forwardLine(line, parsed);
|
|
@@ -1919,7 +2309,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1919
2309
|
return null;
|
|
1920
2310
|
}
|
|
1921
2311
|
}
|
|
1922
|
-
function
|
|
2312
|
+
function roleOf2(m) {
|
|
1923
2313
|
if (!m || typeof m !== "object") return void 0;
|
|
1924
2314
|
if (typeof m.role === "string") return m.role;
|
|
1925
2315
|
const infoRole = m.info?.role;
|
|
@@ -1933,13 +2323,13 @@ function createdOf(m) {
|
|
|
1933
2323
|
if (!m || typeof m !== "object") return void 0;
|
|
1934
2324
|
return m.info?.time?.created ?? m.time?.created;
|
|
1935
2325
|
}
|
|
1936
|
-
function
|
|
2326
|
+
function idOf2(m) {
|
|
1937
2327
|
if (!m || typeof m !== "object") return void 0;
|
|
1938
2328
|
if (typeof m.id === "string") return m.id;
|
|
1939
2329
|
const infoId = m.info?.id;
|
|
1940
2330
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1941
2331
|
}
|
|
1942
|
-
function
|
|
2332
|
+
function parentIdOf2(m) {
|
|
1943
2333
|
if (!m || typeof m !== "object") return void 0;
|
|
1944
2334
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1945
2335
|
const infoParent = m.info?.parentID;
|
|
@@ -1955,8 +2345,12 @@ function errorOf(m) {
|
|
|
1955
2345
|
if (!m || typeof m !== "object") return void 0;
|
|
1956
2346
|
return m.info?.error ?? m.error;
|
|
1957
2347
|
}
|
|
2348
|
+
function isTerminalStopAnswer(m) {
|
|
2349
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2350
|
+
}
|
|
1958
2351
|
function isAssistantInFlight(m) {
|
|
1959
2352
|
if (completedOf(m) == null) return true;
|
|
2353
|
+
if (errorOf(m) != null) return false;
|
|
1960
2354
|
return finishOf(m) === "tool-calls";
|
|
1961
2355
|
}
|
|
1962
2356
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2004,9 +2398,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2004
2398
|
function isSessionActivelyGenerating(messages) {
|
|
2005
2399
|
if (!messages || messages.length === 0) return false;
|
|
2006
2400
|
const last = messages[messages.length - 1];
|
|
2007
|
-
if (
|
|
2401
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2008
2402
|
return completedOf(last) == null;
|
|
2009
2403
|
}
|
|
2404
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2010
2405
|
function sessionLastActivityMs(session) {
|
|
2011
2406
|
const candidates = [
|
|
2012
2407
|
session.time?.updated,
|
|
@@ -2021,30 +2416,60 @@ function sessionLastActivityMs(session) {
|
|
|
2021
2416
|
}
|
|
2022
2417
|
return null;
|
|
2023
2418
|
}
|
|
2024
|
-
|
|
2025
|
-
|
|
2419
|
+
function sessionListPath(options) {
|
|
2420
|
+
const params = new URLSearchParams();
|
|
2421
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2422
|
+
const query = params.toString();
|
|
2423
|
+
return query ? `/session?${query}` : "/session";
|
|
2424
|
+
}
|
|
2425
|
+
async function listSessionPage(port, client, options) {
|
|
2426
|
+
const path = sessionListPath(options);
|
|
2026
2427
|
try {
|
|
2027
|
-
const res = await requestWithClient(port, client,
|
|
2428
|
+
const res = await requestWithClient(port, client, path);
|
|
2028
2429
|
if (!res.ok) return null;
|
|
2029
2430
|
const body = await res.json();
|
|
2030
2431
|
return Array.isArray(body) ? body : null;
|
|
2031
2432
|
} catch (error2) {
|
|
2032
2433
|
console.error(
|
|
2033
|
-
`[
|
|
2434
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2034
2435
|
);
|
|
2035
2436
|
return null;
|
|
2036
2437
|
}
|
|
2037
2438
|
}
|
|
2439
|
+
async function listSessions(port, client, options) {
|
|
2440
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2441
|
+
return listSessionPage(port, client, options);
|
|
2442
|
+
}
|
|
2443
|
+
async function listAllSessions(port, client) {
|
|
2444
|
+
if (client?.version === "v2") {
|
|
2445
|
+
const sessions2 = await listV2Sessions(client);
|
|
2446
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2447
|
+
}
|
|
2448
|
+
const sessions = await listSessionPage(port, client, {
|
|
2449
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2450
|
+
});
|
|
2451
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2452
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2453
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2454
|
+
}
|
|
2455
|
+
return { sessions };
|
|
2456
|
+
}
|
|
2038
2457
|
async function deleteSession(port, id, client) {
|
|
2039
2458
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2040
2459
|
try {
|
|
2041
|
-
const res = await requestWithClient(
|
|
2042
|
-
|
|
2460
|
+
const res = await requestWithClient(
|
|
2461
|
+
port,
|
|
2462
|
+
client,
|
|
2463
|
+
`/session/${id}`,
|
|
2464
|
+
{ method: "DELETE" },
|
|
2465
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2466
|
+
);
|
|
2467
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
2043
2468
|
} catch (error2) {
|
|
2044
2469
|
console.error(
|
|
2045
2470
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2046
2471
|
);
|
|
2047
|
-
return false;
|
|
2472
|
+
return { ok: false, status: "threw" };
|
|
2048
2473
|
}
|
|
2049
2474
|
}
|
|
2050
2475
|
async function sessionExists(port, id, client) {
|
|
@@ -2282,11 +2707,31 @@ function messageText(m) {
|
|
|
2282
2707
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2283
2708
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2284
2709
|
}
|
|
2285
|
-
|
|
2710
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2711
|
+
if (!messages || messages.length === 0) return null;
|
|
2712
|
+
let best = null;
|
|
2713
|
+
for (const message of messages) {
|
|
2714
|
+
if (roleOf2(message) !== "user") continue;
|
|
2715
|
+
const id = idOf2(message);
|
|
2716
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2717
|
+
if (messageText(message) !== content) continue;
|
|
2718
|
+
const created = createdOf(message) ?? 0;
|
|
2719
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2720
|
+
}
|
|
2721
|
+
return best?.id ?? null;
|
|
2722
|
+
}
|
|
2723
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2286
2724
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2287
2725
|
const knownUserIds = new Set(
|
|
2288
|
-
(before ?? []).filter((m) =>
|
|
2726
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2289
2727
|
);
|
|
2728
|
+
if (callbacks) {
|
|
2729
|
+
try {
|
|
2730
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2731
|
+
} catch (error2) {
|
|
2732
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2733
|
+
}
|
|
2734
|
+
}
|
|
2290
2735
|
const parts = [{ type: "text", text: content }];
|
|
2291
2736
|
let pendingOutcomes = null;
|
|
2292
2737
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2325,22 +2770,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2325
2770
|
const READ_BACK_DELAY_MS = 150;
|
|
2326
2771
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2327
2772
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
const id = idOf(m);
|
|
2333
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2334
|
-
if (messageText(m) !== content) continue;
|
|
2335
|
-
const created = createdOf(m) ?? 0;
|
|
2336
|
-
if (best === null || created > best.created) {
|
|
2337
|
-
best = { id, created };
|
|
2338
|
-
}
|
|
2339
|
-
}
|
|
2340
|
-
if (best) {
|
|
2341
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2342
|
-
return best.id;
|
|
2343
|
-
}
|
|
2773
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2774
|
+
if (dispatchedUserMessageId !== null) {
|
|
2775
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2776
|
+
return dispatchedUserMessageId;
|
|
2344
2777
|
}
|
|
2345
2778
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2346
2779
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2351,57 +2784,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2351
2784
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2352
2785
|
if (!messages || messages.length === 0) return null;
|
|
2353
2786
|
const byParent = messages.find(
|
|
2354
|
-
(m) =>
|
|
2787
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2355
2788
|
);
|
|
2356
2789
|
if (byParent) return byParent;
|
|
2357
|
-
const userIndex = messages.findIndex((m) =>
|
|
2790
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2358
2791
|
if (userIndex === -1) return null;
|
|
2359
2792
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2360
|
-
if (
|
|
2793
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2361
2794
|
}
|
|
2362
2795
|
return null;
|
|
2363
2796
|
}
|
|
2364
2797
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2365
2798
|
if (!messages || messages.length === 0) return null;
|
|
2366
2799
|
let lastCorrelated = null;
|
|
2367
|
-
let lastNonErrored = null;
|
|
2368
2800
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2369
2801
|
const m = messages[i];
|
|
2370
|
-
if (
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2802
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2803
|
+
lastCorrelated = m;
|
|
2804
|
+
break;
|
|
2805
|
+
}
|
|
2806
|
+
if (lastCorrelated) {
|
|
2807
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2808
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2809
|
+
const m = messages[i];
|
|
2810
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2811
|
+
return m;
|
|
2812
|
+
}
|
|
2375
2813
|
}
|
|
2814
|
+
return lastCorrelated;
|
|
2376
2815
|
}
|
|
2377
|
-
|
|
2378
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2816
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2379
2817
|
if (userIndex === -1) return null;
|
|
2380
2818
|
let last = null;
|
|
2381
|
-
let
|
|
2819
|
+
let lastAssistantIndex = -1;
|
|
2382
2820
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2383
|
-
const role =
|
|
2821
|
+
const role = roleOf2(messages[i]);
|
|
2384
2822
|
if (role === "user") break;
|
|
2385
2823
|
if (role === "assistant") {
|
|
2386
2824
|
last = messages[i];
|
|
2387
|
-
|
|
2825
|
+
lastAssistantIndex = i;
|
|
2388
2826
|
}
|
|
2389
2827
|
}
|
|
2390
|
-
|
|
2828
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2829
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2830
|
+
const m = messages[i];
|
|
2831
|
+
if (roleOf2(m) === "user") break;
|
|
2832
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2833
|
+
}
|
|
2834
|
+
return last;
|
|
2391
2835
|
}
|
|
2392
2836
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2393
2837
|
if (!messages || messages.length === 0) return [];
|
|
2394
2838
|
const byParent = messages.filter(
|
|
2395
|
-
(message) =>
|
|
2839
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2396
2840
|
);
|
|
2397
2841
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2398
2842
|
if (assistants.length === 0) {
|
|
2399
|
-
const userIndex = messages.findIndex((message) =>
|
|
2843
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2400
2844
|
if (userIndex === -1) return [];
|
|
2401
2845
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2402
2846
|
const message = messages[i];
|
|
2403
|
-
if (
|
|
2404
|
-
if (
|
|
2847
|
+
if (roleOf2(message) === "user") break;
|
|
2848
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2405
2849
|
}
|
|
2406
2850
|
}
|
|
2407
2851
|
const refs = [];
|
|
@@ -2441,7 +2885,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2441
2885
|
if (!messages || messages.length === 0) return [];
|
|
2442
2886
|
const calls = [];
|
|
2443
2887
|
for (const message of messages) {
|
|
2444
|
-
if (
|
|
2888
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2445
2889
|
for (const part of message.parts ?? []) {
|
|
2446
2890
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2447
2891
|
continue;
|
|
@@ -2471,7 +2915,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2471
2915
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2472
2916
|
const unattributed = [];
|
|
2473
2917
|
for (const message of messages ?? []) {
|
|
2474
|
-
if (
|
|
2918
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2475
2919
|
const created = finiteNumber(createdOf(message));
|
|
2476
2920
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2477
2921
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2493,8 +2937,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2493
2937
|
}
|
|
2494
2938
|
function sumAssistantUsage(messages) {
|
|
2495
2939
|
if (!messages || messages.length === 0) return null;
|
|
2496
|
-
const
|
|
2497
|
-
const
|
|
2940
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
2941
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
2942
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2498
2943
|
let sawAnyUsage = false;
|
|
2499
2944
|
let inputSum = 0;
|
|
2500
2945
|
let outputSum = 0;
|
|
@@ -2548,13 +2993,11 @@ function sumAssistantUsage(messages) {
|
|
|
2548
2993
|
function messageUsage(messages, userMessageId) {
|
|
2549
2994
|
if (!messages || messages.length === 0) return null;
|
|
2550
2995
|
const byParentAll = messages.filter(
|
|
2551
|
-
(m) =>
|
|
2996
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2552
2997
|
);
|
|
2553
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2554
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2555
2998
|
let correlated;
|
|
2556
|
-
if (
|
|
2557
|
-
correlated =
|
|
2999
|
+
if (byParentAll.length > 0) {
|
|
3000
|
+
correlated = byParentAll;
|
|
2558
3001
|
} else {
|
|
2559
3002
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2560
3003
|
correlated = reply ? [reply] : [];
|
|
@@ -2564,7 +3007,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2564
3007
|
}
|
|
2565
3008
|
function messageRunState(messages, userMessageId) {
|
|
2566
3009
|
if (!messages || messages.length === 0) return "unknown";
|
|
2567
|
-
const hasUser = messages.some((m) =>
|
|
3010
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2568
3011
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2569
3012
|
if (!hasUser) {
|
|
2570
3013
|
if (!reply) return "unknown";
|
|
@@ -2600,14 +3043,26 @@ function messageError(messages, userMessageId) {
|
|
|
2600
3043
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2601
3044
|
const error2 = errorOf(reply);
|
|
2602
3045
|
if (error2 == null) return null;
|
|
2603
|
-
|
|
2604
|
-
if (typeof error2 === "
|
|
3046
|
+
let rendered;
|
|
3047
|
+
if (typeof error2 === "string") {
|
|
3048
|
+
rendered = error2;
|
|
3049
|
+
} else if (typeof error2 === "object") {
|
|
2605
3050
|
const e = error2;
|
|
2606
3051
|
const dataMessage = e.data?.message;
|
|
2607
|
-
if (typeof dataMessage === "string")
|
|
2608
|
-
|
|
3052
|
+
if (typeof dataMessage === "string") {
|
|
3053
|
+
rendered = dataMessage;
|
|
3054
|
+
} else if (typeof e.message === "string") {
|
|
3055
|
+
rendered = e.message;
|
|
3056
|
+
} else {
|
|
3057
|
+
rendered = "The agent run failed.";
|
|
3058
|
+
}
|
|
3059
|
+
} else {
|
|
3060
|
+
rendered = "The agent run failed.";
|
|
2609
3061
|
}
|
|
2610
|
-
|
|
3062
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3063
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3064
|
+
) : void 0;
|
|
3065
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
2611
3066
|
}
|
|
2612
3067
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2613
3068
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2656,7 +3111,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2656
3111
|
if (!messages || messages.length === 0) return null;
|
|
2657
3112
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2658
3113
|
const message = messages[i];
|
|
2659
|
-
if (
|
|
3114
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2660
3115
|
const created = createdOf(message);
|
|
2661
3116
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2662
3117
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2688,25 +3143,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2688
3143
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2689
3144
|
if (!messages || messages.length === 0) return false;
|
|
2690
3145
|
return messages.some(
|
|
2691
|
-
(m) =>
|
|
3146
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2692
3147
|
);
|
|
2693
3148
|
}
|
|
2694
3149
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2695
3150
|
if (!messages || messages.length === 0) return false;
|
|
2696
|
-
const userIndex = messages.findIndex((message) =>
|
|
3151
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2697
3152
|
if (userIndex === -1) return false;
|
|
2698
3153
|
let hasLaterUser = false;
|
|
2699
3154
|
let hasStartedLaterUser = false;
|
|
2700
3155
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2701
3156
|
const message = messages[i];
|
|
2702
|
-
if (
|
|
3157
|
+
if (roleOf2(message) !== "user") continue;
|
|
2703
3158
|
hasLaterUser = true;
|
|
2704
|
-
const laterUserMessageId =
|
|
3159
|
+
const laterUserMessageId = idOf2(message);
|
|
2705
3160
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2706
3161
|
return false;
|
|
2707
3162
|
}
|
|
2708
3163
|
if (messages.some(
|
|
2709
|
-
(candidate) =>
|
|
3164
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2710
3165
|
)) {
|
|
2711
3166
|
hasStartedLaterUser = true;
|
|
2712
3167
|
}
|
|
@@ -3177,13 +3632,17 @@ async function listV2Sessions(client) {
|
|
|
3177
3632
|
}
|
|
3178
3633
|
async function deleteV2Session(client, sessionId) {
|
|
3179
3634
|
try {
|
|
3180
|
-
await client.request(
|
|
3181
|
-
|
|
3635
|
+
const response = await client.request(
|
|
3636
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3637
|
+
{ method: "DELETE" },
|
|
3638
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3639
|
+
);
|
|
3640
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3182
3641
|
} catch (error2) {
|
|
3183
3642
|
console.error(
|
|
3184
3643
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3185
3644
|
);
|
|
3186
|
-
return false;
|
|
3645
|
+
return { ok: false, status: "threw" };
|
|
3187
3646
|
}
|
|
3188
3647
|
}
|
|
3189
3648
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3805,9 +4264,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3805
4264
|
|
|
3806
4265
|
// src/lib/opencode/session-db-boot.ts
|
|
3807
4266
|
import { spawn as spawn2 } from "node:child_process";
|
|
3808
|
-
import { mkdirSync, statSync as
|
|
4267
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3809
4268
|
import { homedir as homedir4 } from "node:os";
|
|
3810
|
-
import { dirname as
|
|
4269
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3811
4270
|
|
|
3812
4271
|
// src/lib/runner-synchroniser.ts
|
|
3813
4272
|
import { spawn } from "node:child_process";
|
|
@@ -3932,7 +4391,7 @@ function clearMarker(options) {
|
|
|
3932
4391
|
function markNoReplicate(options, message) {
|
|
3933
4392
|
if (options.noReplicateMarker) {
|
|
3934
4393
|
try {
|
|
3935
|
-
mkdirSync(
|
|
4394
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3936
4395
|
writeFileSync(options.noReplicateMarker, "");
|
|
3937
4396
|
} catch (error2) {
|
|
3938
4397
|
options.log(
|
|
@@ -4057,7 +4516,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4057
4516
|
return null;
|
|
4058
4517
|
}
|
|
4059
4518
|
try {
|
|
4060
|
-
if (
|
|
4519
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4061
4520
|
} catch (error2) {
|
|
4062
4521
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4063
4522
|
options.log(
|
|
@@ -4087,7 +4546,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4087
4546
|
return null;
|
|
4088
4547
|
}
|
|
4089
4548
|
try {
|
|
4090
|
-
mkdirSync(
|
|
4549
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
4091
4550
|
writeFileSync(configPath, rendered.stdout);
|
|
4092
4551
|
} catch (error2) {
|
|
4093
4552
|
options.log(
|
|
@@ -4248,7 +4707,7 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4248
4707
|
}
|
|
4249
4708
|
function fileExists(path) {
|
|
4250
4709
|
try {
|
|
4251
|
-
|
|
4710
|
+
statSync3(path);
|
|
4252
4711
|
return true;
|
|
4253
4712
|
} catch (error2) {
|
|
4254
4713
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4337,7 +4796,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4337
4796
|
// src/lib/opencode/session-db-provenance.ts
|
|
4338
4797
|
import { createRequire } from "node:module";
|
|
4339
4798
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4340
|
-
import { dirname as
|
|
4799
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4341
4800
|
var require2 = createRequire(import.meta.url);
|
|
4342
4801
|
function readSessionDbMigrationIds(dbPath) {
|
|
4343
4802
|
let db;
|
|
@@ -4403,7 +4862,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4403
4862
|
}
|
|
4404
4863
|
function saveSessionDbProvenanceState(path, state) {
|
|
4405
4864
|
try {
|
|
4406
|
-
mkdirSync2(
|
|
4865
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4407
4866
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4408
4867
|
`, "utf8");
|
|
4409
4868
|
} catch (error2) {
|
|
@@ -4984,7 +5443,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4984
5443
|
toDelete.push(s.id);
|
|
4985
5444
|
}
|
|
4986
5445
|
}
|
|
4987
|
-
|
|
5446
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5447
|
+
return toDelete.sort(
|
|
5448
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5449
|
+
);
|
|
4988
5450
|
}
|
|
4989
5451
|
var DEFAULT_INTERVAL = "1h";
|
|
4990
5452
|
function resolve(flag, envValue, fallback) {
|
|
@@ -5044,13 +5506,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5044
5506
|
}
|
|
5045
5507
|
|
|
5046
5508
|
// src/lib/opencode/session-db-size.ts
|
|
5047
|
-
import { statSync as
|
|
5509
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5048
5510
|
import { join as join6 } from "node:path";
|
|
5049
5511
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5050
5512
|
function statSessionDbBytes(homeDir) {
|
|
5051
5513
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5052
5514
|
try {
|
|
5053
|
-
return
|
|
5515
|
+
return statSync4(dbPath).size;
|
|
5054
5516
|
} catch (err) {
|
|
5055
5517
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5056
5518
|
if (!isMissingFile) {
|
|
@@ -5076,7 +5538,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5076
5538
|
}
|
|
5077
5539
|
|
|
5078
5540
|
// src/lib/opencode/log-tail.ts
|
|
5079
|
-
import { statSync as
|
|
5541
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5080
5542
|
import { homedir as homedir5 } from "node:os";
|
|
5081
5543
|
import { join as join7 } from "node:path";
|
|
5082
5544
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -5099,7 +5561,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5099
5561
|
let inode = null;
|
|
5100
5562
|
let baselineReady = true;
|
|
5101
5563
|
try {
|
|
5102
|
-
const initial =
|
|
5564
|
+
const initial = statSync5(logPath);
|
|
5103
5565
|
offset = initial.size;
|
|
5104
5566
|
inode = initial.ino;
|
|
5105
5567
|
} catch (error2) {
|
|
@@ -5158,129 +5620,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5158
5620
|
};
|
|
5159
5621
|
}
|
|
5160
5622
|
|
|
5161
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5162
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5163
|
-
import { dirname as dirname4 } from "node:path";
|
|
5164
|
-
function errorMessage(error2) {
|
|
5165
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5166
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5167
|
-
}
|
|
5168
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5169
|
-
try {
|
|
5170
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5171
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5172
|
-
if (availableBytes < requiredBytes) {
|
|
5173
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5174
|
-
}
|
|
5175
|
-
return null;
|
|
5176
|
-
} catch (err) {
|
|
5177
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5178
|
-
}
|
|
5179
|
-
}
|
|
5180
|
-
function readLogicalBytes(db) {
|
|
5181
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5182
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5183
|
-
return pageCount * pageSize;
|
|
5184
|
-
}
|
|
5185
|
-
function readCheckpointResult(db) {
|
|
5186
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5187
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5188
|
-
}
|
|
5189
|
-
async function probeReclaimAvailability(input) {
|
|
5190
|
-
const { dbPath, requiredBytes } = input;
|
|
5191
|
-
let sqlite;
|
|
5192
|
-
try {
|
|
5193
|
-
sqlite = await import("node:sqlite");
|
|
5194
|
-
} catch (err) {
|
|
5195
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5196
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5197
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5198
|
-
}
|
|
5199
|
-
let autoVacuum = null;
|
|
5200
|
-
try {
|
|
5201
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5202
|
-
try {
|
|
5203
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5204
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5205
|
-
} finally {
|
|
5206
|
-
db.close();
|
|
5207
|
-
}
|
|
5208
|
-
} catch (err) {
|
|
5209
|
-
console.warn(
|
|
5210
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5211
|
-
);
|
|
5212
|
-
}
|
|
5213
|
-
if (autoVacuum !== 0) return null;
|
|
5214
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5215
|
-
}
|
|
5216
|
-
async function reclaimSessionDbSpace(input) {
|
|
5217
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5218
|
-
let sqlite;
|
|
5219
|
-
try {
|
|
5220
|
-
sqlite = await import("node:sqlite");
|
|
5221
|
-
} catch (err) {
|
|
5222
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5223
|
-
console.warn(
|
|
5224
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5225
|
-
);
|
|
5226
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5227
|
-
}
|
|
5228
|
-
const { DatabaseSync } = sqlite;
|
|
5229
|
-
let db;
|
|
5230
|
-
try {
|
|
5231
|
-
db = new DatabaseSync(dbPath);
|
|
5232
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5233
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5234
|
-
if (autoVacuum === 0) {
|
|
5235
|
-
if (!allowFullVacuum) {
|
|
5236
|
-
console.warn(
|
|
5237
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5238
|
-
);
|
|
5239
|
-
return {
|
|
5240
|
-
ok: false,
|
|
5241
|
-
skipped: "full-vacuum-blocked",
|
|
5242
|
-
detail: "a session turn is live"
|
|
5243
|
-
};
|
|
5244
|
-
}
|
|
5245
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5246
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5247
|
-
if (skipReason !== null) {
|
|
5248
|
-
console.warn(
|
|
5249
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5250
|
-
);
|
|
5251
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5252
|
-
}
|
|
5253
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5254
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5255
|
-
db.exec("VACUUM");
|
|
5256
|
-
const afterBytes = readLogicalBytes(db);
|
|
5257
|
-
const checkpoint = readCheckpointResult(db);
|
|
5258
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5259
|
-
}
|
|
5260
|
-
if (autoVacuum === 2) {
|
|
5261
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5262
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5263
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5264
|
-
const afterBytes = readLogicalBytes(db);
|
|
5265
|
-
const checkpoint = readCheckpointResult(db);
|
|
5266
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5267
|
-
}
|
|
5268
|
-
console.warn(
|
|
5269
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5270
|
-
);
|
|
5271
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5272
|
-
} catch (err) {
|
|
5273
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5274
|
-
return {
|
|
5275
|
-
ok: false,
|
|
5276
|
-
skipped: "reclaim-error",
|
|
5277
|
-
detail: errorMessage(err)
|
|
5278
|
-
};
|
|
5279
|
-
} finally {
|
|
5280
|
-
db?.close();
|
|
5281
|
-
}
|
|
5282
|
-
}
|
|
5283
|
-
|
|
5284
5623
|
// src/lib/tunnel/connection.ts
|
|
5285
5624
|
import WebSocket2 from "ws";
|
|
5286
5625
|
|
|
@@ -6170,6 +6509,83 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6170
6509
|
};
|
|
6171
6510
|
}
|
|
6172
6511
|
|
|
6512
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6513
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6514
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6515
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6516
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6517
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6518
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6519
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6520
|
+
var BAND_RANK = {
|
|
6521
|
+
low: 1,
|
|
6522
|
+
very_low: 2,
|
|
6523
|
+
critical: 3
|
|
6524
|
+
};
|
|
6525
|
+
function bandFor(freeBytes) {
|
|
6526
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6527
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6528
|
+
return "low";
|
|
6529
|
+
}
|
|
6530
|
+
function formatBytes(bytes) {
|
|
6531
|
+
if (bytes === null) return "unavailable";
|
|
6532
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6533
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6534
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6535
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6536
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6537
|
+
}
|
|
6538
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6539
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6540
|
+
}
|
|
6541
|
+
function evaluateLowDiskWarning(input) {
|
|
6542
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6543
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6544
|
+
return { warning: null, state };
|
|
6545
|
+
}
|
|
6546
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6547
|
+
if (recovered) {
|
|
6548
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6549
|
+
}
|
|
6550
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6551
|
+
if (!low) return { warning: null, state };
|
|
6552
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6553
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6554
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6555
|
+
const worse = currentRank > previousRank;
|
|
6556
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6557
|
+
const nextState = {
|
|
6558
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6559
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6560
|
+
};
|
|
6561
|
+
return {
|
|
6562
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6563
|
+
state: nextState
|
|
6564
|
+
};
|
|
6565
|
+
}
|
|
6566
|
+
|
|
6567
|
+
// src/lib/opencode/session-store-health.ts
|
|
6568
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6569
|
+
function createSessionStoreHealth(options = {}) {
|
|
6570
|
+
const now = options.now ?? (() => Date.now());
|
|
6571
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6572
|
+
let lastUnwritableAt = null;
|
|
6573
|
+
let lastWritableAt = null;
|
|
6574
|
+
return {
|
|
6575
|
+
observeUnwritable: () => {
|
|
6576
|
+
lastUnwritableAt = now();
|
|
6577
|
+
},
|
|
6578
|
+
observeWritable: () => {
|
|
6579
|
+
lastWritableAt = now();
|
|
6580
|
+
},
|
|
6581
|
+
isUnwritable: () => {
|
|
6582
|
+
if (lastUnwritableAt === null) return false;
|
|
6583
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6584
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6585
|
+
}
|
|
6586
|
+
};
|
|
6587
|
+
}
|
|
6588
|
+
|
|
6173
6589
|
// src/lib/channels/driver.ts
|
|
6174
6590
|
import { homedir as homedir6 } from "node:os";
|
|
6175
6591
|
|
|
@@ -6628,6 +7044,7 @@ var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
|
6628
7044
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
6629
7045
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6630
7046
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7047
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6631
7048
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6632
7049
|
var HEARTBEAT_MS = 6e4;
|
|
6633
7050
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6689,6 +7106,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6689
7106
|
maxActiveSessions;
|
|
6690
7107
|
watcherStallMs;
|
|
6691
7108
|
wedgeWarningIntervalMs;
|
|
7109
|
+
sessionStoreHealth;
|
|
6692
7110
|
openCodeClient;
|
|
6693
7111
|
/** Cache of conversationId → opencode sessionId. */
|
|
6694
7112
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6813,10 +7231,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6813
7231
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6814
7232
|
*/
|
|
6815
7233
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7234
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7235
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6816
7236
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6817
7237
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6818
7238
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6819
7239
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7240
|
+
/**
|
|
7241
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7242
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7243
|
+
* identical-text turn, so it never produces a correlation.
|
|
7244
|
+
*/
|
|
7245
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6820
7246
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6821
7247
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6822
7248
|
/**
|
|
@@ -6902,21 +7328,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6902
7328
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6903
7329
|
*/
|
|
6904
7330
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6905
|
-
/**
|
|
6906
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6907
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6908
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6909
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6910
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6911
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6912
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6913
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6914
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6915
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6916
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6917
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6918
|
-
*/
|
|
6919
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6920
7331
|
/**
|
|
6921
7332
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6922
7333
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6934,8 +7345,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6934
7345
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6935
7346
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6936
7347
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6937
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6938
|
-
*
|
|
7348
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7349
|
+
* the terminal-report fence and is never sent again).
|
|
6939
7350
|
*/
|
|
6940
7351
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6941
7352
|
/**
|
|
@@ -7058,6 +7469,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7058
7469
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7059
7470
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7060
7471
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7472
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7061
7473
|
}
|
|
7062
7474
|
get isV2() {
|
|
7063
7475
|
return this.openCodeClient.version === "v2";
|
|
@@ -7073,7 +7485,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7073
7485
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7074
7486
|
}
|
|
7075
7487
|
async listSessions() {
|
|
7076
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7488
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7489
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7490
|
+
});
|
|
7077
7491
|
}
|
|
7078
7492
|
async sessionExists(sessionId) {
|
|
7079
7493
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7250,6 +7664,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7250
7664
|
}
|
|
7251
7665
|
return ids;
|
|
7252
7666
|
}
|
|
7667
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7668
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7669
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7670
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7671
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7672
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7673
|
+
}
|
|
7674
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7675
|
+
}
|
|
7676
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7677
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7678
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7679
|
+
this.awaitingReadopt.delete(messageId);
|
|
7680
|
+
this.sessions.delete(conversationId);
|
|
7681
|
+
this.supersede(conversationId, sessionId);
|
|
7682
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7683
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7684
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7685
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7686
|
+
this.log({
|
|
7687
|
+
level: "error",
|
|
7688
|
+
message: errorMessage3,
|
|
7689
|
+
conversation_id: conversationId,
|
|
7690
|
+
message_id: messageId
|
|
7691
|
+
});
|
|
7692
|
+
const reportError = await this.reportUntrackableAck(
|
|
7693
|
+
conversationId,
|
|
7694
|
+
messageId,
|
|
7695
|
+
null,
|
|
7696
|
+
errorMessage3
|
|
7697
|
+
);
|
|
7698
|
+
if (reportError !== null) {
|
|
7699
|
+
this.log({
|
|
7700
|
+
level: "warn",
|
|
7701
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7702
|
+
conversation_id: conversationId,
|
|
7703
|
+
message_id: messageId
|
|
7704
|
+
});
|
|
7705
|
+
}
|
|
7706
|
+
if (source === "readopt") {
|
|
7707
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7708
|
+
}
|
|
7709
|
+
}
|
|
7253
7710
|
/**
|
|
7254
7711
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7255
7712
|
*
|
|
@@ -7429,6 +7886,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7429
7886
|
}
|
|
7430
7887
|
break;
|
|
7431
7888
|
}
|
|
7889
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7890
|
+
if (correlation) {
|
|
7891
|
+
correlation.attempts += 1;
|
|
7892
|
+
let correlatedId = null;
|
|
7893
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7894
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7895
|
+
conv,
|
|
7896
|
+
message,
|
|
7897
|
+
correlation.sessionId
|
|
7898
|
+
);
|
|
7899
|
+
if (polled.ok)
|
|
7900
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7901
|
+
} else if (correlation.knownUserIds === null) {
|
|
7902
|
+
this.log({
|
|
7903
|
+
level: "debug",
|
|
7904
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7905
|
+
conversation_id: conv.id,
|
|
7906
|
+
message_id: message.id
|
|
7907
|
+
});
|
|
7908
|
+
}
|
|
7909
|
+
if (correlatedId !== null) {
|
|
7910
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7911
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7912
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7913
|
+
this.dispatched.add(message.id);
|
|
7914
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7915
|
+
dispatched += 1;
|
|
7916
|
+
this.log({
|
|
7917
|
+
level: "info",
|
|
7918
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7919
|
+
conversation_id: conv.id,
|
|
7920
|
+
message_id: message.id
|
|
7921
|
+
});
|
|
7922
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7923
|
+
opencode_message_id: correlatedId
|
|
7924
|
+
});
|
|
7925
|
+
continue;
|
|
7926
|
+
}
|
|
7927
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7928
|
+
await this.handoffUntrackableCorrelation(
|
|
7929
|
+
conv.id,
|
|
7930
|
+
message.id,
|
|
7931
|
+
correlation.sessionId,
|
|
7932
|
+
correlation.attempts,
|
|
7933
|
+
"fresh"
|
|
7934
|
+
);
|
|
7935
|
+
break;
|
|
7936
|
+
}
|
|
7937
|
+
break;
|
|
7938
|
+
}
|
|
7432
7939
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7433
7940
|
if (effectiveOpencodeMessageId) {
|
|
7434
7941
|
const outcome = await this.resolveRedrive(
|
|
@@ -7450,6 +7957,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7450
7957
|
model: message.opencode_model ?? void 0
|
|
7451
7958
|
};
|
|
7452
7959
|
let opencodeMessageId;
|
|
7960
|
+
let preDispatchUserIds = null;
|
|
7453
7961
|
try {
|
|
7454
7962
|
this.log({
|
|
7455
7963
|
level: "info",
|
|
@@ -7479,7 +7987,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7479
7987
|
message.content,
|
|
7480
7988
|
options,
|
|
7481
7989
|
sendAttachments,
|
|
7482
|
-
this.openCodeClient
|
|
7990
|
+
this.openCodeClient,
|
|
7991
|
+
{
|
|
7992
|
+
onKnownUserIds: (ids) => {
|
|
7993
|
+
preDispatchUserIds = ids;
|
|
7994
|
+
},
|
|
7995
|
+
onKnownUserIdsError: (error2) => {
|
|
7996
|
+
this.log({
|
|
7997
|
+
level: "warn",
|
|
7998
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
7999
|
+
conversation_id: conv.id,
|
|
8000
|
+
message_id: message.id
|
|
8001
|
+
});
|
|
8002
|
+
}
|
|
8003
|
+
}
|
|
7483
8004
|
)
|
|
7484
8005
|
);
|
|
7485
8006
|
} catch (err) {
|
|
@@ -7487,6 +8008,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7487
8008
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7488
8009
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7489
8010
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8011
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7490
8012
|
this.log({
|
|
7491
8013
|
level: "error",
|
|
7492
8014
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7562,39 +8084,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7562
8084
|
if (this.isV2) {
|
|
7563
8085
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7564
8086
|
}
|
|
7565
|
-
|
|
7566
|
-
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
7570
|
-
|
|
7571
|
-
message_id: message.id
|
|
7572
|
-
});
|
|
7573
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7574
|
-
continue;
|
|
7575
|
-
}
|
|
7576
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7577
|
-
this.sessions.delete(conv.id);
|
|
7578
|
-
this.supersede(conv.id, sessionId);
|
|
7579
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8087
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8088
|
+
sessionId,
|
|
8089
|
+
content: message.content,
|
|
8090
|
+
attempts: 0,
|
|
8091
|
+
knownUserIds: preDispatchUserIds
|
|
8092
|
+
});
|
|
7580
8093
|
this.log({
|
|
7581
|
-
level: "
|
|
7582
|
-
message:
|
|
8094
|
+
level: "warn",
|
|
8095
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7583
8096
|
conversation_id: conv.id,
|
|
7584
8097
|
message_id: message.id
|
|
7585
8098
|
});
|
|
7586
|
-
|
|
7587
|
-
this.log({
|
|
7588
|
-
level: "warn",
|
|
7589
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7590
|
-
conversation_id: conv.id,
|
|
7591
|
-
message_id: message.id
|
|
7592
|
-
});
|
|
7593
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7594
|
-
});
|
|
8099
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7595
8100
|
break;
|
|
7596
8101
|
}
|
|
7597
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7598
8102
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7599
8103
|
this.releasedOpencodeIds.delete(message.id);
|
|
7600
8104
|
this.dispatched.add(message.id);
|
|
@@ -8199,23 +8703,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8199
8703
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8200
8704
|
return 1;
|
|
8201
8705
|
}
|
|
8202
|
-
/**
|
|
8203
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8204
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8205
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8206
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8207
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8208
|
-
* count, since a new session is a genuinely different attempt.
|
|
8209
|
-
*/
|
|
8210
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8211
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8212
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8213
|
-
existing.count += 1;
|
|
8214
|
-
return existing.count;
|
|
8215
|
-
}
|
|
8216
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8217
|
-
return 1;
|
|
8218
|
-
}
|
|
8219
8706
|
/** Record one queued give-up toward the never-started bound. */
|
|
8220
8707
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8221
8708
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8527,8 +9014,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8527
9014
|
b2LastDescendantCheckMs: 0,
|
|
8528
9015
|
b2RootOngoingHeldLogged: false,
|
|
8529
9016
|
b2AbandonedSignalled: false,
|
|
9017
|
+
b2AbandonedWithoutAnswer: false,
|
|
8530
9018
|
ambiguousPinnedSinceMs: 0,
|
|
8531
9019
|
ambiguousResolved: false,
|
|
9020
|
+
sessionStoreFailurePending: false,
|
|
8532
9021
|
neverStartedResolved: false
|
|
8533
9022
|
});
|
|
8534
9023
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8645,8 +9134,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8645
9134
|
b2LastDescendantCheckMs: 0,
|
|
8646
9135
|
b2RootOngoingHeldLogged: false,
|
|
8647
9136
|
b2AbandonedSignalled: false,
|
|
9137
|
+
b2AbandonedWithoutAnswer: false,
|
|
8648
9138
|
ambiguousPinnedSinceMs: 0,
|
|
8649
9139
|
ambiguousResolved: false,
|
|
9140
|
+
sessionStoreFailurePending: false,
|
|
8650
9141
|
neverStartedResolved: false
|
|
8651
9142
|
});
|
|
8652
9143
|
}
|
|
@@ -9060,6 +9551,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9060
9551
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9061
9552
|
this.neverStartedAttempts.delete(id);
|
|
9062
9553
|
}
|
|
9554
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9555
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9556
|
+
return;
|
|
9557
|
+
}
|
|
9063
9558
|
if (inFlight.neverStartedResolved) {
|
|
9064
9559
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9065
9560
|
inFlight.neverStartedResolved = false;
|
|
@@ -9219,10 +9714,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9219
9714
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9220
9715
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9221
9716
|
inFlight.b2AbandonedSignalled = false;
|
|
9717
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9222
9718
|
}
|
|
9223
9719
|
} else {
|
|
9224
9720
|
if (inFlight.b2AbandonedSignalled) {
|
|
9225
|
-
|
|
9721
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9722
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9723
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9724
|
+
sessionId,
|
|
9725
|
+
watcher,
|
|
9726
|
+
inFlight,
|
|
9727
|
+
messages,
|
|
9728
|
+
reply?.info?.finish ?? reply?.finish
|
|
9729
|
+
);
|
|
9730
|
+
} else {
|
|
9731
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9732
|
+
}
|
|
9226
9733
|
return;
|
|
9227
9734
|
}
|
|
9228
9735
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9239,17 +9746,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9239
9746
|
descendantOngoing,
|
|
9240
9747
|
rootOngoing
|
|
9241
9748
|
})) {
|
|
9749
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9750
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9751
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9752
|
+
watched_for_ms: pinnedForMs,
|
|
9753
|
+
run_state: "failed",
|
|
9754
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9755
|
+
});
|
|
9756
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9757
|
+
return;
|
|
9758
|
+
}
|
|
9242
9759
|
inFlight.b2AbandonedSignalled = true;
|
|
9760
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9761
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9762
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9763
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9764
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9765
|
+
watched_for_ms: pinnedForMs,
|
|
9766
|
+
run_state: "failed",
|
|
9767
|
+
finish,
|
|
9768
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9769
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9770
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9771
|
+
});
|
|
9772
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9773
|
+
sessionId,
|
|
9774
|
+
watcher,
|
|
9775
|
+
inFlight,
|
|
9776
|
+
messages,
|
|
9777
|
+
finish
|
|
9778
|
+
);
|
|
9779
|
+
return;
|
|
9780
|
+
}
|
|
9243
9781
|
this.log({
|
|
9244
9782
|
level: "warn",
|
|
9245
9783
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9246
9784
|
conversation_id: conv.id,
|
|
9247
9785
|
message_id: id
|
|
9248
9786
|
});
|
|
9249
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9250
9787
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9251
9788
|
watched_for_ms: pinnedForMs,
|
|
9252
|
-
finish
|
|
9789
|
+
finish,
|
|
9253
9790
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9254
9791
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9255
9792
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9297,6 +9834,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9297
9834
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9298
9835
|
sessionOngoing: ongoing
|
|
9299
9836
|
})) {
|
|
9837
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9838
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9839
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9840
|
+
watched_for_ms: pinnedForMs,
|
|
9841
|
+
run_state: "failed",
|
|
9842
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9843
|
+
});
|
|
9844
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9845
|
+
return;
|
|
9846
|
+
}
|
|
9300
9847
|
inFlight.ambiguousResolved = true;
|
|
9301
9848
|
this.log({
|
|
9302
9849
|
level: "warn",
|
|
@@ -9521,6 +10068,74 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9521
10068
|
}
|
|
9522
10069
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9523
10070
|
}
|
|
10071
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10072
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10073
|
+
const conv = watcher.conv;
|
|
10074
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
10075
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10076
|
+
this.log({
|
|
10077
|
+
level: "warn",
|
|
10078
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10079
|
+
conversation_id: conv.id,
|
|
10080
|
+
message_id: inFlight.evidentMessageId
|
|
10081
|
+
});
|
|
10082
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10083
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10084
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10085
|
+
messages,
|
|
10086
|
+
inFlight.opencodeMessageId,
|
|
10087
|
+
inFlight.evidentMessageId
|
|
10088
|
+
);
|
|
10089
|
+
try {
|
|
10090
|
+
await this.markFailed(
|
|
10091
|
+
conv.id,
|
|
10092
|
+
inFlight.evidentMessageId,
|
|
10093
|
+
sessionId,
|
|
10094
|
+
error2,
|
|
10095
|
+
usage,
|
|
10096
|
+
void 0,
|
|
10097
|
+
usageAgentName,
|
|
10098
|
+
subagentInvocations
|
|
10099
|
+
);
|
|
10100
|
+
} catch (err) {
|
|
10101
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10102
|
+
if (err instanceof ChannelTerminalError) {
|
|
10103
|
+
this.log({
|
|
10104
|
+
level: "warn",
|
|
10105
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10106
|
+
conversation_id: conv.id,
|
|
10107
|
+
message_id: inFlight.evidentMessageId
|
|
10108
|
+
});
|
|
10109
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10110
|
+
return;
|
|
10111
|
+
}
|
|
10112
|
+
if (this.now() >= inFlight.deadline) {
|
|
10113
|
+
this.log({
|
|
10114
|
+
level: "warn",
|
|
10115
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10116
|
+
conversation_id: conv.id,
|
|
10117
|
+
message_id: inFlight.evidentMessageId
|
|
10118
|
+
});
|
|
10119
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10120
|
+
return;
|
|
10121
|
+
}
|
|
10122
|
+
this.log({
|
|
10123
|
+
level: "warn",
|
|
10124
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10125
|
+
conversation_id: conv.id,
|
|
10126
|
+
message_id: inFlight.evidentMessageId
|
|
10127
|
+
});
|
|
10128
|
+
return;
|
|
10129
|
+
}
|
|
10130
|
+
inFlight.done = true;
|
|
10131
|
+
await this.reportSubagentAuthFailures(
|
|
10132
|
+
watcher.conv.id,
|
|
10133
|
+
inFlight.opencodeMessageId,
|
|
10134
|
+
inFlight.evidentMessageId,
|
|
10135
|
+
messages
|
|
10136
|
+
);
|
|
10137
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10138
|
+
}
|
|
9524
10139
|
/**
|
|
9525
10140
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9526
10141
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9576,6 +10191,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9576
10191
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9577
10192
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9578
10193
|
}
|
|
10194
|
+
/**
|
|
10195
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10196
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10197
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10198
|
+
*/
|
|
10199
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10200
|
+
const conv = watcher.conv;
|
|
10201
|
+
const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
10202
|
+
this.log({
|
|
10203
|
+
level: "error",
|
|
10204
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10205
|
+
conversation_id: conv.id,
|
|
10206
|
+
message_id: inFlight.evidentMessageId
|
|
10207
|
+
});
|
|
10208
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10209
|
+
try {
|
|
10210
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10211
|
+
} catch (err) {
|
|
10212
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10213
|
+
if (err instanceof ChannelTerminalError) {
|
|
10214
|
+
this.log({
|
|
10215
|
+
level: "warn",
|
|
10216
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10217
|
+
conversation_id: conv.id,
|
|
10218
|
+
message_id: inFlight.evidentMessageId
|
|
10219
|
+
});
|
|
10220
|
+
this.recordReleasedOpencodeId(
|
|
10221
|
+
inFlight.evidentMessageId,
|
|
10222
|
+
sessionId,
|
|
10223
|
+
inFlight.opencodeMessageId
|
|
10224
|
+
);
|
|
10225
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10226
|
+
return;
|
|
10227
|
+
}
|
|
10228
|
+
if (this.now() >= inFlight.deadline) {
|
|
10229
|
+
this.log({
|
|
10230
|
+
level: "warn",
|
|
10231
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10232
|
+
conversation_id: conv.id,
|
|
10233
|
+
message_id: inFlight.evidentMessageId
|
|
10234
|
+
});
|
|
10235
|
+
this.recordReleasedOpencodeId(
|
|
10236
|
+
inFlight.evidentMessageId,
|
|
10237
|
+
sessionId,
|
|
10238
|
+
inFlight.opencodeMessageId
|
|
10239
|
+
);
|
|
10240
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10241
|
+
return;
|
|
10242
|
+
}
|
|
10243
|
+
this.log({
|
|
10244
|
+
level: "warn",
|
|
10245
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10246
|
+
conversation_id: conv.id,
|
|
10247
|
+
message_id: inFlight.evidentMessageId
|
|
10248
|
+
});
|
|
10249
|
+
return;
|
|
10250
|
+
}
|
|
10251
|
+
inFlight.done = true;
|
|
10252
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10253
|
+
}
|
|
9579
10254
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9580
10255
|
/**
|
|
9581
10256
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9614,11 +10289,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9614
10289
|
}
|
|
9615
10290
|
}
|
|
9616
10291
|
}
|
|
9617
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10292
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9618
10293
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10294
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10295
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10296
|
+
this.awaitingCorrelation.delete(id);
|
|
10297
|
+
this.log({
|
|
10298
|
+
level: "debug",
|
|
10299
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10300
|
+
message_id: id
|
|
10301
|
+
});
|
|
10302
|
+
}
|
|
10303
|
+
}
|
|
9619
10304
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9620
10305
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9621
10306
|
this.untrackableAck.delete(id);
|
|
10307
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9622
10308
|
this.untrackableAckSignalled.delete(id);
|
|
9623
10309
|
this.untrackableAckWarned.delete(id);
|
|
9624
10310
|
this.log({
|
|
@@ -9658,7 +10344,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9658
10344
|
bySession.set(row.opencode_session_id, list);
|
|
9659
10345
|
}
|
|
9660
10346
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9661
|
-
let messages;
|
|
10347
|
+
let messages = null;
|
|
9662
10348
|
try {
|
|
9663
10349
|
if (this.isV2) {
|
|
9664
10350
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9667,9 +10353,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9667
10353
|
level: "warn",
|
|
9668
10354
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9669
10355
|
});
|
|
9670
|
-
|
|
10356
|
+
} else {
|
|
10357
|
+
messages = snapshot;
|
|
9671
10358
|
}
|
|
9672
|
-
messages = snapshot;
|
|
9673
10359
|
} else {
|
|
9674
10360
|
const polled = await pollSessionMessagesForRedrive(
|
|
9675
10361
|
this.port,
|
|
@@ -9682,15 +10368,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9682
10368
|
level: "warn",
|
|
9683
10369
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9684
10370
|
});
|
|
9685
|
-
|
|
10371
|
+
} else {
|
|
10372
|
+
messages = polled.messages;
|
|
9686
10373
|
}
|
|
9687
|
-
messages = polled.messages;
|
|
9688
10374
|
}
|
|
9689
10375
|
} catch (err) {
|
|
9690
10376
|
this.log({
|
|
9691
10377
|
level: "warn",
|
|
9692
10378
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9693
10379
|
});
|
|
10380
|
+
}
|
|
10381
|
+
if (messages === null) {
|
|
10382
|
+
for (const row of sessionRows) {
|
|
10383
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10384
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10385
|
+
}
|
|
10386
|
+
}
|
|
9694
10387
|
continue;
|
|
9695
10388
|
}
|
|
9696
10389
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9700,6 +10393,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9700
10393
|
}
|
|
9701
10394
|
}
|
|
9702
10395
|
}
|
|
10396
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10397
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10398
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10399
|
+
if (!correlation) return false;
|
|
10400
|
+
correlation.attempts += 1;
|
|
10401
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10402
|
+
if (correlatedId !== null) {
|
|
10403
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10404
|
+
const conv = this.convForRow(sessionId, row);
|
|
10405
|
+
const message = this.queuedMessageForRow(row);
|
|
10406
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10407
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10408
|
+
this.dispatched.add(row.id);
|
|
10409
|
+
this.readopted.add(row.id);
|
|
10410
|
+
this.ensureWatcherRunning(sessionId);
|
|
10411
|
+
this.log({
|
|
10412
|
+
level: "info",
|
|
10413
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10414
|
+
conversation_id: row.conversation_id,
|
|
10415
|
+
message_id: row.id
|
|
10416
|
+
});
|
|
10417
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10418
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10419
|
+
opencode_message_id: correlatedId
|
|
10420
|
+
});
|
|
10421
|
+
return true;
|
|
10422
|
+
}
|
|
10423
|
+
if (correlation.knownUserIds === null) {
|
|
10424
|
+
this.log({
|
|
10425
|
+
level: "debug",
|
|
10426
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10427
|
+
conversation_id: row.conversation_id,
|
|
10428
|
+
message_id: row.id
|
|
10429
|
+
});
|
|
10430
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10431
|
+
this.log({
|
|
10432
|
+
level: "warn",
|
|
10433
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10434
|
+
conversation_id: row.conversation_id,
|
|
10435
|
+
message_id: row.id
|
|
10436
|
+
});
|
|
10437
|
+
}
|
|
10438
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10439
|
+
await this.handoffUntrackableCorrelation(
|
|
10440
|
+
row.conversation_id,
|
|
10441
|
+
row.id,
|
|
10442
|
+
correlation.sessionId,
|
|
10443
|
+
correlation.attempts,
|
|
10444
|
+
"readopt"
|
|
10445
|
+
);
|
|
10446
|
+
}
|
|
10447
|
+
return true;
|
|
10448
|
+
}
|
|
9703
10449
|
/**
|
|
9704
10450
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9705
10451
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9707,8 +10453,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9707
10453
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9708
10454
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9709
10455
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9710
|
-
* before the restart)
|
|
9711
|
-
*
|
|
10456
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10457
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10458
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10459
|
+
* `forceReadoptRun`):
|
|
9712
10460
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9713
10461
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9714
10462
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9717,7 +10465,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9717
10465
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9718
10466
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9719
10467
|
* tracking the stored id so the reply correlates by it;
|
|
9720
|
-
* - `unknown`/null id →
|
|
10468
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10469
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9721
10470
|
*
|
|
9722
10471
|
* Only `ChannelAuthError` propagates.
|
|
9723
10472
|
*/
|
|
@@ -9731,6 +10480,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9731
10480
|
});
|
|
9732
10481
|
return;
|
|
9733
10482
|
}
|
|
10483
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9734
10484
|
const ocId = row.opencode_message_id;
|
|
9735
10485
|
const state = messageRunState(messages, ocId ?? "");
|
|
9736
10486
|
if (state === "done") {
|
|
@@ -9797,12 +10547,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9797
10547
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9798
10548
|
this.dontRedispatch.delete(row.id);
|
|
9799
10549
|
this.untrackableAck.delete(row.id);
|
|
10550
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9800
10551
|
this.untrackableAckSignalled.delete(row.id);
|
|
9801
10552
|
this.untrackableAckWarned.delete(row.id);
|
|
9802
10553
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9803
10554
|
return;
|
|
9804
10555
|
}
|
|
9805
|
-
if (this.
|
|
10556
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10557
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10558
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10559
|
+
this.untrackableAckWarned.add(row.id);
|
|
10560
|
+
this.log({
|
|
10561
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10562
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10563
|
+
conversation_id: row.conversation_id,
|
|
10564
|
+
message_id: row.id
|
|
10565
|
+
});
|
|
10566
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10567
|
+
const reportError = await this.reportUntrackableAck(
|
|
10568
|
+
row.conversation_id,
|
|
10569
|
+
row.id,
|
|
10570
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10571
|
+
errorMessage3
|
|
10572
|
+
);
|
|
10573
|
+
if (reportError !== null) {
|
|
10574
|
+
this.log({
|
|
10575
|
+
level: "warn",
|
|
10576
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10577
|
+
conversation_id: row.conversation_id,
|
|
10578
|
+
message_id: row.id
|
|
10579
|
+
});
|
|
10580
|
+
}
|
|
10581
|
+
} else {
|
|
10582
|
+
this.log({
|
|
10583
|
+
level: "debug",
|
|
10584
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10585
|
+
conversation_id: row.conversation_id,
|
|
10586
|
+
message_id: row.id
|
|
10587
|
+
});
|
|
10588
|
+
}
|
|
10589
|
+
return;
|
|
10590
|
+
}
|
|
10591
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9806
10592
|
this.log({
|
|
9807
10593
|
level: "debug",
|
|
9808
10594
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9980,6 +10766,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9980
10766
|
}
|
|
9981
10767
|
this.dontRedispatch.delete(row.id);
|
|
9982
10768
|
this.untrackableAck.delete(row.id);
|
|
10769
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9983
10770
|
this.untrackableAckSignalled.delete(row.id);
|
|
9984
10771
|
this.untrackableAckWarned.delete(row.id);
|
|
9985
10772
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9999,8 +10786,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9999
10786
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
10000
10787
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
10001
10788
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
10002
|
-
* re-entry)
|
|
10003
|
-
*
|
|
10789
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10790
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10791
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
10004
10792
|
*
|
|
10005
10793
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
10006
10794
|
* `processed_at` (Invariant 1).
|
|
@@ -10063,6 +10851,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10063
10851
|
);
|
|
10064
10852
|
}
|
|
10065
10853
|
let ocId;
|
|
10854
|
+
let preDispatchUserIds = null;
|
|
10066
10855
|
try {
|
|
10067
10856
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10068
10857
|
sessionId,
|
|
@@ -10072,7 +10861,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10072
10861
|
row.content,
|
|
10073
10862
|
options,
|
|
10074
10863
|
sendAttachments,
|
|
10075
|
-
this.openCodeClient
|
|
10864
|
+
this.openCodeClient,
|
|
10865
|
+
{
|
|
10866
|
+
onKnownUserIds: (ids) => {
|
|
10867
|
+
preDispatchUserIds = ids;
|
|
10868
|
+
},
|
|
10869
|
+
onKnownUserIdsError: (error2) => {
|
|
10870
|
+
this.log({
|
|
10871
|
+
level: "warn",
|
|
10872
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10873
|
+
conversation_id: row.conversation_id,
|
|
10874
|
+
message_id: row.id
|
|
10875
|
+
});
|
|
10876
|
+
}
|
|
10877
|
+
}
|
|
10076
10878
|
)
|
|
10077
10879
|
);
|
|
10078
10880
|
} catch (err) {
|
|
@@ -10105,6 +10907,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10105
10907
|
}
|
|
10106
10908
|
} else {
|
|
10107
10909
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
10910
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10108
10911
|
}
|
|
10109
10912
|
this.log({
|
|
10110
10913
|
level: "error",
|
|
@@ -10153,39 +10956,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10153
10956
|
}
|
|
10154
10957
|
if (ocId === null) {
|
|
10155
10958
|
this.awaitingReadopt.delete(row.id);
|
|
10156
|
-
|
|
10157
|
-
|
|
10158
|
-
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
this.log({
|
|
10163
|
-
level: "error",
|
|
10164
|
-
message: errorMessage3,
|
|
10165
|
-
conversation_id: row.conversation_id,
|
|
10166
|
-
message_id: row.id
|
|
10167
|
-
});
|
|
10168
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10169
|
-
this.log({
|
|
10170
|
-
level: "warn",
|
|
10171
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10172
|
-
conversation_id: row.conversation_id,
|
|
10173
|
-
message_id: row.id
|
|
10174
|
-
});
|
|
10175
|
-
});
|
|
10176
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10177
|
-
return;
|
|
10178
|
-
}
|
|
10959
|
+
this.awaitingCorrelation.set(row.id, {
|
|
10960
|
+
sessionId,
|
|
10961
|
+
content: row.content,
|
|
10962
|
+
attempts: 0,
|
|
10963
|
+
knownUserIds: preDispatchUserIds
|
|
10964
|
+
});
|
|
10179
10965
|
this.log({
|
|
10180
10966
|
level: "warn",
|
|
10181
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
10967
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10182
10968
|
conversation_id: row.conversation_id,
|
|
10183
10969
|
message_id: row.id
|
|
10184
10970
|
});
|
|
10185
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10186
10971
|
return;
|
|
10187
10972
|
}
|
|
10188
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10189
10973
|
const processedAtMs = this.processedAtMs(row);
|
|
10190
10974
|
this.registerReadopted(
|
|
10191
10975
|
readoptConv,
|
|
@@ -12780,9 +13564,9 @@ async function driveChannels(state, driver) {
|
|
|
12780
13564
|
}
|
|
12781
13565
|
}
|
|
12782
13566
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12783
|
-
var
|
|
13567
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12784
13568
|
function shouldWarnForReclaimSkip(reason) {
|
|
12785
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13569
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12786
13570
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12787
13571
|
if (!version2) return false;
|
|
12788
13572
|
const major = Number(version2[1]);
|
|
@@ -12839,61 +13623,96 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12839
13623
|
async function runSweep(state, driver, config) {
|
|
12840
13624
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12841
13625
|
try {
|
|
12842
|
-
const
|
|
12843
|
-
|
|
12844
|
-
|
|
13626
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13627
|
+
const sessions = enumeration.sessions;
|
|
13628
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13629
|
+
if (enumerationReason !== null) {
|
|
13630
|
+
logActivity(state, {
|
|
12845
13631
|
type: "info",
|
|
12846
|
-
|
|
12847
|
-
|
|
12848
|
-
outcome: "sweep-skipped",
|
|
12849
|
-
reason: "opencode-unreachable"
|
|
12850
|
-
}
|
|
13632
|
+
level: "warn",
|
|
13633
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12851
13634
|
});
|
|
12852
|
-
return;
|
|
12853
13635
|
}
|
|
12854
|
-
const toDelete = selectSessionsToDelete(
|
|
12855
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12856
|
-
{
|
|
12857
|
-
maxAgeMs: config.maxAgeMs,
|
|
12858
|
-
maxCount: config.maxCount,
|
|
12859
|
-
nowMs: Date.now(),
|
|
12860
|
-
protectedIds: driver.protectedSessionIds()
|
|
12861
|
-
}
|
|
12862
|
-
);
|
|
12863
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12864
13636
|
let deleted = 0;
|
|
12865
13637
|
let failed = 0;
|
|
13638
|
+
let failed4xx = 0;
|
|
13639
|
+
let failed5xx = 0;
|
|
13640
|
+
let failedOtherStatus = 0;
|
|
13641
|
+
let failedThrew = 0;
|
|
12866
13642
|
let skippedNewlyActive = 0;
|
|
12867
|
-
|
|
12868
|
-
|
|
12869
|
-
|
|
12870
|
-
|
|
12871
|
-
|
|
12872
|
-
|
|
12873
|
-
|
|
12874
|
-
|
|
13643
|
+
let remaining = 0;
|
|
13644
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13645
|
+
if (sessions !== null) {
|
|
13646
|
+
const toDelete = selectSessionsToDelete(
|
|
13647
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13648
|
+
{
|
|
13649
|
+
maxAgeMs: config.maxAgeMs,
|
|
13650
|
+
maxCount: config.maxCount,
|
|
13651
|
+
nowMs: Date.now(),
|
|
13652
|
+
protectedIds: protectedNow
|
|
13653
|
+
}
|
|
13654
|
+
);
|
|
13655
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13656
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13657
|
+
protectedNow = driver.protectedSessionIds();
|
|
13658
|
+
for (const id of deleteCandidates) {
|
|
13659
|
+
if (protectedNow.has(id)) {
|
|
13660
|
+
skippedNewlyActive++;
|
|
13661
|
+
logActivity(state, {
|
|
13662
|
+
type: "info",
|
|
13663
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13664
|
+
});
|
|
13665
|
+
continue;
|
|
13666
|
+
}
|
|
13667
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13668
|
+
if (outcome.ok) {
|
|
13669
|
+
deleted++;
|
|
13670
|
+
} else if (outcome.status === "threw") {
|
|
13671
|
+
failed++;
|
|
13672
|
+
failedThrew++;
|
|
13673
|
+
} else {
|
|
13674
|
+
failed++;
|
|
13675
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13676
|
+
if (statusClass === 4) failed4xx++;
|
|
13677
|
+
else if (statusClass === 5) failed5xx++;
|
|
13678
|
+
else failedOtherStatus++;
|
|
13679
|
+
}
|
|
12875
13680
|
}
|
|
12876
|
-
|
|
12877
|
-
|
|
13681
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13682
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13683
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13684
|
+
logActivity(state, {
|
|
13685
|
+
type: "info",
|
|
13686
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13687
|
+
});
|
|
12878
13688
|
}
|
|
12879
|
-
const
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
13689
|
+
const sessionMetadata = {
|
|
13690
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13691
|
+
sessions_deleted: deleted,
|
|
13692
|
+
sessions_failed: failed,
|
|
13693
|
+
sessions_failed_4xx: failed4xx,
|
|
13694
|
+
sessions_failed_5xx: failed5xx,
|
|
13695
|
+
sessions_failed_other: failedOtherStatus,
|
|
13696
|
+
sessions_failed_threw: failedThrew,
|
|
13697
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13698
|
+
sessions_remaining: remaining,
|
|
13699
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13700
|
+
};
|
|
12885
13701
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12886
13702
|
dbPath: sessionDbPath(),
|
|
12887
|
-
|
|
13703
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13704
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13705
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12888
13706
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12889
13707
|
});
|
|
12890
13708
|
if (reclaimResult.ok) {
|
|
13709
|
+
state.sessionStoreHealth.observeWritable();
|
|
12891
13710
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12892
13711
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12893
13712
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12894
13713
|
logSessionCleanupActivity(state, "sweep", {
|
|
12895
13714
|
type: "info",
|
|
12896
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13715
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12897
13716
|
metadata: {
|
|
12898
13717
|
outcome: "reclaimed",
|
|
12899
13718
|
mode: reclaimResult.mode,
|
|
@@ -12901,25 +13720,27 @@ async function runSweep(state, driver, config) {
|
|
|
12901
13720
|
after_bytes: reclaimResult.afterBytes,
|
|
12902
13721
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12903
13722
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
13723
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13724
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13725
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13726
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13727
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13728
|
+
...sessionMetadata
|
|
12908
13729
|
}
|
|
12909
13730
|
});
|
|
12910
13731
|
} else {
|
|
13732
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13733
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13734
|
+
}
|
|
12911
13735
|
logSessionCleanupActivity(state, "sweep", {
|
|
12912
13736
|
type: "info",
|
|
12913
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13737
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12914
13738
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12915
13739
|
metadata: {
|
|
12916
13740
|
outcome: "reclaim-skipped",
|
|
12917
13741
|
reason: reclaimResult.skipped,
|
|
12918
13742
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12919
|
-
|
|
12920
|
-
sessions_deleted: deleted,
|
|
12921
|
-
sessions_failed: failed,
|
|
12922
|
-
sessions_skipped_active: skippedNewlyActive
|
|
13743
|
+
...sessionMetadata
|
|
12923
13744
|
}
|
|
12924
13745
|
});
|
|
12925
13746
|
}
|
|
@@ -13176,6 +13997,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13176
13997
|
message: `Resource usage collection: ${warning2}`
|
|
13177
13998
|
});
|
|
13178
13999
|
}
|
|
14000
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14001
|
+
...usage,
|
|
14002
|
+
nowMs: Date.now(),
|
|
14003
|
+
state: state.lowDiskWarningState
|
|
14004
|
+
});
|
|
14005
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14006
|
+
if (lowDiskDecision.warning !== null) {
|
|
14007
|
+
logActivity(state, {
|
|
14008
|
+
type: "info",
|
|
14009
|
+
level: "warn",
|
|
14010
|
+
message: lowDiskDecision.warning
|
|
14011
|
+
});
|
|
14012
|
+
}
|
|
13179
14013
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13180
14014
|
if (result.ok) {
|
|
13181
14015
|
if (consecutiveFailures > 0) {
|
|
@@ -13422,6 +14256,7 @@ async function run(options) {
|
|
|
13422
14256
|
opencodeClient: null,
|
|
13423
14257
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13424
14258
|
sessionDbProvenanceAnomaly: false,
|
|
14259
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13425
14260
|
opencodeProcess: null,
|
|
13426
14261
|
stopOpenCodeLogTail: null,
|
|
13427
14262
|
litestreamProcess: null,
|
|
@@ -13439,6 +14274,7 @@ async function run(options) {
|
|
|
13439
14274
|
openaiUsageRearm: null,
|
|
13440
14275
|
resourceUsageTimer: null,
|
|
13441
14276
|
stopResourceUsageSampling: null,
|
|
14277
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13442
14278
|
credentialSync: null,
|
|
13443
14279
|
authHeader: ""
|
|
13444
14280
|
};
|
|
@@ -13612,6 +14448,34 @@ async function run(options) {
|
|
|
13612
14448
|
} else {
|
|
13613
14449
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13614
14450
|
}
|
|
14451
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14452
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14453
|
+
log2(state, message, level);
|
|
14454
|
+
if (state.interactive && !state.json) {
|
|
14455
|
+
logActivity(state, { type: "info", level, message });
|
|
14456
|
+
}
|
|
14457
|
+
};
|
|
14458
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14459
|
+
if (vaultResolution.error) {
|
|
14460
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14461
|
+
logCredentialVaultResolution(
|
|
14462
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14463
|
+
"warn"
|
|
14464
|
+
);
|
|
14465
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14466
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14467
|
+
logCredentialVaultResolution(
|
|
14468
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14469
|
+
"info"
|
|
14470
|
+
);
|
|
14471
|
+
} else {
|
|
14472
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14473
|
+
logCredentialVaultResolution(
|
|
14474
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14475
|
+
"debug"
|
|
14476
|
+
);
|
|
14477
|
+
}
|
|
14478
|
+
}
|
|
13615
14479
|
if (options.restoreRunnerCredentials) {
|
|
13616
14480
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13617
14481
|
const credentialContext = {
|
|
@@ -13629,10 +14493,13 @@ async function run(options) {
|
|
|
13629
14493
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13630
14494
|
}
|
|
13631
14495
|
state.credentialSync?.arm();
|
|
13632
|
-
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13633
|
-
|
|
13634
|
-
|
|
13635
|
-
|
|
14496
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14497
|
+
() => ({
|
|
14498
|
+
agentId: state.agentId,
|
|
14499
|
+
authHeader: state.authHeader
|
|
14500
|
+
}),
|
|
14501
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14502
|
+
);
|
|
13636
14503
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13637
14504
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13638
14505
|
openCodeActivityForwarder
|
|
@@ -13924,6 +14791,7 @@ async function run(options) {
|
|
|
13924
14791
|
getAuthHeader: () => state.authHeader,
|
|
13925
14792
|
conversationFilter: state.conversationFilter,
|
|
13926
14793
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14794
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13927
14795
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13928
14796
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13929
14797
|
fileSyncDirectories,
|
|
@@ -14005,14 +14873,14 @@ async function run(options) {
|
|
|
14005
14873
|
},
|
|
14006
14874
|
onDisconnected: (code, reason, transition) => {
|
|
14007
14875
|
state.connected = false;
|
|
14008
|
-
const
|
|
14876
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
14009
14877
|
logActivity(state, {
|
|
14010
14878
|
type: "info",
|
|
14011
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
14879
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14012
14880
|
});
|
|
14013
14881
|
emitAgentDisconnected(state.agentId, {
|
|
14014
14882
|
code,
|
|
14015
|
-
reason:
|
|
14883
|
+
reason: safeReason2,
|
|
14016
14884
|
connection_generation: transition.generation
|
|
14017
14885
|
});
|
|
14018
14886
|
if (state.interactive) displayStatus(state);
|
|
@@ -14175,6 +15043,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14175
15043
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14176
15044
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14177
15045
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15046
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14178
15047
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14179
15048
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14180
15049
|
"-a, --agent [id]",
|