@evident-ai/cli 3.5.2-dev.507f0a3 → 3.5.2-dev.60daa5f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1069 -350
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1069,6 +1069,76 @@ import chalk7 from "chalk";
|
|
|
1069
1069
|
// ../../packages/types/src/agents/index.ts
|
|
1070
1070
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1071
1071
|
|
|
1072
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1073
|
+
function roleOf(m) {
|
|
1074
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1075
|
+
if (typeof m.role === "string") return m.role;
|
|
1076
|
+
const infoRole = m.info?.role;
|
|
1077
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1078
|
+
}
|
|
1079
|
+
function idOf(m) {
|
|
1080
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1081
|
+
if (typeof m.id === "string") return m.id;
|
|
1082
|
+
const infoId = m.info?.id;
|
|
1083
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1084
|
+
}
|
|
1085
|
+
function parentIdOf(m) {
|
|
1086
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1087
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1088
|
+
const infoParent = m.info?.parentID;
|
|
1089
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1090
|
+
}
|
|
1091
|
+
function textPartsOf(m) {
|
|
1092
|
+
if (!m || typeof m !== "object") return [];
|
|
1093
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1094
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1095
|
+
}
|
|
1096
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1097
|
+
if (!messages || messages.length === 0) return [];
|
|
1098
|
+
const byParent = messages.filter(
|
|
1099
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1100
|
+
);
|
|
1101
|
+
if (byParent.length > 0) return byParent;
|
|
1102
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1103
|
+
if (userIndex === -1) return [];
|
|
1104
|
+
const run2 = [];
|
|
1105
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1106
|
+
const role = roleOf(messages[i]);
|
|
1107
|
+
if (role === "user") break;
|
|
1108
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1109
|
+
}
|
|
1110
|
+
return run2;
|
|
1111
|
+
}
|
|
1112
|
+
function lastText(assistantRun) {
|
|
1113
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1114
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1115
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1116
|
+
}
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1120
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1121
|
+
if (lastText(run2) !== null) return run2;
|
|
1122
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1123
|
+
if (userIndex === -1) return run2;
|
|
1124
|
+
let continuation;
|
|
1125
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1126
|
+
if (roleOf(messages[i]) === "user") {
|
|
1127
|
+
continuation = messages[i];
|
|
1128
|
+
break;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
const continuationId = idOf(continuation);
|
|
1132
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1133
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1134
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1135
|
+
}
|
|
1136
|
+
return run2;
|
|
1137
|
+
}
|
|
1138
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1139
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1072
1142
|
// ../../packages/types/src/telemetry/index.ts
|
|
1073
1143
|
var TelemetryEventTypes = {
|
|
1074
1144
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1157,7 +1227,7 @@ import ora4 from "ora";
|
|
|
1157
1227
|
import { select as select4 } from "@inquirer/prompts";
|
|
1158
1228
|
|
|
1159
1229
|
// src/lib/telemetry.ts
|
|
1160
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1230
|
+
var CLI_VERSION = (true ? "3.5.2-dev.60daa5f" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1161
1231
|
function getCliVersion() {
|
|
1162
1232
|
return CLI_VERSION;
|
|
1163
1233
|
}
|
|
@@ -1320,6 +1390,219 @@ var EventTypes = {
|
|
|
1320
1390
|
|
|
1321
1391
|
// src/lib/runner-activity-telemetry.ts
|
|
1322
1392
|
import { homedir as homedir2 } from "node:os";
|
|
1393
|
+
|
|
1394
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1395
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1396
|
+
import { dirname as dirname2 } from "node:path";
|
|
1397
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1398
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1399
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1400
|
+
function errorMessage(error2) {
|
|
1401
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1402
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1403
|
+
}
|
|
1404
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1405
|
+
const messages = [];
|
|
1406
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1407
|
+
let current = error2;
|
|
1408
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1409
|
+
visited.add(current);
|
|
1410
|
+
if (typeof current === "string") {
|
|
1411
|
+
messages.push(current);
|
|
1412
|
+
break;
|
|
1413
|
+
}
|
|
1414
|
+
if (current instanceof Error) {
|
|
1415
|
+
messages.push(current.name, current.message);
|
|
1416
|
+
current = current.cause;
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
if (typeof current === "object") {
|
|
1420
|
+
const record = current;
|
|
1421
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1422
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1423
|
+
current = record.cause;
|
|
1424
|
+
continue;
|
|
1425
|
+
}
|
|
1426
|
+
break;
|
|
1427
|
+
}
|
|
1428
|
+
return messages.some(
|
|
1429
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1430
|
+
message
|
|
1431
|
+
)
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1435
|
+
try {
|
|
1436
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1437
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1438
|
+
if (availableBytes < requiredBytes) {
|
|
1439
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1440
|
+
}
|
|
1441
|
+
return null;
|
|
1442
|
+
} catch (err) {
|
|
1443
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
function readLogicalBytes(db) {
|
|
1447
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1448
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1449
|
+
return pageCount * pageSize;
|
|
1450
|
+
}
|
|
1451
|
+
function readFreelistCount(db) {
|
|
1452
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1453
|
+
}
|
|
1454
|
+
function readCheckpointResult(db) {
|
|
1455
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1456
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1457
|
+
}
|
|
1458
|
+
async function probeReclaimAvailability(input) {
|
|
1459
|
+
const { dbPath, requiredBytes } = input;
|
|
1460
|
+
let sqlite;
|
|
1461
|
+
try {
|
|
1462
|
+
sqlite = await import("node:sqlite");
|
|
1463
|
+
} catch (err) {
|
|
1464
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1465
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1466
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1467
|
+
}
|
|
1468
|
+
let autoVacuum = null;
|
|
1469
|
+
try {
|
|
1470
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1471
|
+
try {
|
|
1472
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1473
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1474
|
+
} finally {
|
|
1475
|
+
db.close();
|
|
1476
|
+
}
|
|
1477
|
+
} catch (err) {
|
|
1478
|
+
console.warn(
|
|
1479
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1480
|
+
);
|
|
1481
|
+
}
|
|
1482
|
+
if (autoVacuum !== 0) return null;
|
|
1483
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1484
|
+
}
|
|
1485
|
+
async function reclaimSessionDbSpace(input) {
|
|
1486
|
+
const {
|
|
1487
|
+
dbPath,
|
|
1488
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1489
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1490
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1491
|
+
now = () => performance.now(),
|
|
1492
|
+
allowFullVacuum = true
|
|
1493
|
+
} = input;
|
|
1494
|
+
let sqlite;
|
|
1495
|
+
try {
|
|
1496
|
+
sqlite = await import("node:sqlite");
|
|
1497
|
+
} catch (err) {
|
|
1498
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1499
|
+
console.warn(
|
|
1500
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1501
|
+
);
|
|
1502
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1503
|
+
}
|
|
1504
|
+
const { DatabaseSync } = sqlite;
|
|
1505
|
+
let db;
|
|
1506
|
+
try {
|
|
1507
|
+
db = new DatabaseSync(dbPath);
|
|
1508
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1509
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1510
|
+
if (autoVacuum === 0) {
|
|
1511
|
+
if (!allowFullVacuum) {
|
|
1512
|
+
console.warn(
|
|
1513
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1514
|
+
);
|
|
1515
|
+
return {
|
|
1516
|
+
ok: false,
|
|
1517
|
+
skipped: "full-vacuum-blocked",
|
|
1518
|
+
detail: "a session turn is live"
|
|
1519
|
+
};
|
|
1520
|
+
}
|
|
1521
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1522
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1523
|
+
if (skipReason !== null) {
|
|
1524
|
+
console.warn(
|
|
1525
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1526
|
+
);
|
|
1527
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1528
|
+
}
|
|
1529
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1530
|
+
const freelistBefore = readFreelistCount(db);
|
|
1531
|
+
const startedAt = now();
|
|
1532
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1533
|
+
db.exec("VACUUM");
|
|
1534
|
+
const afterBytes = readLogicalBytes(db);
|
|
1535
|
+
const freelistAfter = readFreelistCount(db);
|
|
1536
|
+
const checkpoint = readCheckpointResult(db);
|
|
1537
|
+
return {
|
|
1538
|
+
ok: true,
|
|
1539
|
+
mode: "convert",
|
|
1540
|
+
beforeBytes,
|
|
1541
|
+
afterBytes,
|
|
1542
|
+
freelistBefore,
|
|
1543
|
+
freelistAfter,
|
|
1544
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1545
|
+
stoppedBy: "freelist-drained",
|
|
1546
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1547
|
+
checkpoint
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
if (autoVacuum === 2) {
|
|
1551
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1552
|
+
const freelistBefore = readFreelistCount(db);
|
|
1553
|
+
const startedAt = now();
|
|
1554
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1555
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1556
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1557
|
+
let pagesAttempted = 0;
|
|
1558
|
+
let freelistAfter = freelistBefore;
|
|
1559
|
+
let stoppedBy = "freelist-drained";
|
|
1560
|
+
while (freelistAfter > 0) {
|
|
1561
|
+
if (pagesAttempted >= pageLimit) {
|
|
1562
|
+
stoppedBy = "page-ceiling";
|
|
1563
|
+
break;
|
|
1564
|
+
}
|
|
1565
|
+
if (now() - startedAt >= budget) {
|
|
1566
|
+
stoppedBy = "time-budget";
|
|
1567
|
+
break;
|
|
1568
|
+
}
|
|
1569
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1570
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1571
|
+
pagesAttempted += pages;
|
|
1572
|
+
freelistAfter = readFreelistCount(db);
|
|
1573
|
+
}
|
|
1574
|
+
const afterBytes = readLogicalBytes(db);
|
|
1575
|
+
const checkpoint = readCheckpointResult(db);
|
|
1576
|
+
return {
|
|
1577
|
+
ok: true,
|
|
1578
|
+
mode: "incremental",
|
|
1579
|
+
beforeBytes,
|
|
1580
|
+
afterBytes,
|
|
1581
|
+
freelistBefore,
|
|
1582
|
+
freelistAfter,
|
|
1583
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1584
|
+
stoppedBy,
|
|
1585
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1586
|
+
checkpoint
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
console.warn(
|
|
1590
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1591
|
+
);
|
|
1592
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1593
|
+
} catch (err) {
|
|
1594
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1595
|
+
return {
|
|
1596
|
+
ok: false,
|
|
1597
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1598
|
+
detail: errorMessage(err)
|
|
1599
|
+
};
|
|
1600
|
+
} finally {
|
|
1601
|
+
db?.close();
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1323
1606
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1324
1607
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1325
1608
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1336,11 +1619,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1336
1619
|
if (!levelMatch) return null;
|
|
1337
1620
|
const level = levelMatch[1].toUpperCase();
|
|
1338
1621
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1339
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1622
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1623
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1340
1624
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1341
1625
|
const parsed = {
|
|
1342
1626
|
level: level === "WARN" ? "warn" : "error",
|
|
1343
|
-
sessionID: sessionMatch?.[1]
|
|
1627
|
+
sessionID: sessionMatch?.[1],
|
|
1628
|
+
messageID: messageIDMatch?.[1]
|
|
1344
1629
|
};
|
|
1345
1630
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1346
1631
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1348,12 +1633,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1348
1633
|
}
|
|
1349
1634
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1350
1635
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1351
|
-
function createOpenCodeActivityForwarder(getContext) {
|
|
1636
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1352
1637
|
let buffer = Buffer.alloc(0);
|
|
1353
1638
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1354
1639
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1355
1640
|
const metadata = {
|
|
1356
1641
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1642
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1357
1643
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1358
1644
|
};
|
|
1359
1645
|
forwardRunnerActivity(
|
|
@@ -1378,10 +1664,19 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1378
1664
|
}
|
|
1379
1665
|
};
|
|
1380
1666
|
const flushLine = (line) => {
|
|
1667
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1668
|
+
try {
|
|
1669
|
+
onSessionStoreWriteFailure();
|
|
1670
|
+
} catch (err) {
|
|
1671
|
+
reportTelemetryDiagnostic(
|
|
1672
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1673
|
+
);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1381
1676
|
const parsed = parseOpenCodeLogLine(line);
|
|
1382
1677
|
if (!parsed) return;
|
|
1383
1678
|
const now = Date.now();
|
|
1384
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1679
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1385
1680
|
expireRepeatStates(now, key);
|
|
1386
1681
|
if (key === void 0) {
|
|
1387
1682
|
forwardLine(line, parsed);
|
|
@@ -1919,7 +2214,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1919
2214
|
return null;
|
|
1920
2215
|
}
|
|
1921
2216
|
}
|
|
1922
|
-
function
|
|
2217
|
+
function roleOf2(m) {
|
|
1923
2218
|
if (!m || typeof m !== "object") return void 0;
|
|
1924
2219
|
if (typeof m.role === "string") return m.role;
|
|
1925
2220
|
const infoRole = m.info?.role;
|
|
@@ -1933,13 +2228,13 @@ function createdOf(m) {
|
|
|
1933
2228
|
if (!m || typeof m !== "object") return void 0;
|
|
1934
2229
|
return m.info?.time?.created ?? m.time?.created;
|
|
1935
2230
|
}
|
|
1936
|
-
function
|
|
2231
|
+
function idOf2(m) {
|
|
1937
2232
|
if (!m || typeof m !== "object") return void 0;
|
|
1938
2233
|
if (typeof m.id === "string") return m.id;
|
|
1939
2234
|
const infoId = m.info?.id;
|
|
1940
2235
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1941
2236
|
}
|
|
1942
|
-
function
|
|
2237
|
+
function parentIdOf2(m) {
|
|
1943
2238
|
if (!m || typeof m !== "object") return void 0;
|
|
1944
2239
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1945
2240
|
const infoParent = m.info?.parentID;
|
|
@@ -2004,9 +2299,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2004
2299
|
function isSessionActivelyGenerating(messages) {
|
|
2005
2300
|
if (!messages || messages.length === 0) return false;
|
|
2006
2301
|
const last = messages[messages.length - 1];
|
|
2007
|
-
if (
|
|
2302
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2008
2303
|
return completedOf(last) == null;
|
|
2009
2304
|
}
|
|
2305
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2010
2306
|
function sessionLastActivityMs(session) {
|
|
2011
2307
|
const candidates = [
|
|
2012
2308
|
session.time?.updated,
|
|
@@ -2021,30 +2317,60 @@ function sessionLastActivityMs(session) {
|
|
|
2021
2317
|
}
|
|
2022
2318
|
return null;
|
|
2023
2319
|
}
|
|
2024
|
-
|
|
2025
|
-
|
|
2320
|
+
function sessionListPath(options) {
|
|
2321
|
+
const params = new URLSearchParams();
|
|
2322
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2323
|
+
const query = params.toString();
|
|
2324
|
+
return query ? `/session?${query}` : "/session";
|
|
2325
|
+
}
|
|
2326
|
+
async function listSessionPage(port, client, options) {
|
|
2327
|
+
const path = sessionListPath(options);
|
|
2026
2328
|
try {
|
|
2027
|
-
const res = await requestWithClient(port, client,
|
|
2329
|
+
const res = await requestWithClient(port, client, path);
|
|
2028
2330
|
if (!res.ok) return null;
|
|
2029
2331
|
const body = await res.json();
|
|
2030
2332
|
return Array.isArray(body) ? body : null;
|
|
2031
2333
|
} catch (error2) {
|
|
2032
2334
|
console.error(
|
|
2033
|
-
`[
|
|
2335
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2034
2336
|
);
|
|
2035
2337
|
return null;
|
|
2036
2338
|
}
|
|
2037
2339
|
}
|
|
2340
|
+
async function listSessions(port, client, options) {
|
|
2341
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2342
|
+
return listSessionPage(port, client, options);
|
|
2343
|
+
}
|
|
2344
|
+
async function listAllSessions(port, client) {
|
|
2345
|
+
if (client?.version === "v2") {
|
|
2346
|
+
const sessions2 = await listV2Sessions(client);
|
|
2347
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2348
|
+
}
|
|
2349
|
+
const sessions = await listSessionPage(port, client, {
|
|
2350
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2351
|
+
});
|
|
2352
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2353
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2354
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2355
|
+
}
|
|
2356
|
+
return { sessions };
|
|
2357
|
+
}
|
|
2038
2358
|
async function deleteSession(port, id, client) {
|
|
2039
2359
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2040
2360
|
try {
|
|
2041
|
-
const res = await requestWithClient(
|
|
2042
|
-
|
|
2361
|
+
const res = await requestWithClient(
|
|
2362
|
+
port,
|
|
2363
|
+
client,
|
|
2364
|
+
`/session/${id}`,
|
|
2365
|
+
{ method: "DELETE" },
|
|
2366
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2367
|
+
);
|
|
2368
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
2043
2369
|
} catch (error2) {
|
|
2044
2370
|
console.error(
|
|
2045
2371
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2046
2372
|
);
|
|
2047
|
-
return false;
|
|
2373
|
+
return { ok: false, status: "threw" };
|
|
2048
2374
|
}
|
|
2049
2375
|
}
|
|
2050
2376
|
async function sessionExists(port, id, client) {
|
|
@@ -2282,11 +2608,31 @@ function messageText(m) {
|
|
|
2282
2608
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2283
2609
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2284
2610
|
}
|
|
2285
|
-
|
|
2611
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2612
|
+
if (!messages || messages.length === 0) return null;
|
|
2613
|
+
let best = null;
|
|
2614
|
+
for (const message of messages) {
|
|
2615
|
+
if (roleOf2(message) !== "user") continue;
|
|
2616
|
+
const id = idOf2(message);
|
|
2617
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2618
|
+
if (messageText(message) !== content) continue;
|
|
2619
|
+
const created = createdOf(message) ?? 0;
|
|
2620
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2621
|
+
}
|
|
2622
|
+
return best?.id ?? null;
|
|
2623
|
+
}
|
|
2624
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2286
2625
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2287
2626
|
const knownUserIds = new Set(
|
|
2288
|
-
(before ?? []).filter((m) =>
|
|
2627
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2289
2628
|
);
|
|
2629
|
+
if (callbacks) {
|
|
2630
|
+
try {
|
|
2631
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2632
|
+
} catch (error2) {
|
|
2633
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2290
2636
|
const parts = [{ type: "text", text: content }];
|
|
2291
2637
|
let pendingOutcomes = null;
|
|
2292
2638
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2325,22 +2671,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2325
2671
|
const READ_BACK_DELAY_MS = 150;
|
|
2326
2672
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2327
2673
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
const id = idOf(m);
|
|
2333
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2334
|
-
if (messageText(m) !== content) continue;
|
|
2335
|
-
const created = createdOf(m) ?? 0;
|
|
2336
|
-
if (best === null || created > best.created) {
|
|
2337
|
-
best = { id, created };
|
|
2338
|
-
}
|
|
2339
|
-
}
|
|
2340
|
-
if (best) {
|
|
2341
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2342
|
-
return best.id;
|
|
2343
|
-
}
|
|
2674
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2675
|
+
if (dispatchedUserMessageId !== null) {
|
|
2676
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2677
|
+
return dispatchedUserMessageId;
|
|
2344
2678
|
}
|
|
2345
2679
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2346
2680
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2351,13 +2685,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2351
2685
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2352
2686
|
if (!messages || messages.length === 0) return null;
|
|
2353
2687
|
const byParent = messages.find(
|
|
2354
|
-
(m) =>
|
|
2688
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2355
2689
|
);
|
|
2356
2690
|
if (byParent) return byParent;
|
|
2357
|
-
const userIndex = messages.findIndex((m) =>
|
|
2691
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2358
2692
|
if (userIndex === -1) return null;
|
|
2359
2693
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2360
|
-
if (
|
|
2694
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2361
2695
|
}
|
|
2362
2696
|
return null;
|
|
2363
2697
|
}
|
|
@@ -2367,7 +2701,7 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2367
2701
|
let lastNonErrored = null;
|
|
2368
2702
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2369
2703
|
const m = messages[i];
|
|
2370
|
-
if (
|
|
2704
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2371
2705
|
if (lastCorrelated === null) lastCorrelated = m;
|
|
2372
2706
|
if (errorOf(m) == null) {
|
|
2373
2707
|
lastNonErrored = m;
|
|
@@ -2375,12 +2709,12 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2375
2709
|
}
|
|
2376
2710
|
}
|
|
2377
2711
|
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
2378
|
-
const userIndex = messages.findIndex((m) =>
|
|
2712
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2379
2713
|
if (userIndex === -1) return null;
|
|
2380
2714
|
let last = null;
|
|
2381
2715
|
let lastOk = null;
|
|
2382
2716
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2383
|
-
const role =
|
|
2717
|
+
const role = roleOf2(messages[i]);
|
|
2384
2718
|
if (role === "user") break;
|
|
2385
2719
|
if (role === "assistant") {
|
|
2386
2720
|
last = messages[i];
|
|
@@ -2392,16 +2726,16 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2392
2726
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2393
2727
|
if (!messages || messages.length === 0) return [];
|
|
2394
2728
|
const byParent = messages.filter(
|
|
2395
|
-
(message) =>
|
|
2729
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2396
2730
|
);
|
|
2397
2731
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2398
2732
|
if (assistants.length === 0) {
|
|
2399
|
-
const userIndex = messages.findIndex((message) =>
|
|
2733
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2400
2734
|
if (userIndex === -1) return [];
|
|
2401
2735
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2402
2736
|
const message = messages[i];
|
|
2403
|
-
if (
|
|
2404
|
-
if (
|
|
2737
|
+
if (roleOf2(message) === "user") break;
|
|
2738
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2405
2739
|
}
|
|
2406
2740
|
}
|
|
2407
2741
|
const refs = [];
|
|
@@ -2441,7 +2775,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2441
2775
|
if (!messages || messages.length === 0) return [];
|
|
2442
2776
|
const calls = [];
|
|
2443
2777
|
for (const message of messages) {
|
|
2444
|
-
if (
|
|
2778
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2445
2779
|
for (const part of message.parts ?? []) {
|
|
2446
2780
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2447
2781
|
continue;
|
|
@@ -2471,7 +2805,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2471
2805
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2472
2806
|
const unattributed = [];
|
|
2473
2807
|
for (const message of messages ?? []) {
|
|
2474
|
-
if (
|
|
2808
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2475
2809
|
const created = finiteNumber(createdOf(message));
|
|
2476
2810
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2477
2811
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2548,7 +2882,7 @@ function sumAssistantUsage(messages) {
|
|
|
2548
2882
|
function messageUsage(messages, userMessageId) {
|
|
2549
2883
|
if (!messages || messages.length === 0) return null;
|
|
2550
2884
|
const byParentAll = messages.filter(
|
|
2551
|
-
(m) =>
|
|
2885
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2552
2886
|
);
|
|
2553
2887
|
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2554
2888
|
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
@@ -2564,7 +2898,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2564
2898
|
}
|
|
2565
2899
|
function messageRunState(messages, userMessageId) {
|
|
2566
2900
|
if (!messages || messages.length === 0) return "unknown";
|
|
2567
|
-
const hasUser = messages.some((m) =>
|
|
2901
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2568
2902
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2569
2903
|
if (!hasUser) {
|
|
2570
2904
|
if (!reply) return "unknown";
|
|
@@ -2656,7 +2990,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2656
2990
|
if (!messages || messages.length === 0) return null;
|
|
2657
2991
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2658
2992
|
const message = messages[i];
|
|
2659
|
-
if (
|
|
2993
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2660
2994
|
const created = createdOf(message);
|
|
2661
2995
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2662
2996
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2688,25 +3022,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2688
3022
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2689
3023
|
if (!messages || messages.length === 0) return false;
|
|
2690
3024
|
return messages.some(
|
|
2691
|
-
(m) =>
|
|
3025
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2692
3026
|
);
|
|
2693
3027
|
}
|
|
2694
3028
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2695
3029
|
if (!messages || messages.length === 0) return false;
|
|
2696
|
-
const userIndex = messages.findIndex((message) =>
|
|
3030
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2697
3031
|
if (userIndex === -1) return false;
|
|
2698
3032
|
let hasLaterUser = false;
|
|
2699
3033
|
let hasStartedLaterUser = false;
|
|
2700
3034
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2701
3035
|
const message = messages[i];
|
|
2702
|
-
if (
|
|
3036
|
+
if (roleOf2(message) !== "user") continue;
|
|
2703
3037
|
hasLaterUser = true;
|
|
2704
|
-
const laterUserMessageId =
|
|
3038
|
+
const laterUserMessageId = idOf2(message);
|
|
2705
3039
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2706
3040
|
return false;
|
|
2707
3041
|
}
|
|
2708
3042
|
if (messages.some(
|
|
2709
|
-
(candidate) =>
|
|
3043
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2710
3044
|
)) {
|
|
2711
3045
|
hasStartedLaterUser = true;
|
|
2712
3046
|
}
|
|
@@ -3177,13 +3511,17 @@ async function listV2Sessions(client) {
|
|
|
3177
3511
|
}
|
|
3178
3512
|
async function deleteV2Session(client, sessionId) {
|
|
3179
3513
|
try {
|
|
3180
|
-
await client.request(
|
|
3181
|
-
|
|
3514
|
+
const response = await client.request(
|
|
3515
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3516
|
+
{ method: "DELETE" },
|
|
3517
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3518
|
+
);
|
|
3519
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3182
3520
|
} catch (error2) {
|
|
3183
3521
|
console.error(
|
|
3184
3522
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3185
3523
|
);
|
|
3186
|
-
return false;
|
|
3524
|
+
return { ok: false, status: "threw" };
|
|
3187
3525
|
}
|
|
3188
3526
|
}
|
|
3189
3527
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3805,9 +4143,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3805
4143
|
|
|
3806
4144
|
// src/lib/opencode/session-db-boot.ts
|
|
3807
4145
|
import { spawn as spawn2 } from "node:child_process";
|
|
3808
|
-
import { mkdirSync, statSync as
|
|
4146
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3809
4147
|
import { homedir as homedir4 } from "node:os";
|
|
3810
|
-
import { dirname as
|
|
4148
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3811
4149
|
|
|
3812
4150
|
// src/lib/runner-synchroniser.ts
|
|
3813
4151
|
import { spawn } from "node:child_process";
|
|
@@ -3932,7 +4270,7 @@ function clearMarker(options) {
|
|
|
3932
4270
|
function markNoReplicate(options, message) {
|
|
3933
4271
|
if (options.noReplicateMarker) {
|
|
3934
4272
|
try {
|
|
3935
|
-
mkdirSync(
|
|
4273
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3936
4274
|
writeFileSync(options.noReplicateMarker, "");
|
|
3937
4275
|
} catch (error2) {
|
|
3938
4276
|
options.log(
|
|
@@ -4057,7 +4395,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4057
4395
|
return null;
|
|
4058
4396
|
}
|
|
4059
4397
|
try {
|
|
4060
|
-
if (
|
|
4398
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4061
4399
|
} catch (error2) {
|
|
4062
4400
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4063
4401
|
options.log(
|
|
@@ -4087,7 +4425,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4087
4425
|
return null;
|
|
4088
4426
|
}
|
|
4089
4427
|
try {
|
|
4090
|
-
mkdirSync(
|
|
4428
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
4091
4429
|
writeFileSync(configPath, rendered.stdout);
|
|
4092
4430
|
} catch (error2) {
|
|
4093
4431
|
options.log(
|
|
@@ -4248,7 +4586,7 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4248
4586
|
}
|
|
4249
4587
|
function fileExists(path) {
|
|
4250
4588
|
try {
|
|
4251
|
-
|
|
4589
|
+
statSync3(path);
|
|
4252
4590
|
return true;
|
|
4253
4591
|
} catch (error2) {
|
|
4254
4592
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4337,7 +4675,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4337
4675
|
// src/lib/opencode/session-db-provenance.ts
|
|
4338
4676
|
import { createRequire } from "node:module";
|
|
4339
4677
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4340
|
-
import { dirname as
|
|
4678
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4341
4679
|
var require2 = createRequire(import.meta.url);
|
|
4342
4680
|
function readSessionDbMigrationIds(dbPath) {
|
|
4343
4681
|
let db;
|
|
@@ -4403,7 +4741,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4403
4741
|
}
|
|
4404
4742
|
function saveSessionDbProvenanceState(path, state) {
|
|
4405
4743
|
try {
|
|
4406
|
-
mkdirSync2(
|
|
4744
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4407
4745
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4408
4746
|
`, "utf8");
|
|
4409
4747
|
} catch (error2) {
|
|
@@ -4984,7 +5322,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4984
5322
|
toDelete.push(s.id);
|
|
4985
5323
|
}
|
|
4986
5324
|
}
|
|
4987
|
-
|
|
5325
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5326
|
+
return toDelete.sort(
|
|
5327
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5328
|
+
);
|
|
4988
5329
|
}
|
|
4989
5330
|
var DEFAULT_INTERVAL = "1h";
|
|
4990
5331
|
function resolve(flag, envValue, fallback) {
|
|
@@ -5044,13 +5385,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5044
5385
|
}
|
|
5045
5386
|
|
|
5046
5387
|
// src/lib/opencode/session-db-size.ts
|
|
5047
|
-
import { statSync as
|
|
5388
|
+
import { statSync as statSync4 } from "node:fs";
|
|
5048
5389
|
import { join as join6 } from "node:path";
|
|
5049
5390
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5050
5391
|
function statSessionDbBytes(homeDir) {
|
|
5051
5392
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5052
5393
|
try {
|
|
5053
|
-
return
|
|
5394
|
+
return statSync4(dbPath).size;
|
|
5054
5395
|
} catch (err) {
|
|
5055
5396
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5056
5397
|
if (!isMissingFile) {
|
|
@@ -5076,7 +5417,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5076
5417
|
}
|
|
5077
5418
|
|
|
5078
5419
|
// src/lib/opencode/log-tail.ts
|
|
5079
|
-
import { statSync as
|
|
5420
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5080
5421
|
import { homedir as homedir5 } from "node:os";
|
|
5081
5422
|
import { join as join7 } from "node:path";
|
|
5082
5423
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -5099,7 +5440,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5099
5440
|
let inode = null;
|
|
5100
5441
|
let baselineReady = true;
|
|
5101
5442
|
try {
|
|
5102
|
-
const initial =
|
|
5443
|
+
const initial = statSync5(logPath);
|
|
5103
5444
|
offset = initial.size;
|
|
5104
5445
|
inode = initial.ino;
|
|
5105
5446
|
} catch (error2) {
|
|
@@ -5158,129 +5499,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5158
5499
|
};
|
|
5159
5500
|
}
|
|
5160
5501
|
|
|
5161
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5162
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5163
|
-
import { dirname as dirname4 } from "node:path";
|
|
5164
|
-
function errorMessage(error2) {
|
|
5165
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5166
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5167
|
-
}
|
|
5168
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5169
|
-
try {
|
|
5170
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5171
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5172
|
-
if (availableBytes < requiredBytes) {
|
|
5173
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5174
|
-
}
|
|
5175
|
-
return null;
|
|
5176
|
-
} catch (err) {
|
|
5177
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5178
|
-
}
|
|
5179
|
-
}
|
|
5180
|
-
function readLogicalBytes(db) {
|
|
5181
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5182
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5183
|
-
return pageCount * pageSize;
|
|
5184
|
-
}
|
|
5185
|
-
function readCheckpointResult(db) {
|
|
5186
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5187
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5188
|
-
}
|
|
5189
|
-
async function probeReclaimAvailability(input) {
|
|
5190
|
-
const { dbPath, requiredBytes } = input;
|
|
5191
|
-
let sqlite;
|
|
5192
|
-
try {
|
|
5193
|
-
sqlite = await import("node:sqlite");
|
|
5194
|
-
} catch (err) {
|
|
5195
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5196
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5197
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5198
|
-
}
|
|
5199
|
-
let autoVacuum = null;
|
|
5200
|
-
try {
|
|
5201
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5202
|
-
try {
|
|
5203
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5204
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5205
|
-
} finally {
|
|
5206
|
-
db.close();
|
|
5207
|
-
}
|
|
5208
|
-
} catch (err) {
|
|
5209
|
-
console.warn(
|
|
5210
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5211
|
-
);
|
|
5212
|
-
}
|
|
5213
|
-
if (autoVacuum !== 0) return null;
|
|
5214
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5215
|
-
}
|
|
5216
|
-
async function reclaimSessionDbSpace(input) {
|
|
5217
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5218
|
-
let sqlite;
|
|
5219
|
-
try {
|
|
5220
|
-
sqlite = await import("node:sqlite");
|
|
5221
|
-
} catch (err) {
|
|
5222
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5223
|
-
console.warn(
|
|
5224
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5225
|
-
);
|
|
5226
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5227
|
-
}
|
|
5228
|
-
const { DatabaseSync } = sqlite;
|
|
5229
|
-
let db;
|
|
5230
|
-
try {
|
|
5231
|
-
db = new DatabaseSync(dbPath);
|
|
5232
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5233
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5234
|
-
if (autoVacuum === 0) {
|
|
5235
|
-
if (!allowFullVacuum) {
|
|
5236
|
-
console.warn(
|
|
5237
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5238
|
-
);
|
|
5239
|
-
return {
|
|
5240
|
-
ok: false,
|
|
5241
|
-
skipped: "full-vacuum-blocked",
|
|
5242
|
-
detail: "a session turn is live"
|
|
5243
|
-
};
|
|
5244
|
-
}
|
|
5245
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5246
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5247
|
-
if (skipReason !== null) {
|
|
5248
|
-
console.warn(
|
|
5249
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5250
|
-
);
|
|
5251
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5252
|
-
}
|
|
5253
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5254
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5255
|
-
db.exec("VACUUM");
|
|
5256
|
-
const afterBytes = readLogicalBytes(db);
|
|
5257
|
-
const checkpoint = readCheckpointResult(db);
|
|
5258
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5259
|
-
}
|
|
5260
|
-
if (autoVacuum === 2) {
|
|
5261
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5262
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5263
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5264
|
-
const afterBytes = readLogicalBytes(db);
|
|
5265
|
-
const checkpoint = readCheckpointResult(db);
|
|
5266
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5267
|
-
}
|
|
5268
|
-
console.warn(
|
|
5269
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5270
|
-
);
|
|
5271
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5272
|
-
} catch (err) {
|
|
5273
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5274
|
-
return {
|
|
5275
|
-
ok: false,
|
|
5276
|
-
skipped: "reclaim-error",
|
|
5277
|
-
detail: errorMessage(err)
|
|
5278
|
-
};
|
|
5279
|
-
} finally {
|
|
5280
|
-
db?.close();
|
|
5281
|
-
}
|
|
5282
|
-
}
|
|
5283
|
-
|
|
5284
5502
|
// src/lib/tunnel/connection.ts
|
|
5285
5503
|
import WebSocket2 from "ws";
|
|
5286
5504
|
|
|
@@ -6170,6 +6388,83 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6170
6388
|
};
|
|
6171
6389
|
}
|
|
6172
6390
|
|
|
6391
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6392
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6393
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6394
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6395
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6396
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6397
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6398
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6399
|
+
var BAND_RANK = {
|
|
6400
|
+
low: 1,
|
|
6401
|
+
very_low: 2,
|
|
6402
|
+
critical: 3
|
|
6403
|
+
};
|
|
6404
|
+
function bandFor(freeBytes) {
|
|
6405
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6406
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6407
|
+
return "low";
|
|
6408
|
+
}
|
|
6409
|
+
function formatBytes(bytes) {
|
|
6410
|
+
if (bytes === null) return "unavailable";
|
|
6411
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6412
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6413
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6414
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6415
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6416
|
+
}
|
|
6417
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6418
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6419
|
+
}
|
|
6420
|
+
function evaluateLowDiskWarning(input) {
|
|
6421
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6422
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6423
|
+
return { warning: null, state };
|
|
6424
|
+
}
|
|
6425
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6426
|
+
if (recovered) {
|
|
6427
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6428
|
+
}
|
|
6429
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6430
|
+
if (!low) return { warning: null, state };
|
|
6431
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6432
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6433
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6434
|
+
const worse = currentRank > previousRank;
|
|
6435
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6436
|
+
const nextState = {
|
|
6437
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6438
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6439
|
+
};
|
|
6440
|
+
return {
|
|
6441
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6442
|
+
state: nextState
|
|
6443
|
+
};
|
|
6444
|
+
}
|
|
6445
|
+
|
|
6446
|
+
// src/lib/opencode/session-store-health.ts
|
|
6447
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6448
|
+
function createSessionStoreHealth(options = {}) {
|
|
6449
|
+
const now = options.now ?? (() => Date.now());
|
|
6450
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6451
|
+
let lastUnwritableAt = null;
|
|
6452
|
+
let lastWritableAt = null;
|
|
6453
|
+
return {
|
|
6454
|
+
observeUnwritable: () => {
|
|
6455
|
+
lastUnwritableAt = now();
|
|
6456
|
+
},
|
|
6457
|
+
observeWritable: () => {
|
|
6458
|
+
lastWritableAt = now();
|
|
6459
|
+
},
|
|
6460
|
+
isUnwritable: () => {
|
|
6461
|
+
if (lastUnwritableAt === null) return false;
|
|
6462
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6463
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6464
|
+
}
|
|
6465
|
+
};
|
|
6466
|
+
}
|
|
6467
|
+
|
|
6173
6468
|
// src/lib/channels/driver.ts
|
|
6174
6469
|
import { homedir as homedir6 } from "node:os";
|
|
6175
6470
|
|
|
@@ -6628,6 +6923,7 @@ var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
|
6628
6923
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
6629
6924
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6630
6925
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
6926
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6631
6927
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6632
6928
|
var HEARTBEAT_MS = 6e4;
|
|
6633
6929
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6689,6 +6985,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6689
6985
|
maxActiveSessions;
|
|
6690
6986
|
watcherStallMs;
|
|
6691
6987
|
wedgeWarningIntervalMs;
|
|
6988
|
+
sessionStoreHealth;
|
|
6692
6989
|
openCodeClient;
|
|
6693
6990
|
/** Cache of conversationId → opencode sessionId. */
|
|
6694
6991
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6813,10 +7110,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6813
7110
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6814
7111
|
*/
|
|
6815
7112
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7113
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7114
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6816
7115
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6817
7116
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6818
7117
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6819
7118
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7119
|
+
/**
|
|
7120
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7121
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7122
|
+
* identical-text turn, so it never produces a correlation.
|
|
7123
|
+
*/
|
|
7124
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6820
7125
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6821
7126
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6822
7127
|
/**
|
|
@@ -6902,21 +7207,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6902
7207
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6903
7208
|
*/
|
|
6904
7209
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6905
|
-
/**
|
|
6906
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6907
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6908
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6909
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6910
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6911
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6912
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6913
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6914
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6915
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6916
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6917
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6918
|
-
*/
|
|
6919
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6920
7210
|
/**
|
|
6921
7211
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6922
7212
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6934,8 +7224,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6934
7224
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6935
7225
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6936
7226
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6937
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6938
|
-
*
|
|
7227
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7228
|
+
* the terminal-report fence and is never sent again).
|
|
6939
7229
|
*/
|
|
6940
7230
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6941
7231
|
/**
|
|
@@ -7058,6 +7348,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7058
7348
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7059
7349
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7060
7350
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7351
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7061
7352
|
}
|
|
7062
7353
|
get isV2() {
|
|
7063
7354
|
return this.openCodeClient.version === "v2";
|
|
@@ -7073,7 +7364,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7073
7364
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7074
7365
|
}
|
|
7075
7366
|
async listSessions() {
|
|
7076
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7367
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7368
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7369
|
+
});
|
|
7077
7370
|
}
|
|
7078
7371
|
async sessionExists(sessionId) {
|
|
7079
7372
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7250,6 +7543,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7250
7543
|
}
|
|
7251
7544
|
return ids;
|
|
7252
7545
|
}
|
|
7546
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7547
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7548
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7549
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7550
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7551
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7552
|
+
}
|
|
7553
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7554
|
+
}
|
|
7555
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7556
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7557
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7558
|
+
this.awaitingReadopt.delete(messageId);
|
|
7559
|
+
this.sessions.delete(conversationId);
|
|
7560
|
+
this.supersede(conversationId, sessionId);
|
|
7561
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7562
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7563
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7564
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7565
|
+
this.log({
|
|
7566
|
+
level: "error",
|
|
7567
|
+
message: errorMessage3,
|
|
7568
|
+
conversation_id: conversationId,
|
|
7569
|
+
message_id: messageId
|
|
7570
|
+
});
|
|
7571
|
+
const reportError = await this.reportUntrackableAck(
|
|
7572
|
+
conversationId,
|
|
7573
|
+
messageId,
|
|
7574
|
+
null,
|
|
7575
|
+
errorMessage3
|
|
7576
|
+
);
|
|
7577
|
+
if (reportError !== null) {
|
|
7578
|
+
this.log({
|
|
7579
|
+
level: "warn",
|
|
7580
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7581
|
+
conversation_id: conversationId,
|
|
7582
|
+
message_id: messageId
|
|
7583
|
+
});
|
|
7584
|
+
}
|
|
7585
|
+
if (source === "readopt") {
|
|
7586
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7587
|
+
}
|
|
7588
|
+
}
|
|
7253
7589
|
/**
|
|
7254
7590
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7255
7591
|
*
|
|
@@ -7429,6 +7765,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7429
7765
|
}
|
|
7430
7766
|
break;
|
|
7431
7767
|
}
|
|
7768
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7769
|
+
if (correlation) {
|
|
7770
|
+
correlation.attempts += 1;
|
|
7771
|
+
let correlatedId = null;
|
|
7772
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7773
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7774
|
+
conv,
|
|
7775
|
+
message,
|
|
7776
|
+
correlation.sessionId
|
|
7777
|
+
);
|
|
7778
|
+
if (polled.ok)
|
|
7779
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7780
|
+
} else if (correlation.knownUserIds === null) {
|
|
7781
|
+
this.log({
|
|
7782
|
+
level: "debug",
|
|
7783
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7784
|
+
conversation_id: conv.id,
|
|
7785
|
+
message_id: message.id
|
|
7786
|
+
});
|
|
7787
|
+
}
|
|
7788
|
+
if (correlatedId !== null) {
|
|
7789
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7790
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7791
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7792
|
+
this.dispatched.add(message.id);
|
|
7793
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7794
|
+
dispatched += 1;
|
|
7795
|
+
this.log({
|
|
7796
|
+
level: "info",
|
|
7797
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7798
|
+
conversation_id: conv.id,
|
|
7799
|
+
message_id: message.id
|
|
7800
|
+
});
|
|
7801
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7802
|
+
opencode_message_id: correlatedId
|
|
7803
|
+
});
|
|
7804
|
+
continue;
|
|
7805
|
+
}
|
|
7806
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7807
|
+
await this.handoffUntrackableCorrelation(
|
|
7808
|
+
conv.id,
|
|
7809
|
+
message.id,
|
|
7810
|
+
correlation.sessionId,
|
|
7811
|
+
correlation.attempts,
|
|
7812
|
+
"fresh"
|
|
7813
|
+
);
|
|
7814
|
+
break;
|
|
7815
|
+
}
|
|
7816
|
+
break;
|
|
7817
|
+
}
|
|
7432
7818
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7433
7819
|
if (effectiveOpencodeMessageId) {
|
|
7434
7820
|
const outcome = await this.resolveRedrive(
|
|
@@ -7450,6 +7836,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7450
7836
|
model: message.opencode_model ?? void 0
|
|
7451
7837
|
};
|
|
7452
7838
|
let opencodeMessageId;
|
|
7839
|
+
let preDispatchUserIds = null;
|
|
7453
7840
|
try {
|
|
7454
7841
|
this.log({
|
|
7455
7842
|
level: "info",
|
|
@@ -7479,7 +7866,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7479
7866
|
message.content,
|
|
7480
7867
|
options,
|
|
7481
7868
|
sendAttachments,
|
|
7482
|
-
this.openCodeClient
|
|
7869
|
+
this.openCodeClient,
|
|
7870
|
+
{
|
|
7871
|
+
onKnownUserIds: (ids) => {
|
|
7872
|
+
preDispatchUserIds = ids;
|
|
7873
|
+
},
|
|
7874
|
+
onKnownUserIdsError: (error2) => {
|
|
7875
|
+
this.log({
|
|
7876
|
+
level: "warn",
|
|
7877
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
7878
|
+
conversation_id: conv.id,
|
|
7879
|
+
message_id: message.id
|
|
7880
|
+
});
|
|
7881
|
+
}
|
|
7882
|
+
}
|
|
7483
7883
|
)
|
|
7484
7884
|
);
|
|
7485
7885
|
} catch (err) {
|
|
@@ -7487,6 +7887,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7487
7887
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7488
7888
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7489
7889
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
7890
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7490
7891
|
this.log({
|
|
7491
7892
|
level: "error",
|
|
7492
7893
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7562,39 +7963,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7562
7963
|
if (this.isV2) {
|
|
7563
7964
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7564
7965
|
}
|
|
7565
|
-
|
|
7566
|
-
|
|
7567
|
-
|
|
7568
|
-
|
|
7569
|
-
|
|
7570
|
-
|
|
7571
|
-
message_id: message.id
|
|
7572
|
-
});
|
|
7573
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7574
|
-
continue;
|
|
7575
|
-
}
|
|
7576
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7577
|
-
this.sessions.delete(conv.id);
|
|
7578
|
-
this.supersede(conv.id, sessionId);
|
|
7579
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
7966
|
+
this.awaitingCorrelation.set(message.id, {
|
|
7967
|
+
sessionId,
|
|
7968
|
+
content: message.content,
|
|
7969
|
+
attempts: 0,
|
|
7970
|
+
knownUserIds: preDispatchUserIds
|
|
7971
|
+
});
|
|
7580
7972
|
this.log({
|
|
7581
|
-
level: "
|
|
7582
|
-
message:
|
|
7973
|
+
level: "warn",
|
|
7974
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7583
7975
|
conversation_id: conv.id,
|
|
7584
7976
|
message_id: message.id
|
|
7585
7977
|
});
|
|
7586
|
-
|
|
7587
|
-
this.log({
|
|
7588
|
-
level: "warn",
|
|
7589
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7590
|
-
conversation_id: conv.id,
|
|
7591
|
-
message_id: message.id
|
|
7592
|
-
});
|
|
7593
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7594
|
-
});
|
|
7978
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7595
7979
|
break;
|
|
7596
7980
|
}
|
|
7597
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7598
7981
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7599
7982
|
this.releasedOpencodeIds.delete(message.id);
|
|
7600
7983
|
this.dispatched.add(message.id);
|
|
@@ -8199,23 +8582,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8199
8582
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8200
8583
|
return 1;
|
|
8201
8584
|
}
|
|
8202
|
-
/**
|
|
8203
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8204
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8205
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8206
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8207
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8208
|
-
* count, since a new session is a genuinely different attempt.
|
|
8209
|
-
*/
|
|
8210
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8211
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8212
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8213
|
-
existing.count += 1;
|
|
8214
|
-
return existing.count;
|
|
8215
|
-
}
|
|
8216
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8217
|
-
return 1;
|
|
8218
|
-
}
|
|
8219
8585
|
/** Record one queued give-up toward the never-started bound. */
|
|
8220
8586
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8221
8587
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8527,8 +8893,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8527
8893
|
b2LastDescendantCheckMs: 0,
|
|
8528
8894
|
b2RootOngoingHeldLogged: false,
|
|
8529
8895
|
b2AbandonedSignalled: false,
|
|
8896
|
+
b2AbandonedWithoutAnswer: false,
|
|
8530
8897
|
ambiguousPinnedSinceMs: 0,
|
|
8531
8898
|
ambiguousResolved: false,
|
|
8899
|
+
sessionStoreFailurePending: false,
|
|
8532
8900
|
neverStartedResolved: false
|
|
8533
8901
|
});
|
|
8534
8902
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8645,8 +9013,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8645
9013
|
b2LastDescendantCheckMs: 0,
|
|
8646
9014
|
b2RootOngoingHeldLogged: false,
|
|
8647
9015
|
b2AbandonedSignalled: false,
|
|
9016
|
+
b2AbandonedWithoutAnswer: false,
|
|
8648
9017
|
ambiguousPinnedSinceMs: 0,
|
|
8649
9018
|
ambiguousResolved: false,
|
|
9019
|
+
sessionStoreFailurePending: false,
|
|
8650
9020
|
neverStartedResolved: false
|
|
8651
9021
|
});
|
|
8652
9022
|
}
|
|
@@ -9060,6 +9430,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9060
9430
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9061
9431
|
this.neverStartedAttempts.delete(id);
|
|
9062
9432
|
}
|
|
9433
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9434
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9435
|
+
return;
|
|
9436
|
+
}
|
|
9063
9437
|
if (inFlight.neverStartedResolved) {
|
|
9064
9438
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9065
9439
|
inFlight.neverStartedResolved = false;
|
|
@@ -9219,10 +9593,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9219
9593
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9220
9594
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9221
9595
|
inFlight.b2AbandonedSignalled = false;
|
|
9596
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9222
9597
|
}
|
|
9223
9598
|
} else {
|
|
9224
9599
|
if (inFlight.b2AbandonedSignalled) {
|
|
9225
|
-
|
|
9600
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9601
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9602
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9603
|
+
sessionId,
|
|
9604
|
+
watcher,
|
|
9605
|
+
inFlight,
|
|
9606
|
+
messages,
|
|
9607
|
+
reply?.info?.finish ?? reply?.finish
|
|
9608
|
+
);
|
|
9609
|
+
} else {
|
|
9610
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9611
|
+
}
|
|
9226
9612
|
return;
|
|
9227
9613
|
}
|
|
9228
9614
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9239,17 +9625,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9239
9625
|
descendantOngoing,
|
|
9240
9626
|
rootOngoing
|
|
9241
9627
|
})) {
|
|
9628
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9629
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9630
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9631
|
+
watched_for_ms: pinnedForMs,
|
|
9632
|
+
run_state: "failed",
|
|
9633
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9634
|
+
});
|
|
9635
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9636
|
+
return;
|
|
9637
|
+
}
|
|
9242
9638
|
inFlight.b2AbandonedSignalled = true;
|
|
9639
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9640
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9641
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9642
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9643
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9644
|
+
watched_for_ms: pinnedForMs,
|
|
9645
|
+
run_state: "failed",
|
|
9646
|
+
finish,
|
|
9647
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9648
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9649
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9650
|
+
});
|
|
9651
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9652
|
+
sessionId,
|
|
9653
|
+
watcher,
|
|
9654
|
+
inFlight,
|
|
9655
|
+
messages,
|
|
9656
|
+
finish
|
|
9657
|
+
);
|
|
9658
|
+
return;
|
|
9659
|
+
}
|
|
9243
9660
|
this.log({
|
|
9244
9661
|
level: "warn",
|
|
9245
9662
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9246
9663
|
conversation_id: conv.id,
|
|
9247
9664
|
message_id: id
|
|
9248
9665
|
});
|
|
9249
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9250
9666
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9251
9667
|
watched_for_ms: pinnedForMs,
|
|
9252
|
-
finish
|
|
9668
|
+
finish,
|
|
9253
9669
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9254
9670
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9255
9671
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9297,6 +9713,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9297
9713
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9298
9714
|
sessionOngoing: ongoing
|
|
9299
9715
|
})) {
|
|
9716
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9717
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9718
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9719
|
+
watched_for_ms: pinnedForMs,
|
|
9720
|
+
run_state: "failed",
|
|
9721
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9722
|
+
});
|
|
9723
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9724
|
+
return;
|
|
9725
|
+
}
|
|
9300
9726
|
inFlight.ambiguousResolved = true;
|
|
9301
9727
|
this.log({
|
|
9302
9728
|
level: "warn",
|
|
@@ -9521,6 +9947,74 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9521
9947
|
}
|
|
9522
9948
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9523
9949
|
}
|
|
9950
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
9951
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
9952
|
+
const conv = watcher.conv;
|
|
9953
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
9954
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
9955
|
+
this.log({
|
|
9956
|
+
level: "warn",
|
|
9957
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
9958
|
+
conversation_id: conv.id,
|
|
9959
|
+
message_id: inFlight.evidentMessageId
|
|
9960
|
+
});
|
|
9961
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
9962
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
9963
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
9964
|
+
messages,
|
|
9965
|
+
inFlight.opencodeMessageId,
|
|
9966
|
+
inFlight.evidentMessageId
|
|
9967
|
+
);
|
|
9968
|
+
try {
|
|
9969
|
+
await this.markFailed(
|
|
9970
|
+
conv.id,
|
|
9971
|
+
inFlight.evidentMessageId,
|
|
9972
|
+
sessionId,
|
|
9973
|
+
error2,
|
|
9974
|
+
usage,
|
|
9975
|
+
void 0,
|
|
9976
|
+
usageAgentName,
|
|
9977
|
+
subagentInvocations
|
|
9978
|
+
);
|
|
9979
|
+
} catch (err) {
|
|
9980
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
9981
|
+
if (err instanceof ChannelTerminalError) {
|
|
9982
|
+
this.log({
|
|
9983
|
+
level: "warn",
|
|
9984
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9985
|
+
conversation_id: conv.id,
|
|
9986
|
+
message_id: inFlight.evidentMessageId
|
|
9987
|
+
});
|
|
9988
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9989
|
+
return;
|
|
9990
|
+
}
|
|
9991
|
+
if (this.now() >= inFlight.deadline) {
|
|
9992
|
+
this.log({
|
|
9993
|
+
level: "warn",
|
|
9994
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
9995
|
+
conversation_id: conv.id,
|
|
9996
|
+
message_id: inFlight.evidentMessageId
|
|
9997
|
+
});
|
|
9998
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9999
|
+
return;
|
|
10000
|
+
}
|
|
10001
|
+
this.log({
|
|
10002
|
+
level: "warn",
|
|
10003
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10004
|
+
conversation_id: conv.id,
|
|
10005
|
+
message_id: inFlight.evidentMessageId
|
|
10006
|
+
});
|
|
10007
|
+
return;
|
|
10008
|
+
}
|
|
10009
|
+
inFlight.done = true;
|
|
10010
|
+
await this.reportSubagentAuthFailures(
|
|
10011
|
+
watcher.conv.id,
|
|
10012
|
+
inFlight.opencodeMessageId,
|
|
10013
|
+
inFlight.evidentMessageId,
|
|
10014
|
+
messages
|
|
10015
|
+
);
|
|
10016
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10017
|
+
}
|
|
9524
10018
|
/**
|
|
9525
10019
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9526
10020
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9576,6 +10070,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9576
10070
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9577
10071
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9578
10072
|
}
|
|
10073
|
+
/**
|
|
10074
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10075
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10076
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10077
|
+
*/
|
|
10078
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10079
|
+
const conv = watcher.conv;
|
|
10080
|
+
const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
10081
|
+
this.log({
|
|
10082
|
+
level: "error",
|
|
10083
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10084
|
+
conversation_id: conv.id,
|
|
10085
|
+
message_id: inFlight.evidentMessageId
|
|
10086
|
+
});
|
|
10087
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10088
|
+
try {
|
|
10089
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10090
|
+
} catch (err) {
|
|
10091
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10092
|
+
if (err instanceof ChannelTerminalError) {
|
|
10093
|
+
this.log({
|
|
10094
|
+
level: "warn",
|
|
10095
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10096
|
+
conversation_id: conv.id,
|
|
10097
|
+
message_id: inFlight.evidentMessageId
|
|
10098
|
+
});
|
|
10099
|
+
this.recordReleasedOpencodeId(
|
|
10100
|
+
inFlight.evidentMessageId,
|
|
10101
|
+
sessionId,
|
|
10102
|
+
inFlight.opencodeMessageId
|
|
10103
|
+
);
|
|
10104
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10105
|
+
return;
|
|
10106
|
+
}
|
|
10107
|
+
if (this.now() >= inFlight.deadline) {
|
|
10108
|
+
this.log({
|
|
10109
|
+
level: "warn",
|
|
10110
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10111
|
+
conversation_id: conv.id,
|
|
10112
|
+
message_id: inFlight.evidentMessageId
|
|
10113
|
+
});
|
|
10114
|
+
this.recordReleasedOpencodeId(
|
|
10115
|
+
inFlight.evidentMessageId,
|
|
10116
|
+
sessionId,
|
|
10117
|
+
inFlight.opencodeMessageId
|
|
10118
|
+
);
|
|
10119
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10120
|
+
return;
|
|
10121
|
+
}
|
|
10122
|
+
this.log({
|
|
10123
|
+
level: "warn",
|
|
10124
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10125
|
+
conversation_id: conv.id,
|
|
10126
|
+
message_id: inFlight.evidentMessageId
|
|
10127
|
+
});
|
|
10128
|
+
return;
|
|
10129
|
+
}
|
|
10130
|
+
inFlight.done = true;
|
|
10131
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10132
|
+
}
|
|
9579
10133
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9580
10134
|
/**
|
|
9581
10135
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9614,11 +10168,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9614
10168
|
}
|
|
9615
10169
|
}
|
|
9616
10170
|
}
|
|
9617
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10171
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9618
10172
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10173
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10174
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10175
|
+
this.awaitingCorrelation.delete(id);
|
|
10176
|
+
this.log({
|
|
10177
|
+
level: "debug",
|
|
10178
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10179
|
+
message_id: id
|
|
10180
|
+
});
|
|
10181
|
+
}
|
|
10182
|
+
}
|
|
9619
10183
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9620
10184
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9621
10185
|
this.untrackableAck.delete(id);
|
|
10186
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9622
10187
|
this.untrackableAckSignalled.delete(id);
|
|
9623
10188
|
this.untrackableAckWarned.delete(id);
|
|
9624
10189
|
this.log({
|
|
@@ -9658,7 +10223,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9658
10223
|
bySession.set(row.opencode_session_id, list);
|
|
9659
10224
|
}
|
|
9660
10225
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9661
|
-
let messages;
|
|
10226
|
+
let messages = null;
|
|
9662
10227
|
try {
|
|
9663
10228
|
if (this.isV2) {
|
|
9664
10229
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9667,9 +10232,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9667
10232
|
level: "warn",
|
|
9668
10233
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9669
10234
|
});
|
|
9670
|
-
|
|
10235
|
+
} else {
|
|
10236
|
+
messages = snapshot;
|
|
9671
10237
|
}
|
|
9672
|
-
messages = snapshot;
|
|
9673
10238
|
} else {
|
|
9674
10239
|
const polled = await pollSessionMessagesForRedrive(
|
|
9675
10240
|
this.port,
|
|
@@ -9682,15 +10247,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9682
10247
|
level: "warn",
|
|
9683
10248
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9684
10249
|
});
|
|
9685
|
-
|
|
10250
|
+
} else {
|
|
10251
|
+
messages = polled.messages;
|
|
9686
10252
|
}
|
|
9687
|
-
messages = polled.messages;
|
|
9688
10253
|
}
|
|
9689
10254
|
} catch (err) {
|
|
9690
10255
|
this.log({
|
|
9691
10256
|
level: "warn",
|
|
9692
10257
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9693
10258
|
});
|
|
10259
|
+
}
|
|
10260
|
+
if (messages === null) {
|
|
10261
|
+
for (const row of sessionRows) {
|
|
10262
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10263
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10264
|
+
}
|
|
10265
|
+
}
|
|
9694
10266
|
continue;
|
|
9695
10267
|
}
|
|
9696
10268
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9700,6 +10272,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9700
10272
|
}
|
|
9701
10273
|
}
|
|
9702
10274
|
}
|
|
10275
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10276
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10277
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10278
|
+
if (!correlation) return false;
|
|
10279
|
+
correlation.attempts += 1;
|
|
10280
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10281
|
+
if (correlatedId !== null) {
|
|
10282
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10283
|
+
const conv = this.convForRow(sessionId, row);
|
|
10284
|
+
const message = this.queuedMessageForRow(row);
|
|
10285
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10286
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10287
|
+
this.dispatched.add(row.id);
|
|
10288
|
+
this.readopted.add(row.id);
|
|
10289
|
+
this.ensureWatcherRunning(sessionId);
|
|
10290
|
+
this.log({
|
|
10291
|
+
level: "info",
|
|
10292
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10293
|
+
conversation_id: row.conversation_id,
|
|
10294
|
+
message_id: row.id
|
|
10295
|
+
});
|
|
10296
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10297
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10298
|
+
opencode_message_id: correlatedId
|
|
10299
|
+
});
|
|
10300
|
+
return true;
|
|
10301
|
+
}
|
|
10302
|
+
if (correlation.knownUserIds === null) {
|
|
10303
|
+
this.log({
|
|
10304
|
+
level: "debug",
|
|
10305
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10306
|
+
conversation_id: row.conversation_id,
|
|
10307
|
+
message_id: row.id
|
|
10308
|
+
});
|
|
10309
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10310
|
+
this.log({
|
|
10311
|
+
level: "warn",
|
|
10312
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10313
|
+
conversation_id: row.conversation_id,
|
|
10314
|
+
message_id: row.id
|
|
10315
|
+
});
|
|
10316
|
+
}
|
|
10317
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10318
|
+
await this.handoffUntrackableCorrelation(
|
|
10319
|
+
row.conversation_id,
|
|
10320
|
+
row.id,
|
|
10321
|
+
correlation.sessionId,
|
|
10322
|
+
correlation.attempts,
|
|
10323
|
+
"readopt"
|
|
10324
|
+
);
|
|
10325
|
+
}
|
|
10326
|
+
return true;
|
|
10327
|
+
}
|
|
9703
10328
|
/**
|
|
9704
10329
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9705
10330
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9707,8 +10332,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9707
10332
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9708
10333
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9709
10334
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9710
|
-
* before the restart)
|
|
9711
|
-
*
|
|
10335
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10336
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10337
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10338
|
+
* `forceReadoptRun`):
|
|
9712
10339
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9713
10340
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9714
10341
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9717,7 +10344,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9717
10344
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9718
10345
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9719
10346
|
* tracking the stored id so the reply correlates by it;
|
|
9720
|
-
* - `unknown`/null id →
|
|
10347
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10348
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9721
10349
|
*
|
|
9722
10350
|
* Only `ChannelAuthError` propagates.
|
|
9723
10351
|
*/
|
|
@@ -9731,6 +10359,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9731
10359
|
});
|
|
9732
10360
|
return;
|
|
9733
10361
|
}
|
|
10362
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9734
10363
|
const ocId = row.opencode_message_id;
|
|
9735
10364
|
const state = messageRunState(messages, ocId ?? "");
|
|
9736
10365
|
if (state === "done") {
|
|
@@ -9797,12 +10426,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9797
10426
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9798
10427
|
this.dontRedispatch.delete(row.id);
|
|
9799
10428
|
this.untrackableAck.delete(row.id);
|
|
10429
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9800
10430
|
this.untrackableAckSignalled.delete(row.id);
|
|
9801
10431
|
this.untrackableAckWarned.delete(row.id);
|
|
9802
10432
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9803
10433
|
return;
|
|
9804
10434
|
}
|
|
9805
|
-
if (this.
|
|
10435
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10436
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10437
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10438
|
+
this.untrackableAckWarned.add(row.id);
|
|
10439
|
+
this.log({
|
|
10440
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10441
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10442
|
+
conversation_id: row.conversation_id,
|
|
10443
|
+
message_id: row.id
|
|
10444
|
+
});
|
|
10445
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10446
|
+
const reportError = await this.reportUntrackableAck(
|
|
10447
|
+
row.conversation_id,
|
|
10448
|
+
row.id,
|
|
10449
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10450
|
+
errorMessage3
|
|
10451
|
+
);
|
|
10452
|
+
if (reportError !== null) {
|
|
10453
|
+
this.log({
|
|
10454
|
+
level: "warn",
|
|
10455
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10456
|
+
conversation_id: row.conversation_id,
|
|
10457
|
+
message_id: row.id
|
|
10458
|
+
});
|
|
10459
|
+
}
|
|
10460
|
+
} else {
|
|
10461
|
+
this.log({
|
|
10462
|
+
level: "debug",
|
|
10463
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10464
|
+
conversation_id: row.conversation_id,
|
|
10465
|
+
message_id: row.id
|
|
10466
|
+
});
|
|
10467
|
+
}
|
|
10468
|
+
return;
|
|
10469
|
+
}
|
|
10470
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9806
10471
|
this.log({
|
|
9807
10472
|
level: "debug",
|
|
9808
10473
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9980,6 +10645,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9980
10645
|
}
|
|
9981
10646
|
this.dontRedispatch.delete(row.id);
|
|
9982
10647
|
this.untrackableAck.delete(row.id);
|
|
10648
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9983
10649
|
this.untrackableAckSignalled.delete(row.id);
|
|
9984
10650
|
this.untrackableAckWarned.delete(row.id);
|
|
9985
10651
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9999,8 +10665,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9999
10665
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
10000
10666
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
10001
10667
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
10002
|
-
* re-entry)
|
|
10003
|
-
*
|
|
10668
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10669
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10670
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
10004
10671
|
*
|
|
10005
10672
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
10006
10673
|
* `processed_at` (Invariant 1).
|
|
@@ -10063,6 +10730,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10063
10730
|
);
|
|
10064
10731
|
}
|
|
10065
10732
|
let ocId;
|
|
10733
|
+
let preDispatchUserIds = null;
|
|
10066
10734
|
try {
|
|
10067
10735
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10068
10736
|
sessionId,
|
|
@@ -10072,7 +10740,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10072
10740
|
row.content,
|
|
10073
10741
|
options,
|
|
10074
10742
|
sendAttachments,
|
|
10075
|
-
this.openCodeClient
|
|
10743
|
+
this.openCodeClient,
|
|
10744
|
+
{
|
|
10745
|
+
onKnownUserIds: (ids) => {
|
|
10746
|
+
preDispatchUserIds = ids;
|
|
10747
|
+
},
|
|
10748
|
+
onKnownUserIdsError: (error2) => {
|
|
10749
|
+
this.log({
|
|
10750
|
+
level: "warn",
|
|
10751
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10752
|
+
conversation_id: row.conversation_id,
|
|
10753
|
+
message_id: row.id
|
|
10754
|
+
});
|
|
10755
|
+
}
|
|
10756
|
+
}
|
|
10076
10757
|
)
|
|
10077
10758
|
);
|
|
10078
10759
|
} catch (err) {
|
|
@@ -10105,6 +10786,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10105
10786
|
}
|
|
10106
10787
|
} else {
|
|
10107
10788
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
10789
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10108
10790
|
}
|
|
10109
10791
|
this.log({
|
|
10110
10792
|
level: "error",
|
|
@@ -10153,39 +10835,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10153
10835
|
}
|
|
10154
10836
|
if (ocId === null) {
|
|
10155
10837
|
this.awaitingReadopt.delete(row.id);
|
|
10156
|
-
|
|
10157
|
-
|
|
10158
|
-
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
this.log({
|
|
10163
|
-
level: "error",
|
|
10164
|
-
message: errorMessage3,
|
|
10165
|
-
conversation_id: row.conversation_id,
|
|
10166
|
-
message_id: row.id
|
|
10167
|
-
});
|
|
10168
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10169
|
-
this.log({
|
|
10170
|
-
level: "warn",
|
|
10171
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10172
|
-
conversation_id: row.conversation_id,
|
|
10173
|
-
message_id: row.id
|
|
10174
|
-
});
|
|
10175
|
-
});
|
|
10176
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10177
|
-
return;
|
|
10178
|
-
}
|
|
10838
|
+
this.awaitingCorrelation.set(row.id, {
|
|
10839
|
+
sessionId,
|
|
10840
|
+
content: row.content,
|
|
10841
|
+
attempts: 0,
|
|
10842
|
+
knownUserIds: preDispatchUserIds
|
|
10843
|
+
});
|
|
10179
10844
|
this.log({
|
|
10180
10845
|
level: "warn",
|
|
10181
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
10846
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10182
10847
|
conversation_id: row.conversation_id,
|
|
10183
10848
|
message_id: row.id
|
|
10184
10849
|
});
|
|
10185
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10186
10850
|
return;
|
|
10187
10851
|
}
|
|
10188
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10189
10852
|
const processedAtMs = this.processedAtMs(row);
|
|
10190
10853
|
this.registerReadopted(
|
|
10191
10854
|
readoptConv,
|
|
@@ -12780,9 +13443,9 @@ async function driveChannels(state, driver) {
|
|
|
12780
13443
|
}
|
|
12781
13444
|
}
|
|
12782
13445
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12783
|
-
var
|
|
13446
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12784
13447
|
function shouldWarnForReclaimSkip(reason) {
|
|
12785
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13448
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12786
13449
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12787
13450
|
if (!version2) return false;
|
|
12788
13451
|
const major = Number(version2[1]);
|
|
@@ -12839,61 +13502,96 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12839
13502
|
async function runSweep(state, driver, config) {
|
|
12840
13503
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12841
13504
|
try {
|
|
12842
|
-
const
|
|
12843
|
-
|
|
12844
|
-
|
|
13505
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13506
|
+
const sessions = enumeration.sessions;
|
|
13507
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13508
|
+
if (enumerationReason !== null) {
|
|
13509
|
+
logActivity(state, {
|
|
12845
13510
|
type: "info",
|
|
12846
|
-
|
|
12847
|
-
|
|
12848
|
-
outcome: "sweep-skipped",
|
|
12849
|
-
reason: "opencode-unreachable"
|
|
12850
|
-
}
|
|
13511
|
+
level: "warn",
|
|
13512
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12851
13513
|
});
|
|
12852
|
-
return;
|
|
12853
13514
|
}
|
|
12854
|
-
const toDelete = selectSessionsToDelete(
|
|
12855
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12856
|
-
{
|
|
12857
|
-
maxAgeMs: config.maxAgeMs,
|
|
12858
|
-
maxCount: config.maxCount,
|
|
12859
|
-
nowMs: Date.now(),
|
|
12860
|
-
protectedIds: driver.protectedSessionIds()
|
|
12861
|
-
}
|
|
12862
|
-
);
|
|
12863
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12864
13515
|
let deleted = 0;
|
|
12865
13516
|
let failed = 0;
|
|
13517
|
+
let failed4xx = 0;
|
|
13518
|
+
let failed5xx = 0;
|
|
13519
|
+
let failedOtherStatus = 0;
|
|
13520
|
+
let failedThrew = 0;
|
|
12866
13521
|
let skippedNewlyActive = 0;
|
|
12867
|
-
|
|
12868
|
-
|
|
12869
|
-
|
|
12870
|
-
|
|
12871
|
-
|
|
12872
|
-
|
|
12873
|
-
|
|
12874
|
-
|
|
13522
|
+
let remaining = 0;
|
|
13523
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13524
|
+
if (sessions !== null) {
|
|
13525
|
+
const toDelete = selectSessionsToDelete(
|
|
13526
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13527
|
+
{
|
|
13528
|
+
maxAgeMs: config.maxAgeMs,
|
|
13529
|
+
maxCount: config.maxCount,
|
|
13530
|
+
nowMs: Date.now(),
|
|
13531
|
+
protectedIds: protectedNow
|
|
13532
|
+
}
|
|
13533
|
+
);
|
|
13534
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13535
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13536
|
+
protectedNow = driver.protectedSessionIds();
|
|
13537
|
+
for (const id of deleteCandidates) {
|
|
13538
|
+
if (protectedNow.has(id)) {
|
|
13539
|
+
skippedNewlyActive++;
|
|
13540
|
+
logActivity(state, {
|
|
13541
|
+
type: "info",
|
|
13542
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13543
|
+
});
|
|
13544
|
+
continue;
|
|
13545
|
+
}
|
|
13546
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13547
|
+
if (outcome.ok) {
|
|
13548
|
+
deleted++;
|
|
13549
|
+
} else if (outcome.status === "threw") {
|
|
13550
|
+
failed++;
|
|
13551
|
+
failedThrew++;
|
|
13552
|
+
} else {
|
|
13553
|
+
failed++;
|
|
13554
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13555
|
+
if (statusClass === 4) failed4xx++;
|
|
13556
|
+
else if (statusClass === 5) failed5xx++;
|
|
13557
|
+
else failedOtherStatus++;
|
|
13558
|
+
}
|
|
12875
13559
|
}
|
|
12876
|
-
|
|
12877
|
-
|
|
13560
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13561
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13562
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13563
|
+
logActivity(state, {
|
|
13564
|
+
type: "info",
|
|
13565
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13566
|
+
});
|
|
12878
13567
|
}
|
|
12879
|
-
const
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
13568
|
+
const sessionMetadata = {
|
|
13569
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13570
|
+
sessions_deleted: deleted,
|
|
13571
|
+
sessions_failed: failed,
|
|
13572
|
+
sessions_failed_4xx: failed4xx,
|
|
13573
|
+
sessions_failed_5xx: failed5xx,
|
|
13574
|
+
sessions_failed_other: failedOtherStatus,
|
|
13575
|
+
sessions_failed_threw: failedThrew,
|
|
13576
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13577
|
+
sessions_remaining: remaining,
|
|
13578
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13579
|
+
};
|
|
12885
13580
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12886
13581
|
dbPath: sessionDbPath(),
|
|
12887
|
-
|
|
13582
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13583
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13584
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12888
13585
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12889
13586
|
});
|
|
12890
13587
|
if (reclaimResult.ok) {
|
|
13588
|
+
state.sessionStoreHealth.observeWritable();
|
|
12891
13589
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12892
13590
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12893
13591
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12894
13592
|
logSessionCleanupActivity(state, "sweep", {
|
|
12895
13593
|
type: "info",
|
|
12896
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13594
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12897
13595
|
metadata: {
|
|
12898
13596
|
outcome: "reclaimed",
|
|
12899
13597
|
mode: reclaimResult.mode,
|
|
@@ -12901,25 +13599,27 @@ async function runSweep(state, driver, config) {
|
|
|
12901
13599
|
after_bytes: reclaimResult.afterBytes,
|
|
12902
13600
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12903
13601
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
13602
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13603
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13604
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13605
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13606
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13607
|
+
...sessionMetadata
|
|
12908
13608
|
}
|
|
12909
13609
|
});
|
|
12910
13610
|
} else {
|
|
13611
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13612
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13613
|
+
}
|
|
12911
13614
|
logSessionCleanupActivity(state, "sweep", {
|
|
12912
13615
|
type: "info",
|
|
12913
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13616
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12914
13617
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12915
13618
|
metadata: {
|
|
12916
13619
|
outcome: "reclaim-skipped",
|
|
12917
13620
|
reason: reclaimResult.skipped,
|
|
12918
13621
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12919
|
-
|
|
12920
|
-
sessions_deleted: deleted,
|
|
12921
|
-
sessions_failed: failed,
|
|
12922
|
-
sessions_skipped_active: skippedNewlyActive
|
|
13622
|
+
...sessionMetadata
|
|
12923
13623
|
}
|
|
12924
13624
|
});
|
|
12925
13625
|
}
|
|
@@ -13176,6 +13876,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13176
13876
|
message: `Resource usage collection: ${warning2}`
|
|
13177
13877
|
});
|
|
13178
13878
|
}
|
|
13879
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
13880
|
+
...usage,
|
|
13881
|
+
nowMs: Date.now(),
|
|
13882
|
+
state: state.lowDiskWarningState
|
|
13883
|
+
});
|
|
13884
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
13885
|
+
if (lowDiskDecision.warning !== null) {
|
|
13886
|
+
logActivity(state, {
|
|
13887
|
+
type: "info",
|
|
13888
|
+
level: "warn",
|
|
13889
|
+
message: lowDiskDecision.warning
|
|
13890
|
+
});
|
|
13891
|
+
}
|
|
13179
13892
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13180
13893
|
if (result.ok) {
|
|
13181
13894
|
if (consecutiveFailures > 0) {
|
|
@@ -13422,6 +14135,7 @@ async function run(options) {
|
|
|
13422
14135
|
opencodeClient: null,
|
|
13423
14136
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13424
14137
|
sessionDbProvenanceAnomaly: false,
|
|
14138
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13425
14139
|
opencodeProcess: null,
|
|
13426
14140
|
stopOpenCodeLogTail: null,
|
|
13427
14141
|
litestreamProcess: null,
|
|
@@ -13439,6 +14153,7 @@ async function run(options) {
|
|
|
13439
14153
|
openaiUsageRearm: null,
|
|
13440
14154
|
resourceUsageTimer: null,
|
|
13441
14155
|
stopResourceUsageSampling: null,
|
|
14156
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13442
14157
|
credentialSync: null,
|
|
13443
14158
|
authHeader: ""
|
|
13444
14159
|
};
|
|
@@ -13629,10 +14344,13 @@ async function run(options) {
|
|
|
13629
14344
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13630
14345
|
}
|
|
13631
14346
|
state.credentialSync?.arm();
|
|
13632
|
-
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13633
|
-
|
|
13634
|
-
|
|
13635
|
-
|
|
14347
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14348
|
+
() => ({
|
|
14349
|
+
agentId: state.agentId,
|
|
14350
|
+
authHeader: state.authHeader
|
|
14351
|
+
}),
|
|
14352
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14353
|
+
);
|
|
13636
14354
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13637
14355
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13638
14356
|
openCodeActivityForwarder
|
|
@@ -13924,6 +14642,7 @@ async function run(options) {
|
|
|
13924
14642
|
getAuthHeader: () => state.authHeader,
|
|
13925
14643
|
conversationFilter: state.conversationFilter,
|
|
13926
14644
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14645
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13927
14646
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13928
14647
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13929
14648
|
fileSyncDirectories,
|