@evident-ai/cli 3.5.2-dev.3e69672 → 3.5.2-dev.4118041
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +422 -101
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1322,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1322
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1323
1323
|
|
|
1324
1324
|
// src/lib/telemetry.ts
|
|
1325
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.4118041" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1326
1326
|
function getCliVersion() {
|
|
1327
1327
|
return CLI_VERSION;
|
|
1328
1328
|
}
|
|
@@ -1483,20 +1483,47 @@ var EventTypes = {
|
|
|
1483
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1484
1484
|
};
|
|
1485
1485
|
|
|
1486
|
-
// src/lib/runner-activity-telemetry.ts
|
|
1487
|
-
import { homedir as homedir2 } from "node:os";
|
|
1488
|
-
|
|
1489
1486
|
// src/lib/opencode/session-db-reclaim.ts
|
|
1490
1487
|
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1491
1488
|
import { dirname as dirname2 } from "node:path";
|
|
1492
|
-
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1491
|
+
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1493
1519
|
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1494
1520
|
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1495
1522
|
function errorMessage(error2) {
|
|
1496
1523
|
if (!(error2 instanceof Error)) return String(error2);
|
|
1497
1524
|
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1498
1525
|
}
|
|
1499
|
-
function
|
|
1526
|
+
function errorMessages(error2) {
|
|
1500
1527
|
const messages = [];
|
|
1501
1528
|
const visited = /* @__PURE__ */ new Set();
|
|
1502
1529
|
let current = error2;
|
|
@@ -1520,12 +1547,81 @@ function isSessionStoreWriteFailure(error2) {
|
|
|
1520
1547
|
}
|
|
1521
1548
|
break;
|
|
1522
1549
|
}
|
|
1523
|
-
return messages
|
|
1550
|
+
return messages;
|
|
1551
|
+
}
|
|
1552
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1553
|
+
return errorMessages(error2).some(
|
|
1524
1554
|
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1525
1555
|
message
|
|
1526
1556
|
)
|
|
1527
1557
|
);
|
|
1528
1558
|
}
|
|
1559
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1560
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1561
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1562
|
+
function isCauseBearingText(text) {
|
|
1563
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1564
|
+
}
|
|
1565
|
+
function structuredLogFields(line) {
|
|
1566
|
+
const fields = [];
|
|
1567
|
+
for (let index = 0; index < line.length; index++) {
|
|
1568
|
+
if (line[index] === '"') {
|
|
1569
|
+
index++;
|
|
1570
|
+
while (index < line.length) {
|
|
1571
|
+
if (line[index] === "\\") {
|
|
1572
|
+
index++;
|
|
1573
|
+
} else if (line[index] === '"') {
|
|
1574
|
+
break;
|
|
1575
|
+
}
|
|
1576
|
+
index++;
|
|
1577
|
+
}
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1581
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1582
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1583
|
+
const valueStart = index + key.length;
|
|
1584
|
+
if (line[valueStart] !== '"') {
|
|
1585
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1586
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1587
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
let valueEnd = valueStart + 1;
|
|
1591
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1592
|
+
if (line[valueEnd] === "\\") {
|
|
1593
|
+
valueEnd++;
|
|
1594
|
+
} else if (line[valueEnd] === '"') {
|
|
1595
|
+
break;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1599
|
+
index = valueEnd;
|
|
1600
|
+
}
|
|
1601
|
+
return fields;
|
|
1602
|
+
}
|
|
1603
|
+
function structuredCauseMessages(text) {
|
|
1604
|
+
const fields = structuredLogFields(text);
|
|
1605
|
+
if (fields.length > 0) return fields;
|
|
1606
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1607
|
+
}
|
|
1608
|
+
function causeTokenFromText(text) {
|
|
1609
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1610
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1611
|
+
const match = pattern.exec(text);
|
|
1612
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1613
|
+
return match[1];
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
return null;
|
|
1617
|
+
}
|
|
1618
|
+
function sessionStoreCauseToken(value) {
|
|
1619
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1620
|
+
const token = causeTokenFromText(message);
|
|
1621
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1622
|
+
}
|
|
1623
|
+
return null;
|
|
1624
|
+
}
|
|
1529
1625
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1530
1626
|
try {
|
|
1531
1627
|
const fsStats = statfsSync(dirname2(dbPath));
|
|
@@ -1547,6 +1643,7 @@ function readFreelistCount(db) {
|
|
|
1547
1643
|
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1548
1644
|
}
|
|
1549
1645
|
function readCheckpointResult(db) {
|
|
1646
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1550
1647
|
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1551
1648
|
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1552
1649
|
}
|
|
@@ -1564,7 +1661,7 @@ async function probeReclaimAvailability(input) {
|
|
|
1564
1661
|
try {
|
|
1565
1662
|
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1566
1663
|
try {
|
|
1567
|
-
db.exec(
|
|
1664
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1568
1665
|
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1569
1666
|
} finally {
|
|
1570
1667
|
db.close();
|
|
@@ -1584,7 +1681,8 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1584
1681
|
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1585
1682
|
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1586
1683
|
now = () => performance.now(),
|
|
1587
|
-
|
|
1684
|
+
hasLiveTurn = false,
|
|
1685
|
+
hasProvenanceAnomaly = false
|
|
1588
1686
|
} = input;
|
|
1589
1687
|
let sqlite;
|
|
1590
1688
|
try {
|
|
@@ -1600,17 +1698,19 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1600
1698
|
let db;
|
|
1601
1699
|
try {
|
|
1602
1700
|
db = new DatabaseSync(dbPath);
|
|
1603
|
-
db.exec(
|
|
1701
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1604
1702
|
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1605
1703
|
if (autoVacuum === 0) {
|
|
1606
|
-
if (
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1704
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1705
|
+
const detail = [
|
|
1706
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1707
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1708
|
+
].join(" and ");
|
|
1709
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1610
1710
|
return {
|
|
1611
1711
|
ok: false,
|
|
1612
1712
|
skipped: "full-vacuum-blocked",
|
|
1613
|
-
detail
|
|
1713
|
+
detail
|
|
1614
1714
|
};
|
|
1615
1715
|
}
|
|
1616
1716
|
const fileBytesForGuard = statSync2(dbPath).size;
|
|
@@ -1628,7 +1728,9 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1628
1728
|
db.exec("VACUUM");
|
|
1629
1729
|
const afterBytes = readLogicalBytes(db);
|
|
1630
1730
|
const freelistAfter = readFreelistCount(db);
|
|
1731
|
+
const checkpointStartedAt = now();
|
|
1631
1732
|
const checkpoint = readCheckpointResult(db);
|
|
1733
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1632
1734
|
return {
|
|
1633
1735
|
ok: true,
|
|
1634
1736
|
mode: "convert",
|
|
@@ -1639,10 +1741,18 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1639
1741
|
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1640
1742
|
stoppedBy: "freelist-drained",
|
|
1641
1743
|
elapsedMs: Math.max(0, now() - startedAt),
|
|
1744
|
+
checkpointElapsedMs,
|
|
1642
1745
|
checkpoint
|
|
1643
1746
|
};
|
|
1644
1747
|
}
|
|
1645
1748
|
if (autoVacuum === 2) {
|
|
1749
|
+
if (hasLiveTurn) {
|
|
1750
|
+
const detail = "a session turn is live";
|
|
1751
|
+
console.warn(
|
|
1752
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1753
|
+
);
|
|
1754
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1755
|
+
}
|
|
1646
1756
|
const beforeBytes = readLogicalBytes(db);
|
|
1647
1757
|
const freelistBefore = readFreelistCount(db);
|
|
1648
1758
|
const startedAt = now();
|
|
@@ -1667,7 +1777,9 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1667
1777
|
freelistAfter = readFreelistCount(db);
|
|
1668
1778
|
}
|
|
1669
1779
|
const afterBytes = readLogicalBytes(db);
|
|
1780
|
+
const checkpointStartedAt = now();
|
|
1670
1781
|
const checkpoint = readCheckpointResult(db);
|
|
1782
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1671
1783
|
return {
|
|
1672
1784
|
ok: true,
|
|
1673
1785
|
mode: "incremental",
|
|
@@ -1678,6 +1790,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1678
1790
|
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1679
1791
|
stoppedBy,
|
|
1680
1792
|
elapsedMs: Math.max(0, now() - startedAt),
|
|
1793
|
+
checkpointElapsedMs,
|
|
1681
1794
|
checkpoint
|
|
1682
1795
|
};
|
|
1683
1796
|
}
|
|
@@ -1759,16 +1872,22 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
|
|
|
1759
1872
|
}
|
|
1760
1873
|
};
|
|
1761
1874
|
const flushLine = (line) => {
|
|
1762
|
-
|
|
1875
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1876
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1877
|
+
const cause = sessionStoreCauseToken(line);
|
|
1878
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1763
1879
|
try {
|
|
1764
|
-
onSessionStoreWriteFailure(
|
|
1880
|
+
onSessionStoreWriteFailure({
|
|
1881
|
+
isDiskFailure,
|
|
1882
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1883
|
+
cause
|
|
1884
|
+
});
|
|
1765
1885
|
} catch (err) {
|
|
1766
1886
|
reportTelemetryDiagnostic(
|
|
1767
1887
|
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1768
1888
|
);
|
|
1769
1889
|
}
|
|
1770
1890
|
}
|
|
1771
|
-
const parsed = parseOpenCodeLogLine(line);
|
|
1772
1891
|
if (!parsed) return;
|
|
1773
1892
|
const now = Date.now();
|
|
1774
1893
|
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
@@ -1817,34 +1936,21 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
|
|
|
1817
1936
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1818
1937
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1819
1938
|
var MAX_METADATA_ENTRIES = 20;
|
|
1820
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1821
|
-
function redactHomeDirectory(message) {
|
|
1822
|
-
const home = homedir2();
|
|
1823
|
-
if (!home || home === "/") {
|
|
1824
|
-
return message;
|
|
1825
|
-
}
|
|
1826
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1827
|
-
let result = "";
|
|
1828
|
-
let searchFrom = 0;
|
|
1829
|
-
let homeIndex;
|
|
1830
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1831
|
-
const homeEnd = homeIndex + home.length;
|
|
1832
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1833
|
-
const nextCharacter = message[homeEnd];
|
|
1834
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1835
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1836
|
-
result += atPathBoundary ? "~" : home;
|
|
1837
|
-
searchFrom = homeEnd;
|
|
1838
|
-
}
|
|
1839
|
-
return result + message.slice(searchFrom);
|
|
1840
|
-
}
|
|
1841
|
-
function redactDiagnosticText(message) {
|
|
1842
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1843
|
-
return redactHomeDirectory(redacted);
|
|
1844
|
-
}
|
|
1845
1939
|
function truncate(message) {
|
|
1846
1940
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1847
|
-
|
|
1941
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1942
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1943
|
+
let previousDropped = -1;
|
|
1944
|
+
while (dropped !== previousDropped) {
|
|
1945
|
+
previousDropped = dropped;
|
|
1946
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1947
|
+
dropped = nextDropped;
|
|
1948
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1949
|
+
}
|
|
1950
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1951
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1952
|
+
const tailLength = remaining - headLength;
|
|
1953
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1848
1954
|
}
|
|
1849
1955
|
function sanitiseMetadata(metadata) {
|
|
1850
1956
|
if (!metadata) return {};
|
|
@@ -2054,12 +2160,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
2054
2160
|
metadata: withoutContractFields(record),
|
|
2055
2161
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
2056
2162
|
};
|
|
2057
|
-
case "session_db_boot_refused":
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2163
|
+
case "session_db_boot_refused": {
|
|
2164
|
+
const message = (() => {
|
|
2165
|
+
switch (record.reason) {
|
|
2166
|
+
case "local_discard_failed":
|
|
2167
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2168
|
+
case "replica_separation_unproven":
|
|
2169
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2170
|
+
case "verification_timeout_before_separation":
|
|
2171
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2172
|
+
case "verification_timeout_during_separation":
|
|
2173
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2174
|
+
default:
|
|
2175
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2176
|
+
}
|
|
2177
|
+
})();
|
|
2178
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2179
|
+
}
|
|
2063
2180
|
case "schema_provenance_mismatch":
|
|
2064
2181
|
return {
|
|
2065
2182
|
level,
|
|
@@ -2464,7 +2581,18 @@ async function deleteSession(port, id, client) {
|
|
|
2464
2581
|
{ method: "DELETE" },
|
|
2465
2582
|
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2466
2583
|
);
|
|
2467
|
-
|
|
2584
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2585
|
+
return { ok: true, status: res.status };
|
|
2586
|
+
}
|
|
2587
|
+
try {
|
|
2588
|
+
const body = await res.text();
|
|
2589
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2590
|
+
} catch (error2) {
|
|
2591
|
+
console.error(
|
|
2592
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2593
|
+
);
|
|
2594
|
+
return { ok: false, status: res.status };
|
|
2595
|
+
}
|
|
2468
2596
|
} catch (error2) {
|
|
2469
2597
|
console.error(
|
|
2470
2598
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
@@ -3064,6 +3192,18 @@ function messageError(messages, userMessageId) {
|
|
|
3064
3192
|
) : void 0;
|
|
3065
3193
|
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3066
3194
|
}
|
|
3195
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3196
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3197
|
+
if (!reply) return null;
|
|
3198
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3199
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3200
|
+
return {
|
|
3201
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3202
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3203
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3204
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3205
|
+
};
|
|
3206
|
+
}
|
|
3067
3207
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
3068
3208
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3069
3209
|
const error2 = errorOf(reply);
|
|
@@ -3637,7 +3777,18 @@ async function deleteV2Session(client, sessionId) {
|
|
|
3637
3777
|
{ method: "DELETE" },
|
|
3638
3778
|
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3639
3779
|
);
|
|
3640
|
-
|
|
3780
|
+
if (response.status >= 200 && response.status < 300) {
|
|
3781
|
+
return { ok: true, status: response.status };
|
|
3782
|
+
}
|
|
3783
|
+
try {
|
|
3784
|
+
const body = await response.text();
|
|
3785
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
3786
|
+
} catch (error2) {
|
|
3787
|
+
console.error(
|
|
3788
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3789
|
+
);
|
|
3790
|
+
return { ok: false, status: response.status };
|
|
3791
|
+
}
|
|
3641
3792
|
} catch (error2) {
|
|
3642
3793
|
console.error(
|
|
3643
3794
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
@@ -4346,9 +4497,12 @@ function runSynchroniser(args, opts) {
|
|
|
4346
4497
|
|
|
4347
4498
|
// src/lib/opencode/session-db-boot.ts
|
|
4348
4499
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
4349
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4500
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
4350
4501
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
4351
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4502
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4503
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4504
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4505
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
4352
4506
|
function commandError(result) {
|
|
4353
4507
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
4354
4508
|
}
|
|
@@ -4659,20 +4813,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4659
4813
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4660
4814
|
env: {
|
|
4661
4815
|
...env,
|
|
4662
|
-
// The synchroniser reads
|
|
4663
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4816
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4664
4817
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4665
4818
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4819
|
+
),
|
|
4820
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4821
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4822
|
+
),
|
|
4823
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4824
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4825
|
+
),
|
|
4826
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4827
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4666
4828
|
)
|
|
4667
4829
|
}
|
|
4668
4830
|
});
|
|
4669
4831
|
logSynchroniserDiagnostics(result, options);
|
|
4670
4832
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4833
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4834
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4835
|
+
options.log(
|
|
4836
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4837
|
+
"error"
|
|
4838
|
+
);
|
|
4839
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4840
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4841
|
+
}
|
|
4671
4842
|
options.log(
|
|
4672
4843
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4673
4844
|
"warn"
|
|
4674
4845
|
);
|
|
4675
|
-
return false;
|
|
4846
|
+
return { verifyFatal: false };
|
|
4676
4847
|
}
|
|
4677
4848
|
if (result.code === 34) {
|
|
4678
4849
|
reportRecord(
|
|
@@ -4682,14 +4853,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4682
4853
|
null,
|
|
4683
4854
|
options
|
|
4684
4855
|
);
|
|
4685
|
-
return true;
|
|
4856
|
+
return { verifyFatal: true };
|
|
4686
4857
|
}
|
|
4687
4858
|
if (result.code === 33) {
|
|
4688
4859
|
options.log(
|
|
4689
4860
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4690
4861
|
"warn"
|
|
4691
4862
|
);
|
|
4692
|
-
return false;
|
|
4863
|
+
return { verifyFatal: false };
|
|
4693
4864
|
}
|
|
4694
4865
|
if (result.code !== 0) {
|
|
4695
4866
|
options.log(
|
|
@@ -4697,13 +4868,13 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4697
4868
|
"warn"
|
|
4698
4869
|
);
|
|
4699
4870
|
}
|
|
4700
|
-
return false;
|
|
4871
|
+
return { verifyFatal: false };
|
|
4701
4872
|
}
|
|
4702
4873
|
options.log(
|
|
4703
4874
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4704
4875
|
"debug"
|
|
4705
4876
|
);
|
|
4706
|
-
return false;
|
|
4877
|
+
return { verifyFatal: false };
|
|
4707
4878
|
}
|
|
4708
4879
|
function fileExists(path) {
|
|
4709
4880
|
try {
|
|
@@ -4790,7 +4961,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4790
4961
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4791
4962
|
return { verifyFatal: false };
|
|
4792
4963
|
}
|
|
4793
|
-
return
|
|
4964
|
+
return verifySessionDb(options, configPath, env);
|
|
4794
4965
|
}
|
|
4795
4966
|
|
|
4796
4967
|
// src/lib/opencode/session-db-provenance.ts
|
|
@@ -6571,6 +6742,7 @@ function createSessionStoreHealth(options = {}) {
|
|
|
6571
6742
|
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6572
6743
|
let lastUnwritableAt = null;
|
|
6573
6744
|
let lastWritableAt = null;
|
|
6745
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6574
6746
|
return {
|
|
6575
6747
|
observeUnwritable: () => {
|
|
6576
6748
|
lastUnwritableAt = now();
|
|
@@ -6582,6 +6754,23 @@ function createSessionStoreHealth(options = {}) {
|
|
|
6582
6754
|
if (lastUnwritableAt === null) return false;
|
|
6583
6755
|
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6584
6756
|
return now() - lastUnwritableAt < stalenessMs;
|
|
6757
|
+
},
|
|
6758
|
+
observeCause: (sessionId, cause) => {
|
|
6759
|
+
if (sessionId === null) return;
|
|
6760
|
+
causes.delete(sessionId);
|
|
6761
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6762
|
+
while (causes.size > 256) {
|
|
6763
|
+
const oldest = causes.keys().next().value;
|
|
6764
|
+
if (typeof oldest !== "string") break;
|
|
6765
|
+
causes.delete(oldest);
|
|
6766
|
+
}
|
|
6767
|
+
},
|
|
6768
|
+
readCause: (sessionId) => {
|
|
6769
|
+
const observed = causes.get(sessionId);
|
|
6770
|
+
if (!observed) return null;
|
|
6771
|
+
causes.delete(sessionId);
|
|
6772
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6773
|
+
return observed.cause;
|
|
6585
6774
|
}
|
|
6586
6775
|
};
|
|
6587
6776
|
}
|
|
@@ -7022,6 +7211,11 @@ function messageIdOf(m) {
|
|
|
7022
7211
|
const infoId = m.info?.id;
|
|
7023
7212
|
return typeof infoId === "string" ? infoId : void 0;
|
|
7024
7213
|
}
|
|
7214
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7215
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7216
|
+
if (replyShape === null) return base;
|
|
7217
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7218
|
+
}
|
|
7025
7219
|
function cleanImageMime(contentType) {
|
|
7026
7220
|
if (!contentType) return null;
|
|
7027
7221
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -7042,6 +7236,10 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
7042
7236
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
7043
7237
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
7044
7238
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7239
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7240
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7241
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7242
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
7045
7243
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
7046
7244
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7047
7245
|
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
@@ -7199,6 +7397,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7199
7397
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
7200
7398
|
*/
|
|
7201
7399
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7400
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
7202
7401
|
/**
|
|
7203
7402
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
7204
7403
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -9039,6 +9238,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9039
9238
|
this.bufferedSessionErrors.delete(oldest);
|
|
9040
9239
|
}
|
|
9041
9240
|
}
|
|
9241
|
+
retainSessionError(event) {
|
|
9242
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9243
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9244
|
+
reason: event.reason,
|
|
9245
|
+
observedAt: this.now()
|
|
9246
|
+
});
|
|
9247
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9248
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9249
|
+
if (typeof oldest !== "string") break;
|
|
9250
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9251
|
+
}
|
|
9252
|
+
}
|
|
9253
|
+
readRetainedSessionError(sessionId) {
|
|
9254
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9255
|
+
if (!retained) return null;
|
|
9256
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9257
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9258
|
+
return null;
|
|
9259
|
+
}
|
|
9260
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9261
|
+
return retained.reason;
|
|
9262
|
+
}
|
|
9042
9263
|
/**
|
|
9043
9264
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
9044
9265
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -9365,9 +9586,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9365
9586
|
return;
|
|
9366
9587
|
}
|
|
9367
9588
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9589
|
+
this.retainSessionError(event);
|
|
9368
9590
|
this.log({
|
|
9369
|
-
level: "
|
|
9370
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9591
|
+
level: "warn",
|
|
9592
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
9371
9593
|
conversation_id: watcher.conv.id
|
|
9372
9594
|
});
|
|
9373
9595
|
return;
|
|
@@ -9397,9 +9619,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9397
9619
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
9398
9620
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
9399
9621
|
if (state !== "queued") {
|
|
9622
|
+
this.retainSessionError(event);
|
|
9400
9623
|
this.log({
|
|
9401
|
-
level: "
|
|
9402
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9624
|
+
level: "warn",
|
|
9625
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
9403
9626
|
conversation_id: watcher.conv.id,
|
|
9404
9627
|
message_id: inFlight.evidentMessageId
|
|
9405
9628
|
});
|
|
@@ -9628,7 +9851,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9628
9851
|
if (state === "failed") {
|
|
9629
9852
|
this.anchorDeliveryDeadline(inFlight);
|
|
9630
9853
|
if (!inFlight.done) {
|
|
9631
|
-
const
|
|
9854
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9855
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9856
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9857
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9858
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9859
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9860
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9861
|
+
(detail) => detail !== null
|
|
9862
|
+
);
|
|
9863
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9632
9864
|
this.log({
|
|
9633
9865
|
level: "error",
|
|
9634
9866
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9718,17 +9950,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9718
9950
|
}
|
|
9719
9951
|
} else {
|
|
9720
9952
|
if (inFlight.b2AbandonedSignalled) {
|
|
9953
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9954
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9721
9955
|
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9722
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9723
9956
|
await this.settleMessageAbandonedWithoutAnswer(
|
|
9724
9957
|
sessionId,
|
|
9725
9958
|
watcher,
|
|
9726
9959
|
inFlight,
|
|
9727
9960
|
messages,
|
|
9728
|
-
|
|
9961
|
+
finish
|
|
9729
9962
|
);
|
|
9730
9963
|
} else {
|
|
9731
|
-
await this.settleMessageDone(
|
|
9964
|
+
await this.settleMessageDone(
|
|
9965
|
+
sessionId,
|
|
9966
|
+
watcher,
|
|
9967
|
+
inFlight,
|
|
9968
|
+
messages,
|
|
9969
|
+
composeAbandonmentNote(
|
|
9970
|
+
finish,
|
|
9971
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
9972
|
+
"with-answer"
|
|
9973
|
+
)
|
|
9974
|
+
);
|
|
9732
9975
|
}
|
|
9733
9976
|
return;
|
|
9734
9977
|
}
|
|
@@ -9791,7 +10034,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9791
10034
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9792
10035
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9793
10036
|
});
|
|
9794
|
-
await this.settleMessageDone(
|
|
10037
|
+
await this.settleMessageDone(
|
|
10038
|
+
sessionId,
|
|
10039
|
+
watcher,
|
|
10040
|
+
inFlight,
|
|
10041
|
+
messages,
|
|
10042
|
+
composeAbandonmentNote(
|
|
10043
|
+
finish,
|
|
10044
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10045
|
+
"with-answer"
|
|
10046
|
+
)
|
|
10047
|
+
);
|
|
9795
10048
|
return;
|
|
9796
10049
|
}
|
|
9797
10050
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9991,15 +10244,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9991
10244
|
}
|
|
9992
10245
|
}
|
|
9993
10246
|
/**
|
|
9994
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9995
|
-
*
|
|
9996
|
-
*
|
|
9997
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9998
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9999
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
10000
|
-
* and risking the two copies silently drifting apart.
|
|
10247
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10248
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10249
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
10001
10250
|
*/
|
|
10002
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10251
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
10003
10252
|
const conv = watcher.conv;
|
|
10004
10253
|
this.anchorDeliveryDeadline(inFlight);
|
|
10005
10254
|
if (!inFlight.done) {
|
|
@@ -10026,7 +10275,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10026
10275
|
title,
|
|
10027
10276
|
usage,
|
|
10028
10277
|
usageAgentName,
|
|
10029
|
-
subagentInvocations
|
|
10278
|
+
subagentInvocations,
|
|
10279
|
+
note
|
|
10030
10280
|
);
|
|
10031
10281
|
} catch (err) {
|
|
10032
10282
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -10071,7 +10321,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10071
10321
|
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10072
10322
|
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10073
10323
|
const conv = watcher.conv;
|
|
10074
|
-
const error2 =
|
|
10324
|
+
const error2 = composeAbandonmentNote(
|
|
10325
|
+
finish,
|
|
10326
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10327
|
+
"without-answer"
|
|
10328
|
+
);
|
|
10075
10329
|
this.anchorDeliveryDeadline(inFlight);
|
|
10076
10330
|
this.log({
|
|
10077
10331
|
level: "warn",
|
|
@@ -10198,7 +10452,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10198
10452
|
*/
|
|
10199
10453
|
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10200
10454
|
const conv = watcher.conv;
|
|
10201
|
-
const error2 =
|
|
10455
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10202
10456
|
this.log({
|
|
10203
10457
|
level: "error",
|
|
10204
10458
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
@@ -11875,7 +12129,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11875
12129
|
* watcher retries next tick within the
|
|
11876
12130
|
* deadline, Finding 4).
|
|
11877
12131
|
*/
|
|
11878
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12132
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11879
12133
|
const res = await this.fetchImpl(
|
|
11880
12134
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11881
12135
|
{
|
|
@@ -11893,7 +12147,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11893
12147
|
...title ? { title } : {},
|
|
11894
12148
|
...usage ? usage : {},
|
|
11895
12149
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11896
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12150
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12151
|
+
...error2 ? { error: error2 } : {}
|
|
11897
12152
|
})
|
|
11898
12153
|
}
|
|
11899
12154
|
);
|
|
@@ -13565,6 +13820,9 @@ async function driveChannels(state, driver) {
|
|
|
13565
13820
|
}
|
|
13566
13821
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
13567
13822
|
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
13823
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
13824
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
13825
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
13568
13826
|
function shouldWarnForReclaimSkip(reason) {
|
|
13569
13827
|
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
13570
13828
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
@@ -13652,8 +13910,12 @@ async function runSweep(state, driver, config) {
|
|
|
13652
13910
|
protectedIds: protectedNow
|
|
13653
13911
|
}
|
|
13654
13912
|
);
|
|
13655
|
-
const
|
|
13656
|
-
|
|
13913
|
+
const eligibleToDelete = toDelete.filter(
|
|
13914
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
13915
|
+
);
|
|
13916
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13917
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
13918
|
+
const answeredFailures = [];
|
|
13657
13919
|
protectedNow = driver.protectedSessionIds();
|
|
13658
13920
|
for (const id of deleteCandidates) {
|
|
13659
13921
|
if (protectedNow.has(id)) {
|
|
@@ -13676,8 +13938,44 @@ async function runSweep(state, driver, config) {
|
|
|
13676
13938
|
if (statusClass === 4) failed4xx++;
|
|
13677
13939
|
else if (statusClass === 5) failed5xx++;
|
|
13678
13940
|
else failedOtherStatus++;
|
|
13941
|
+
answeredFailures.push({
|
|
13942
|
+
id,
|
|
13943
|
+
status: outcome.status,
|
|
13944
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
13945
|
+
});
|
|
13679
13946
|
}
|
|
13680
13947
|
}
|
|
13948
|
+
const givenUpFailures = [];
|
|
13949
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
13950
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
13951
|
+
} else {
|
|
13952
|
+
for (const failure of answeredFailures) {
|
|
13953
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
13954
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
13955
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
13956
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
13957
|
+
givenUpFailures.push(failure);
|
|
13958
|
+
}
|
|
13959
|
+
}
|
|
13960
|
+
}
|
|
13961
|
+
for (const failure of givenUpFailures) {
|
|
13962
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
13963
|
+
0,
|
|
13964
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
13965
|
+
);
|
|
13966
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
13967
|
+
type: "info",
|
|
13968
|
+
level: "warn",
|
|
13969
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
13970
|
+
metadata: {
|
|
13971
|
+
outcome: "session-delete-given-up",
|
|
13972
|
+
session_id: failure.id,
|
|
13973
|
+
status: failure.status,
|
|
13974
|
+
body_excerpt: excerpt,
|
|
13975
|
+
given_up_count: givenUpFailures.length
|
|
13976
|
+
}
|
|
13977
|
+
});
|
|
13978
|
+
}
|
|
13681
13979
|
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13682
13980
|
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13683
13981
|
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
@@ -13698,21 +13996,31 @@ async function runSweep(state, driver, config) {
|
|
|
13698
13996
|
sessions_remaining: remaining,
|
|
13699
13997
|
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13700
13998
|
};
|
|
13999
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
13701
14000
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
13702
14001
|
dbPath: sessionDbPath(),
|
|
13703
14002
|
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13704
14003
|
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13705
14004
|
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
13706
|
-
|
|
14005
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14006
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
13707
14007
|
});
|
|
13708
14008
|
if (reclaimResult.ok) {
|
|
13709
14009
|
state.sessionStoreHealth.observeWritable();
|
|
13710
14010
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
13711
14011
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
13712
14012
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
14013
|
+
const loopElapsedMs = Math.max(
|
|
14014
|
+
0,
|
|
14015
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14016
|
+
);
|
|
14017
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14018
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14019
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
13713
14020
|
logSessionCleanupActivity(state, "sweep", {
|
|
13714
14021
|
type: "info",
|
|
13715
|
-
|
|
14022
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14023
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
13716
14024
|
metadata: {
|
|
13717
14025
|
outcome: "reclaimed",
|
|
13718
14026
|
mode: reclaimResult.mode,
|
|
@@ -13725,6 +14033,7 @@ async function runSweep(state, driver, config) {
|
|
|
13725
14033
|
freelist_after: reclaimResult.freelistAfter,
|
|
13726
14034
|
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13727
14035
|
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14036
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
13728
14037
|
...sessionMetadata
|
|
13729
14038
|
}
|
|
13730
14039
|
});
|
|
@@ -14268,6 +14577,7 @@ async function run(options) {
|
|
|
14268
14577
|
messageCount: 0,
|
|
14269
14578
|
lastProxiedActivityAt: null,
|
|
14270
14579
|
sessionCleanupTimers: [],
|
|
14580
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
14271
14581
|
claudeUsageTimer: null,
|
|
14272
14582
|
claudeUsageRearm: null,
|
|
14273
14583
|
openaiUsageTimer: null,
|
|
@@ -14498,7 +14808,12 @@ async function run(options) {
|
|
|
14498
14808
|
agentId: state.agentId,
|
|
14499
14809
|
authHeader: state.authHeader
|
|
14500
14810
|
}),
|
|
14501
|
-
|
|
14811
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14812
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14813
|
+
if (cause !== null && sessionId !== null) {
|
|
14814
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14815
|
+
}
|
|
14816
|
+
}
|
|
14502
14817
|
);
|
|
14503
14818
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14504
14819
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
@@ -14509,22 +14824,27 @@ async function run(options) {
|
|
|
14509
14824
|
openCodeLogTail.stop();
|
|
14510
14825
|
};
|
|
14511
14826
|
let sessionDbVerifyFatal = false;
|
|
14827
|
+
let sessionDbVerifyFatalReason;
|
|
14512
14828
|
if (!options.restoreSessionDb) {
|
|
14513
|
-
|
|
14829
|
+
logActivity(state, {
|
|
14830
|
+
type: "info",
|
|
14831
|
+
level: "warn",
|
|
14832
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14833
|
+
});
|
|
14514
14834
|
} else {
|
|
14515
14835
|
const health = await checkOpenCodeHealth(state.port);
|
|
14516
14836
|
if (health.healthy) {
|
|
14517
|
-
|
|
14518
|
-
|
|
14519
|
-
|
|
14520
|
-
"
|
|
14521
|
-
);
|
|
14837
|
+
logActivity(state, {
|
|
14838
|
+
type: "info",
|
|
14839
|
+
level: "warn",
|
|
14840
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14841
|
+
});
|
|
14522
14842
|
} else if (health.authFailed) {
|
|
14523
|
-
|
|
14524
|
-
|
|
14525
|
-
|
|
14526
|
-
"
|
|
14527
|
-
);
|
|
14843
|
+
logActivity(state, {
|
|
14844
|
+
type: "info",
|
|
14845
|
+
level: "warn",
|
|
14846
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14847
|
+
});
|
|
14528
14848
|
} else {
|
|
14529
14849
|
const result = await restoreAndVerifySessionDb({
|
|
14530
14850
|
dbPath: sessionDbPath(),
|
|
@@ -14541,12 +14861,13 @@ async function run(options) {
|
|
|
14541
14861
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
14542
14862
|
});
|
|
14543
14863
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14864
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
14544
14865
|
}
|
|
14545
14866
|
}
|
|
14546
14867
|
reportSessionDbRecovery(state);
|
|
14547
14868
|
if (sessionDbVerifyFatal) {
|
|
14548
14869
|
throw new Error(
|
|
14549
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14870
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
14550
14871
|
);
|
|
14551
14872
|
}
|
|
14552
14873
|
applyRunnerOpenCodeConfig({
|