@evident-ai/cli 3.5.2-dev.3e69672 → 3.5.2-dev.4118041

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1322,7 +1322,7 @@ import ora4 from "ora";
1322
1322
  import { select as select4 } from "@inquirer/prompts";
1323
1323
 
1324
1324
  // src/lib/telemetry.ts
1325
- var CLI_VERSION = (true ? "3.5.2-dev.3e69672" : void 0) ?? process.env.npm_package_version ?? "unknown";
1325
+ var CLI_VERSION = (true ? "3.5.2-dev.4118041" : void 0) ?? process.env.npm_package_version ?? "unknown";
1326
1326
  function getCliVersion() {
1327
1327
  return CLI_VERSION;
1328
1328
  }
@@ -1483,20 +1483,47 @@ var EventTypes = {
1483
1483
  DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
1484
1484
  };
1485
1485
 
1486
- // src/lib/runner-activity-telemetry.ts
1487
- import { homedir as homedir2 } from "node:os";
1488
-
1489
1486
  // src/lib/opencode/session-db-reclaim.ts
1490
1487
  import { statSync as statSync2, statfsSync } from "node:fs";
1491
1488
  import { dirname as dirname2 } from "node:path";
1492
- var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
1489
+
1490
+ // src/lib/diagnostic-redaction.ts
1491
+ import { homedir as homedir2 } from "node:os";
1492
+ function redactHomeDirectory(message) {
1493
+ const home = homedir2();
1494
+ if (!home || home === "/") {
1495
+ return message;
1496
+ }
1497
+ const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
1498
+ let result = "";
1499
+ let searchFrom = 0;
1500
+ let homeIndex;
1501
+ while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
1502
+ const homeEnd = homeIndex + home.length;
1503
+ const previousCharacter = message[homeIndex - 1];
1504
+ const nextCharacter = message[homeEnd];
1505
+ const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
1506
+ result += message.slice(searchFrom, homeIndex);
1507
+ result += atPathBoundary ? "~" : home;
1508
+ searchFrom = homeEnd;
1509
+ }
1510
+ return result + message.slice(searchFrom);
1511
+ }
1512
+ function redactDiagnosticText(message) {
1513
+ const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
1514
+ return redactHomeDirectory(redacted);
1515
+ }
1516
+
1517
+ // src/lib/opencode/session-db-reclaim.ts
1518
+ var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
1493
1519
  var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
1494
1520
  var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
1521
+ var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
1495
1522
  function errorMessage(error2) {
1496
1523
  if (!(error2 instanceof Error)) return String(error2);
1497
1524
  return error2.cause instanceof Error ? error2.cause.message : error2.message;
1498
1525
  }
1499
- function isSessionStoreWriteFailure(error2) {
1526
+ function errorMessages(error2) {
1500
1527
  const messages = [];
1501
1528
  const visited = /* @__PURE__ */ new Set();
1502
1529
  let current = error2;
@@ -1520,12 +1547,81 @@ function isSessionStoreWriteFailure(error2) {
1520
1547
  }
1521
1548
  break;
1522
1549
  }
1523
- return messages.some(
1550
+ return messages;
1551
+ }
1552
+ function isSessionStoreWriteFailure(error2) {
1553
+ return errorMessages(error2).some(
1524
1554
  (message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
1525
1555
  message
1526
1556
  )
1527
1557
  );
1528
1558
  }
1559
+ var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
1560
+ var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
1561
+ var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
1562
+ function isCauseBearingText(text) {
1563
+ return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
1564
+ }
1565
+ function structuredLogFields(line) {
1566
+ const fields = [];
1567
+ for (let index = 0; index < line.length; index++) {
1568
+ if (line[index] === '"') {
1569
+ index++;
1570
+ while (index < line.length) {
1571
+ if (line[index] === "\\") {
1572
+ index++;
1573
+ } else if (line[index] === '"') {
1574
+ break;
1575
+ }
1576
+ index++;
1577
+ }
1578
+ continue;
1579
+ }
1580
+ const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
1581
+ const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
1582
+ if (!atFieldBoundary || key === null) continue;
1583
+ const valueStart = index + key.length;
1584
+ if (line[valueStart] !== '"') {
1585
+ const valueEnd2 = line.indexOf(" ", valueStart);
1586
+ fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
1587
+ index = valueEnd2 === -1 ? line.length : valueEnd2;
1588
+ continue;
1589
+ }
1590
+ let valueEnd = valueStart + 1;
1591
+ for (; valueEnd < line.length; valueEnd++) {
1592
+ if (line[valueEnd] === "\\") {
1593
+ valueEnd++;
1594
+ } else if (line[valueEnd] === '"') {
1595
+ break;
1596
+ }
1597
+ }
1598
+ fields.push(line.slice(valueStart + 1, valueEnd));
1599
+ index = valueEnd;
1600
+ }
1601
+ return fields;
1602
+ }
1603
+ function structuredCauseMessages(text) {
1604
+ const fields = structuredLogFields(text);
1605
+ if (fields.length > 0) return fields;
1606
+ return isCauseBearingText(text) ? [text] : [];
1607
+ }
1608
+ function causeTokenFromText(text) {
1609
+ const paramsIndex = text.search(PARAMS_FIELD);
1610
+ for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
1611
+ const match = pattern.exec(text);
1612
+ if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
1613
+ return match[1];
1614
+ }
1615
+ }
1616
+ return null;
1617
+ }
1618
+ function sessionStoreCauseToken(value) {
1619
+ for (const message of structuredCauseMessages(value)) {
1620
+ const token = causeTokenFromText(message);
1621
+ if (token !== null) return redactDiagnosticText(token).slice(0, 120);
1622
+ }
1623
+ return null;
1624
+ }
1529
1625
  function insufficientSpaceReason(dbPath, requiredBytes) {
1530
1626
  try {
1531
1627
  const fsStats = statfsSync(dirname2(dbPath));
@@ -1547,6 +1643,7 @@ function readFreelistCount(db) {
1547
1643
  return db.prepare("PRAGMA freelist_count").get().freelist_count;
1548
1644
  }
1549
1645
  function readCheckpointResult(db) {
1646
+ db.exec("PRAGMA busy_timeout=0");
1550
1647
  const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
1551
1648
  return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
1552
1649
  }
@@ -1564,7 +1661,7 @@ async function probeReclaimAvailability(input) {
1564
1661
  try {
1565
1662
  const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
1566
1663
  try {
1567
- db.exec("PRAGMA busy_timeout=5000");
1664
+ db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
1568
1665
  autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
1569
1666
  } finally {
1570
1667
  db.close();
@@ -1584,7 +1681,8 @@ async function reclaimSessionDbSpace(input) {
1584
1681
  timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
1585
1682
  chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
1586
1683
  now = () => performance.now(),
1587
- allowFullVacuum = true
1684
+ hasLiveTurn = false,
1685
+ hasProvenanceAnomaly = false
1588
1686
  } = input;
1589
1687
  let sqlite;
1590
1688
  try {
@@ -1600,17 +1698,19 @@ async function reclaimSessionDbSpace(input) {
1600
1698
  let db;
1601
1699
  try {
1602
1700
  db = new DatabaseSync(dbPath);
1603
- db.exec("PRAGMA busy_timeout=5000");
1701
+ db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
1604
1702
  const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
1605
1703
  if (autoVacuum === 0) {
1606
- if (!allowFullVacuum) {
1607
- console.warn(
1608
- `[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
1609
- );
1704
+ if (hasLiveTurn || hasProvenanceAnomaly) {
1705
+ const detail = [
1706
+ ...hasLiveTurn ? ["a session turn is live"] : [],
1707
+ ...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
1708
+ ].join(" and ");
1709
+ console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
1610
1710
  return {
1611
1711
  ok: false,
1612
1712
  skipped: "full-vacuum-blocked",
1613
- detail: "a session turn is live"
1713
+ detail
1614
1714
  };
1615
1715
  }
1616
1716
  const fileBytesForGuard = statSync2(dbPath).size;
@@ -1628,7 +1728,9 @@ async function reclaimSessionDbSpace(input) {
1628
1728
  db.exec("VACUUM");
1629
1729
  const afterBytes = readLogicalBytes(db);
1630
1730
  const freelistAfter = readFreelistCount(db);
1731
+ const checkpointStartedAt = now();
1631
1732
  const checkpoint = readCheckpointResult(db);
1733
+ const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
1632
1734
  return {
1633
1735
  ok: true,
1634
1736
  mode: "convert",
@@ -1639,10 +1741,18 @@ async function reclaimSessionDbSpace(input) {
1639
1741
  pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
1640
1742
  stoppedBy: "freelist-drained",
1641
1743
  elapsedMs: Math.max(0, now() - startedAt),
1744
+ checkpointElapsedMs,
1642
1745
  checkpoint
1643
1746
  };
1644
1747
  }
1645
1748
  if (autoVacuum === 2) {
1749
+ if (hasLiveTurn) {
1750
+ const detail = "a session turn is live";
1751
+ console.warn(
1752
+ `[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
1753
+ );
1754
+ return { ok: false, skipped: "live-turn", detail };
1755
+ }
1646
1756
  const beforeBytes = readLogicalBytes(db);
1647
1757
  const freelistBefore = readFreelistCount(db);
1648
1758
  const startedAt = now();
@@ -1667,7 +1777,9 @@ async function reclaimSessionDbSpace(input) {
1667
1777
  freelistAfter = readFreelistCount(db);
1668
1778
  }
1669
1779
  const afterBytes = readLogicalBytes(db);
1780
+ const checkpointStartedAt = now();
1670
1781
  const checkpoint = readCheckpointResult(db);
1782
+ const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
1671
1783
  return {
1672
1784
  ok: true,
1673
1785
  mode: "incremental",
@@ -1678,6 +1790,7 @@ async function reclaimSessionDbSpace(input) {
1678
1790
  pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
1679
1791
  stoppedBy,
1680
1792
  elapsedMs: Math.max(0, now() - startedAt),
1793
+ checkpointElapsedMs,
1681
1794
  checkpoint
1682
1795
  };
1683
1796
  }
@@ -1759,16 +1872,22 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
1759
1872
  }
1760
1873
  };
1761
1874
  const flushLine = (line) => {
1762
- if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
1875
+ const parsed = parseOpenCodeLogLine(line);
1876
+ const isDiskFailure = isSessionStoreWriteFailure(line);
1877
+ const cause = sessionStoreCauseToken(line);
1878
+ if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
1763
1879
  try {
1764
- onSessionStoreWriteFailure();
1880
+ onSessionStoreWriteFailure({
1881
+ isDiskFailure,
1882
+ sessionId: parsed?.sessionID ?? null,
1883
+ cause
1884
+ });
1765
1885
  } catch (err) {
1766
1886
  reportTelemetryDiagnostic(
1767
1887
  `Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
1768
1888
  );
1769
1889
  }
1770
1890
  }
1771
- const parsed = parseOpenCodeLogLine(line);
1772
1891
  if (!parsed) return;
1773
1892
  const now = Date.now();
1774
1893
  const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
@@ -1817,34 +1936,21 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
1817
1936
  var MAX_MESSAGE_LENGTH = 500;
1818
1937
  var MAX_METADATA_VALUE_LENGTH = 200;
1819
1938
  var MAX_METADATA_ENTRIES = 20;
1820
- var TRUNCATION_MARKER = "\u2026";
1821
- function redactHomeDirectory(message) {
1822
- const home = homedir2();
1823
- if (!home || home === "/") {
1824
- return message;
1825
- }
1826
- const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
1827
- let result = "";
1828
- let searchFrom = 0;
1829
- let homeIndex;
1830
- while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
1831
- const homeEnd = homeIndex + home.length;
1832
- const previousCharacter = message[homeIndex - 1];
1833
- const nextCharacter = message[homeEnd];
1834
- const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
1835
- result += message.slice(searchFrom, homeIndex);
1836
- result += atPathBoundary ? "~" : home;
1837
- searchFrom = homeEnd;
1838
- }
1839
- return result + message.slice(searchFrom);
1840
- }
1841
- function redactDiagnosticText(message) {
1842
- const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
1843
- return redactHomeDirectory(redacted);
1844
- }
1845
1939
  function truncate(message) {
1846
1940
  if (message.length <= MAX_MESSAGE_LENGTH) return message;
1847
- return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
1941
+ let dropped = message.length - MAX_MESSAGE_LENGTH;
1942
+ let marker = `\u2026${dropped} chars omitted\u2026`;
1943
+ let previousDropped = -1;
1944
+ while (dropped !== previousDropped) {
1945
+ previousDropped = dropped;
1946
+ const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
1947
+ dropped = nextDropped;
1948
+ marker = `\u2026${dropped} chars omitted\u2026`;
1949
+ }
1950
+ const remaining = MAX_MESSAGE_LENGTH - marker.length;
1951
+ const headLength = Math.ceil(remaining * 0.6);
1952
+ const tailLength = remaining - headLength;
1953
+ return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
1848
1954
  }
1849
1955
  function sanitiseMetadata(metadata) {
1850
1956
  if (!metadata) return {};
@@ -2054,12 +2160,23 @@ function buildSessionDbRecoveryActivity(record) {
2054
2160
  metadata: withoutContractFields(record),
2055
2161
  message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
2056
2162
  };
2057
- case "session_db_boot_refused":
2058
- return {
2059
- level,
2060
- metadata: withoutContractFields(record),
2061
- message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
2062
- };
2163
+ case "session_db_boot_refused": {
2164
+ const message = (() => {
2165
+ switch (record.reason) {
2166
+ case "local_discard_failed":
2167
+ return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
2168
+ case "replica_separation_unproven":
2169
+ return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
2170
+ case "verification_timeout_before_separation":
2171
+ return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
2172
+ case "verification_timeout_during_separation":
2173
+ return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
2174
+ default:
2175
+ return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
2176
+ }
2177
+ })();
2178
+ return { level, metadata: withoutContractFields(record), message };
2179
+ }
2063
2180
  case "schema_provenance_mismatch":
2064
2181
  return {
2065
2182
  level,
@@ -2464,7 +2581,18 @@ async function deleteSession(port, id, client) {
2464
2581
  { method: "DELETE" },
2465
2582
  { allowStatuses: ALL_HTTP_STATUSES }
2466
2583
  );
2467
- return { ok: res.status >= 200 && res.status < 300, status: res.status };
2584
+ if (res.status >= 200 && res.status < 300) {
2585
+ return { ok: true, status: res.status };
2586
+ }
2587
+ try {
2588
+ const body = await res.text();
2589
+ return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
2590
+ } catch (error2) {
2591
+ console.error(
2592
+ `[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
2593
+ );
2594
+ return { ok: false, status: res.status };
2595
+ }
2468
2596
  } catch (error2) {
2469
2597
  console.error(
2470
2598
  `[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
@@ -3064,6 +3192,18 @@ function messageError(messages, userMessageId) {
3064
3192
  ) : void 0;
3065
3193
  return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
3066
3194
  }
3195
+ function describeAbandonedReplyShape(messages, userMessageId) {
3196
+ const reply = findLastAssistantReplyFor(messages, userMessageId);
3197
+ if (!reply) return null;
3198
+ const parts = Array.isArray(reply.parts) ? reply.parts : [];
3199
+ const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
3200
+ return {
3201
+ partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
3202
+ droppedPartCount: Math.max(0, parts.length - 12),
3203
+ lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
3204
+ lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
3205
+ };
3206
+ }
3067
3207
  function isAbortedTerminalReply(messages, userMessageId) {
3068
3208
  const reply = findLastAssistantReplyFor(messages, userMessageId);
3069
3209
  const error2 = errorOf(reply);
@@ -3637,7 +3777,18 @@ async function deleteV2Session(client, sessionId) {
3637
3777
  { method: "DELETE" },
3638
3778
  { allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
3639
3779
  );
3640
- return { ok: response.status >= 200 && response.status < 300, status: response.status };
3780
+ if (response.status >= 200 && response.status < 300) {
3781
+ return { ok: true, status: response.status };
3782
+ }
3783
+ try {
3784
+ const body = await response.text();
3785
+ return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
3786
+ } catch (error2) {
3787
+ console.error(
3788
+ `[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
3789
+ );
3790
+ return { ok: false, status: response.status };
3791
+ }
3641
3792
  } catch (error2) {
3642
3793
  console.error(
3643
3794
  `[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
@@ -4346,9 +4497,12 @@ function runSynchroniser(args, opts) {
4346
4497
 
4347
4498
  // src/lib/opencode/session-db-boot.ts
4348
4499
  var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
4349
- var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
4500
+ var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
4350
4501
  var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
4351
- var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
4502
+ var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
4503
+ var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
4504
+ var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
4505
+ var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
4352
4506
  function commandError(result) {
4353
4507
  return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
4354
4508
  }
@@ -4659,20 +4813,37 @@ async function verifySessionDb(options, configPath, env) {
4659
4813
  timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
4660
4814
  env: {
4661
4815
  ...env,
4662
- // The synchroniser reads this value in SECONDS. Keep this at 120, not
4663
- // 120_000, so the walkback gives up before the outer process bound.
4816
+ // The synchroniser reads all four values in SECONDS.
4664
4817
  EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
4665
4818
  SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
4819
+ ),
4820
+ EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
4821
+ SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
4822
+ ),
4823
+ EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
4824
+ SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
4825
+ ),
4826
+ EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
4827
+ SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
4666
4828
  )
4667
4829
  }
4668
4830
  });
4669
4831
  logSynchroniserDiagnostics(result, options);
4670
4832
  if (result.timedOut || result.code === 124 || result.code === 137) {
4833
+ if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
4834
+ const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
4835
+ options.log(
4836
+ `SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
4837
+ "error"
4838
+ );
4839
+ reportRecord("verify", "session_db_boot_refused", reason, null, options);
4840
+ return { verifyFatal: true, verifyFatalReason: reason };
4841
+ }
4671
4842
  options.log(
4672
4843
  `SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
4673
4844
  "warn"
4674
4845
  );
4675
- return false;
4846
+ return { verifyFatal: false };
4676
4847
  }
4677
4848
  if (result.code === 34) {
4678
4849
  reportRecord(
@@ -4682,14 +4853,14 @@ async function verifySessionDb(options, configPath, env) {
4682
4853
  null,
4683
4854
  options
4684
4855
  );
4685
- return true;
4856
+ return { verifyFatal: true };
4686
4857
  }
4687
4858
  if (result.code === 33) {
4688
4859
  options.log(
4689
4860
  "SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
4690
4861
  "warn"
4691
4862
  );
4692
- return false;
4863
+ return { verifyFatal: false };
4693
4864
  }
4694
4865
  if (result.code !== 0) {
4695
4866
  options.log(
@@ -4697,13 +4868,13 @@ async function verifySessionDb(options, configPath, env) {
4697
4868
  "warn"
4698
4869
  );
4699
4870
  }
4700
- return false;
4871
+ return { verifyFatal: false };
4701
4872
  }
4702
4873
  options.log(
4703
4874
  "skipping session-DB verification: this boot's session DB was not proven safe to replicate",
4704
4875
  "debug"
4705
4876
  );
4706
- return false;
4877
+ return { verifyFatal: false };
4707
4878
  }
4708
4879
  function fileExists(path) {
4709
4880
  try {
@@ -4790,7 +4961,7 @@ async function restoreAndVerifySessionDb(options) {
4790
4961
  if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
4791
4962
  return { verifyFatal: false };
4792
4963
  }
4793
- return { verifyFatal: await verifySessionDb(options, configPath, env) };
4964
+ return verifySessionDb(options, configPath, env);
4794
4965
  }
4795
4966
 
4796
4967
  // src/lib/opencode/session-db-provenance.ts
@@ -6571,6 +6742,7 @@ function createSessionStoreHealth(options = {}) {
6571
6742
  const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
6572
6743
  let lastUnwritableAt = null;
6573
6744
  let lastWritableAt = null;
6745
+ const causes = /* @__PURE__ */ new Map();
6574
6746
  return {
6575
6747
  observeUnwritable: () => {
6576
6748
  lastUnwritableAt = now();
@@ -6582,6 +6754,23 @@ function createSessionStoreHealth(options = {}) {
6582
6754
  if (lastUnwritableAt === null) return false;
6583
6755
  if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
6584
6756
  return now() - lastUnwritableAt < stalenessMs;
6757
+ },
6758
+ observeCause: (sessionId, cause) => {
6759
+ if (sessionId === null) return;
6760
+ causes.delete(sessionId);
6761
+ causes.set(sessionId, { cause, observedAt: now() });
6762
+ while (causes.size > 256) {
6763
+ const oldest = causes.keys().next().value;
6764
+ if (typeof oldest !== "string") break;
6765
+ causes.delete(oldest);
6766
+ }
6767
+ },
6768
+ readCause: (sessionId) => {
6769
+ const observed = causes.get(sessionId);
6770
+ if (!observed) return null;
6771
+ causes.delete(sessionId);
6772
+ if (now() - observed.observedAt >= stalenessMs) return null;
6773
+ return observed.cause;
6585
6774
  }
6586
6775
  };
6587
6776
  }
@@ -7022,6 +7211,11 @@ function messageIdOf(m) {
7022
7211
  const infoId = m.info?.id;
7023
7212
  return typeof infoId === "string" ? infoId : void 0;
7024
7213
  }
7214
+ function composeAbandonmentNote(finish, replyShape, outcome) {
7215
+ const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
7216
+ if (replyShape === null) return base;
7217
+ return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
7218
+ }
7025
7219
  function cleanImageMime(contentType) {
7026
7220
  if (!contentType) return null;
7027
7221
  const media = contentType.split(";")[0].trim().toLowerCase();
@@ -7042,6 +7236,10 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
7042
7236
  var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
7043
7237
  var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
7044
7238
  var MAX_BUFFERED_SESSION_ERRORS = 256;
7239
+ var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
7240
+ var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
7241
+ var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
7242
+ var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
7045
7243
  var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
7046
7244
  var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
7047
7245
  var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
@@ -7199,6 +7397,7 @@ var ChannelDriver = class _ChannelDriver {
7199
7397
  * with a short TTL so an unmatched session cannot retain an event indefinitely.
7200
7398
  */
7201
7399
  bufferedSessionErrors = /* @__PURE__ */ new Map();
7400
+ retainedSessionErrors = /* @__PURE__ */ new Map();
7202
7401
  /**
7203
7402
  * AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
7204
7403
  * dispatched and are still in-flight. A message in this set is never
@@ -9039,6 +9238,28 @@ var ChannelDriver = class _ChannelDriver {
9039
9238
  this.bufferedSessionErrors.delete(oldest);
9040
9239
  }
9041
9240
  }
9241
+ retainSessionError(event) {
9242
+ this.retainedSessionErrors.delete(event.sessionId);
9243
+ this.retainedSessionErrors.set(event.sessionId, {
9244
+ reason: event.reason,
9245
+ observedAt: this.now()
9246
+ });
9247
+ while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
9248
+ const oldest = this.retainedSessionErrors.keys().next().value;
9249
+ if (typeof oldest !== "string") break;
9250
+ this.retainedSessionErrors.delete(oldest);
9251
+ }
9252
+ }
9253
+ readRetainedSessionError(sessionId) {
9254
+ const retained = this.retainedSessionErrors.get(sessionId);
9255
+ if (!retained) return null;
9256
+ if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
9257
+ this.retainedSessionErrors.delete(sessionId);
9258
+ return null;
9259
+ }
9260
+ this.retainedSessionErrors.delete(sessionId);
9261
+ return retained.reason;
9262
+ }
9042
9263
  /**
9043
9264
  * Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
9044
9265
  * EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
@@ -9365,9 +9586,10 @@ var ChannelDriver = class _ChannelDriver {
9365
9586
  return;
9366
9587
  }
9367
9588
  if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
9589
+ this.retainSessionError(event);
9368
9590
  this.log({
9369
- level: "debug",
9370
- message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
9591
+ level: "warn",
9592
+ message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
9371
9593
  conversation_id: watcher.conv.id
9372
9594
  });
9373
9595
  return;
@@ -9397,9 +9619,10 @@ var ChannelDriver = class _ChannelDriver {
9397
9619
  const messages = await this.getSessionMessages(event.sessionId);
9398
9620
  const state = messageRunState(messages, inFlight.opencodeMessageId);
9399
9621
  if (state !== "queued") {
9622
+ this.retainSessionError(event);
9400
9623
  this.log({
9401
- level: "debug",
9402
- message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state}; leaving it to transcript polling`,
9624
+ level: "warn",
9625
+ message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
9403
9626
  conversation_id: watcher.conv.id,
9404
9627
  message_id: inFlight.evidentMessageId
9405
9628
  });
@@ -9628,7 +9851,16 @@ var ChannelDriver = class _ChannelDriver {
9628
9851
  if (state === "failed") {
9629
9852
  this.anchorDeliveryDeadline(inFlight);
9630
9853
  if (!inFlight.done) {
9631
- const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
9854
+ const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
9855
+ const retainedReason = this.readRetainedSessionError(sessionId);
9856
+ const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
9857
+ const observedCause = this.sessionStoreHealth?.readCause(sessionId);
9858
+ const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
9859
+ const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
9860
+ const runnerDetails = [retainedError, causeError, storeError].filter(
9861
+ (detail) => detail !== null
9862
+ );
9863
+ const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
9632
9864
  this.log({
9633
9865
  level: "error",
9634
9866
  message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
@@ -9718,17 +9950,28 @@ var ChannelDriver = class _ChannelDriver {
9718
9950
  }
9719
9951
  } else {
9720
9952
  if (inFlight.b2AbandonedSignalled) {
9953
+ const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
9954
+ const finish = reply?.info?.finish ?? reply?.finish;
9721
9955
  if (inFlight.b2AbandonedWithoutAnswer) {
9722
- const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
9723
9956
  await this.settleMessageAbandonedWithoutAnswer(
9724
9957
  sessionId,
9725
9958
  watcher,
9726
9959
  inFlight,
9727
9960
  messages,
9728
- reply?.info?.finish ?? reply?.finish
9961
+ finish
9729
9962
  );
9730
9963
  } else {
9731
- await this.settleMessageDone(sessionId, watcher, inFlight, messages);
9964
+ await this.settleMessageDone(
9965
+ sessionId,
9966
+ watcher,
9967
+ inFlight,
9968
+ messages,
9969
+ composeAbandonmentNote(
9970
+ finish,
9971
+ describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
9972
+ "with-answer"
9973
+ )
9974
+ );
9732
9975
  }
9733
9976
  return;
9734
9977
  }
@@ -9791,7 +10034,17 @@ var ChannelDriver = class _ChannelDriver {
9791
10034
  ...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
9792
10035
  opencode_message_id: inFlight.opencodeMessageId
9793
10036
  });
9794
- await this.settleMessageDone(sessionId, watcher, inFlight, messages);
10037
+ await this.settleMessageDone(
10038
+ sessionId,
10039
+ watcher,
10040
+ inFlight,
10041
+ messages,
10042
+ composeAbandonmentNote(
10043
+ finish,
10044
+ describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
10045
+ "with-answer"
10046
+ )
10047
+ );
9795
10048
  return;
9796
10049
  }
9797
10050
  if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
@@ -9991,15 +10244,11 @@ var ChannelDriver = class _ChannelDriver {
9991
10244
  }
9992
10245
  }
9993
10246
  /**
9994
- * Settle a message whose run-state has resolved `'done'` — extracted verbatim
9995
- * (pure refactor, no behavior change) from `serviceInFlightMessage`'s former
9996
- * inline `state === 'done'` branch body, so a SECOND caller (the #721
9997
- * b2-abandonment resolution) can reach the exact same completion behavior
9998
- * (delivery-deadline anchoring, title resolution, usage extraction, and
9999
- * `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
10000
- * and risking the two copies silently drifting apart.
10247
+ * Settle a message whose run-state has resolved `'done'`. Normal completion and
10248
+ * confirmed b2 abandonment share the same delivery, usage, and terminal handling.
10249
+ * Normal completions omit the optional terminal diagnostic.
10001
10250
  */
10002
- async settleMessageDone(sessionId, watcher, inFlight, messages) {
10251
+ async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
10003
10252
  const conv = watcher.conv;
10004
10253
  this.anchorDeliveryDeadline(inFlight);
10005
10254
  if (!inFlight.done) {
@@ -10026,7 +10275,8 @@ var ChannelDriver = class _ChannelDriver {
10026
10275
  title,
10027
10276
  usage,
10028
10277
  usageAgentName,
10029
- subagentInvocations
10278
+ subagentInvocations,
10279
+ note
10030
10280
  );
10031
10281
  } catch (err) {
10032
10282
  if (err instanceof ChannelAuthError) throw err;
@@ -10071,7 +10321,11 @@ var ChannelDriver = class _ChannelDriver {
10071
10321
  /** Report a confirmed b2 abandonment that produced no answer text. */
10072
10322
  async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
10073
10323
  const conv = watcher.conv;
10074
- const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
10324
+ const error2 = composeAbandonmentNote(
10325
+ finish,
10326
+ describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
10327
+ "without-answer"
10328
+ );
10075
10329
  this.anchorDeliveryDeadline(inFlight);
10076
10330
  this.log({
10077
10331
  level: "warn",
@@ -10198,7 +10452,7 @@ var ChannelDriver = class _ChannelDriver {
10198
10452
  */
10199
10453
  async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
10200
10454
  const conv = watcher.conv;
10201
- const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
10455
+ const error2 = SESSION_STORE_UNWRITABLE_ERROR;
10202
10456
  this.log({
10203
10457
  level: "error",
10204
10458
  message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
@@ -11875,7 +12129,7 @@ var ChannelDriver = class _ChannelDriver {
11875
12129
  * watcher retries next tick within the
11876
12130
  * deadline, Finding 4).
11877
12131
  */
11878
- async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
12132
+ async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
11879
12133
  const res = await this.fetchImpl(
11880
12134
  `${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
11881
12135
  {
@@ -11893,7 +12147,8 @@ var ChannelDriver = class _ChannelDriver {
11893
12147
  ...title ? { title } : {},
11894
12148
  ...usage ? usage : {},
11895
12149
  ...usageAgentName ? { usage_agent_name: usageAgentName } : {},
11896
- ...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
12150
+ ...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
12151
+ ...error2 ? { error: error2 } : {}
11897
12152
  })
11898
12153
  }
11899
12154
  );
@@ -13565,6 +13820,9 @@ async function driveChannels(state, driver) {
13565
13820
  }
13566
13821
  var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
13567
13822
  var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
13823
+ var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
13824
+ var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
13825
+ var MAX_SESSION_DELETE_BODY_LENGTH = 200;
13568
13826
  function shouldWarnForReclaimSkip(reason) {
13569
13827
  if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
13570
13828
  const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
@@ -13652,8 +13910,12 @@ async function runSweep(state, driver, config) {
13652
13910
  protectedIds: protectedNow
13653
13911
  }
13654
13912
  );
13655
- const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
13656
- remaining = toDelete.length - deleteCandidates.length;
13913
+ const eligibleToDelete = toDelete.filter(
13914
+ (id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
13915
+ );
13916
+ const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
13917
+ remaining = eligibleToDelete.length - deleteCandidates.length;
13918
+ const answeredFailures = [];
13657
13919
  protectedNow = driver.protectedSessionIds();
13658
13920
  for (const id of deleteCandidates) {
13659
13921
  if (protectedNow.has(id)) {
@@ -13676,8 +13938,44 @@ async function runSweep(state, driver, config) {
13676
13938
  if (statusClass === 4) failed4xx++;
13677
13939
  else if (statusClass === 5) failed5xx++;
13678
13940
  else failedOtherStatus++;
13941
+ answeredFailures.push({
13942
+ id,
13943
+ status: outcome.status,
13944
+ ...outcome.body !== void 0 ? { body: outcome.body } : {}
13945
+ });
13679
13946
  }
13680
13947
  }
13948
+ const givenUpFailures = [];
13949
+ if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
13950
+ for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
13951
+ } else {
13952
+ for (const failure of answeredFailures) {
13953
+ const previous = state.sessionDeleteFailures.get(failure.id);
13954
+ const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
13955
+ state.sessionDeleteFailures.set(failure.id, record);
13956
+ if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
13957
+ givenUpFailures.push(failure);
13958
+ }
13959
+ }
13960
+ }
13961
+ for (const failure of givenUpFailures) {
13962
+ const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
13963
+ 0,
13964
+ MAX_SESSION_DELETE_BODY_LENGTH
13965
+ );
13966
+ logSessionCleanupActivity(state, "sweep", {
13967
+ type: "info",
13968
+ level: "warn",
13969
+ message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
13970
+ metadata: {
13971
+ outcome: "session-delete-given-up",
13972
+ session_id: failure.id,
13973
+ status: failure.status,
13974
+ body_excerpt: excerpt,
13975
+ given_up_count: givenUpFailures.length
13976
+ }
13977
+ });
13978
+ }
13681
13979
  const failedNote = failed > 0 ? `, failed ${failed}` : "";
13682
13980
  const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
13683
13981
  const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
@@ -13698,21 +13996,31 @@ async function runSweep(state, driver, config) {
13698
13996
  sessions_remaining: remaining,
13699
13997
  ...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
13700
13998
  };
13999
+ const protectedForReclaim = driver.protectedSessionIds();
13701
14000
  const reclaimResult = await reclaimSessionDbSpace({
13702
14001
  dbPath: sessionDbPath(),
13703
14002
  chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
13704
14003
  pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
13705
14004
  timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
13706
- allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
14005
+ hasLiveTurn: protectedForReclaim.size > 0,
14006
+ hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
13707
14007
  });
13708
14008
  if (reclaimResult.ok) {
13709
14009
  state.sessionStoreHealth.observeWritable();
13710
14010
  const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
13711
14011
  const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
13712
14012
  const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
14013
+ const loopElapsedMs = Math.max(
14014
+ 0,
14015
+ reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
14016
+ );
14017
+ const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
14018
+ const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
14019
+ const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
13713
14020
  logSessionCleanupActivity(state, "sweep", {
13714
14021
  type: "info",
13715
- message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
14022
+ level: reclaimIsSlow ? "warn" : "info",
14023
+ message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
13716
14024
  metadata: {
13717
14025
  outcome: "reclaimed",
13718
14026
  mode: reclaimResult.mode,
@@ -13725,6 +14033,7 @@ async function runSweep(state, driver, config) {
13725
14033
  freelist_after: reclaimResult.freelistAfter,
13726
14034
  pages_reclaimed: reclaimResult.pagesReclaimed,
13727
14035
  reclaim_elapsed_ms: reclaimResult.elapsedMs,
14036
+ checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
13728
14037
  ...sessionMetadata
13729
14038
  }
13730
14039
  });
@@ -14268,6 +14577,7 @@ async function run(options) {
14268
14577
  messageCount: 0,
14269
14578
  lastProxiedActivityAt: null,
14270
14579
  sessionCleanupTimers: [],
14580
+ sessionDeleteFailures: /* @__PURE__ */ new Map(),
14271
14581
  claudeUsageTimer: null,
14272
14582
  claudeUsageRearm: null,
14273
14583
  openaiUsageTimer: null,
@@ -14498,7 +14808,12 @@ async function run(options) {
14498
14808
  agentId: state.agentId,
14499
14809
  authHeader: state.authHeader
14500
14810
  }),
14501
- state.sessionStoreHealth.observeUnwritable
14811
+ ({ isDiskFailure, sessionId, cause }) => {
14812
+ if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
14813
+ if (cause !== null && sessionId !== null) {
14814
+ state.sessionStoreHealth.observeCause(sessionId, cause);
14815
+ }
14816
+ }
14502
14817
  );
14503
14818
  const openCodeLogTail = tailOpenCodeLogFile(
14504
14819
  resolveOpenCodeLogPath(homedir7(), process.env),
@@ -14509,22 +14824,27 @@ async function run(options) {
14509
14824
  openCodeLogTail.stop();
14510
14825
  };
14511
14826
  let sessionDbVerifyFatal = false;
14827
+ let sessionDbVerifyFatalReason;
14512
14828
  if (!options.restoreSessionDb) {
14513
- log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
14829
+ logActivity(state, {
14830
+ type: "info",
14831
+ level: "warn",
14832
+ message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
14833
+ });
14514
14834
  } else {
14515
14835
  const health = await checkOpenCodeHealth(state.port);
14516
14836
  if (health.healthy) {
14517
- log2(
14518
- state,
14519
- "Skipping session-DB restore: OpenCode is already serving this database",
14520
- "debug"
14521
- );
14837
+ logActivity(state, {
14838
+ type: "info",
14839
+ level: "warn",
14840
+ message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
14841
+ });
14522
14842
  } else if (health.authFailed) {
14523
- log2(
14524
- state,
14525
- "Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use",
14526
- "debug"
14527
- );
14843
+ logActivity(state, {
14844
+ type: "info",
14845
+ level: "warn",
14846
+ message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
14847
+ });
14528
14848
  } else {
14529
14849
  const result = await restoreAndVerifySessionDb({
14530
14850
  dbPath: sessionDbPath(),
@@ -14541,12 +14861,13 @@ async function run(options) {
14541
14861
  reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
14542
14862
  });
14543
14863
  sessionDbVerifyFatal = result.verifyFatal;
14864
+ sessionDbVerifyFatalReason = result.verifyFatalReason;
14544
14865
  }
14545
14866
  }
14546
14867
  reportSessionDbRecovery(state);
14547
14868
  if (sessionDbVerifyFatal) {
14548
14869
  throw new Error(
14549
- "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
14870
+ sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
14550
14871
  );
14551
14872
  }
14552
14873
  applyRunnerOpenCodeConfig({