@evident-ai/cli 3.3.1-dev.e98fa27 → 3.4.1-dev.11f0c53

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -722,6 +722,14 @@ function toReportedWindow(window) {
722
722
  if (!window) return null;
723
723
  return { utilization: window.utilization, resets_at: window.resetsAt };
724
724
  }
725
+ function toReportedOwner(snapshot) {
726
+ if (!snapshot.owner) return null;
727
+ return {
728
+ email: snapshot.owner.email,
729
+ organization_name: snapshot.owner.organizationName,
730
+ rate_limit_tier: snapshot.owner.rateLimitTier
731
+ };
732
+ }
725
733
  async function reportClaudeUsage(agentId, authHeader, snapshot) {
726
734
  try {
727
735
  const apiUrl = getApiUrlConfig();
@@ -730,7 +738,71 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
730
738
  headers: { Authorization: authHeader, "Content-Type": "application/json" },
731
739
  body: JSON.stringify({
732
740
  five_hour: toReportedWindow(snapshot.fiveHour),
733
- seven_day: toReportedWindow(snapshot.sevenDay)
741
+ seven_day: toReportedWindow(snapshot.sevenDay),
742
+ owner: toReportedOwner(snapshot)
743
+ }),
744
+ signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
745
+ });
746
+ if (!response.ok) {
747
+ const serverMessage = await readErrorMessage(response);
748
+ return {
749
+ ok: false,
750
+ error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
751
+ };
752
+ }
753
+ return { ok: true };
754
+ } catch (error2) {
755
+ return { ok: false, error: describeBestEffortError(error2) };
756
+ }
757
+ }
758
+ function toReportedOpenAiWindow(window) {
759
+ if (!window) return null;
760
+ return {
761
+ utilization: window.utilization,
762
+ window_minutes: window.windowMinutes,
763
+ resets_at: window.resetsAt
764
+ };
765
+ }
766
+ async function reportOpenAiUsage(agentId, authHeader, snapshot) {
767
+ try {
768
+ const apiUrl = getApiUrlConfig();
769
+ const response = await fetch(`${apiUrl}/runners/${agentId}/openai-usage`, {
770
+ method: "POST",
771
+ headers: { Authorization: authHeader, "Content-Type": "application/json" },
772
+ body: JSON.stringify({
773
+ primary: toReportedOpenAiWindow(snapshot.primary),
774
+ secondary: toReportedOpenAiWindow(snapshot.secondary),
775
+ has_credits: snapshot.hasCredits,
776
+ credits_unlimited: snapshot.creditsUnlimited
777
+ }),
778
+ signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
779
+ });
780
+ if (!response.ok) {
781
+ const serverMessage = await readErrorMessage(response);
782
+ return {
783
+ ok: false,
784
+ error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
785
+ };
786
+ }
787
+ return { ok: true };
788
+ } catch (error2) {
789
+ return { ok: false, error: describeBestEffortError(error2) };
790
+ }
791
+ }
792
+ async function reportResourceUsage(agentId, authHeader, usage) {
793
+ try {
794
+ const apiUrl = getApiUrlConfig();
795
+ const response = await fetch(`${apiUrl}/runners/${agentId}/resource-usage`, {
796
+ method: "POST",
797
+ headers: { Authorization: authHeader, "Content-Type": "application/json" },
798
+ body: JSON.stringify({
799
+ cpu_percent: usage.cpuPercent,
800
+ cpu_count: usage.cpuCount,
801
+ memory_total_bytes: usage.memoryTotalBytes,
802
+ memory_available_bytes: usage.memoryAvailableBytes,
803
+ disk_total_bytes: usage.diskTotalBytes,
804
+ disk_free_bytes: usage.diskFreeBytes,
805
+ opencode_db_bytes: usage.opencodeDbBytes
734
806
  }),
735
807
  signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
736
808
  });
@@ -933,7 +1005,11 @@ import { readFileSync } from "fs";
933
1005
  import { homedir } from "os";
934
1006
  import { join } from "path";
935
1007
  var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
1008
+ var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
1009
+ var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
936
1010
  var KEYCHAIN_SERVICE = "Claude Code-credentials";
1011
+ var cachedOwner = null;
1012
+ var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
937
1013
  function parseClaudeCliCredentials(raw) {
938
1014
  let parsed;
939
1015
  try {
@@ -967,7 +1043,7 @@ function readClaudeCliCredentials() {
967
1043
  }
968
1044
  }
969
1045
  try {
970
- const raw = readFileSync(join(homedir(), ".claude", ".credentials.json"), "utf-8");
1046
+ const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
971
1047
  return parseClaudeCliCredentials(raw);
972
1048
  } catch (err) {
973
1049
  const code = err.code;
@@ -1006,6 +1082,47 @@ function toWindow(value) {
1006
1082
  }
1007
1083
  return { utilization: window.utilization, resetsAt };
1008
1084
  }
1085
+ function ownerLookupFailure(error2) {
1086
+ const name = error2?.name;
1087
+ return name === "TimeoutError" || name === "AbortError" ? "timed out" : "request failed";
1088
+ }
1089
+ async function getClaudeUsageOwner(accessToken) {
1090
+ if (cachedOwner?.accessToken === accessToken) {
1091
+ return { owner: cachedOwner.owner, ownerLookupError: null };
1092
+ }
1093
+ try {
1094
+ const response = await fetch(CLAUDE_PROFILE_URL, {
1095
+ headers: {
1096
+ Authorization: `Bearer ${accessToken}`,
1097
+ "Content-Type": "application/json",
1098
+ "anthropic-version": "2023-06-01"
1099
+ },
1100
+ signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
1101
+ });
1102
+ if (!response.ok) {
1103
+ return { owner: null, ownerLookupError: `HTTP ${response.status}` };
1104
+ }
1105
+ let body;
1106
+ try {
1107
+ body = await response.json();
1108
+ } catch (error2) {
1109
+ return { owner: null, ownerLookupError: "malformed response" };
1110
+ }
1111
+ const profile = body;
1112
+ if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
1113
+ return { owner: null, ownerLookupError: "malformed response" };
1114
+ }
1115
+ const owner = {
1116
+ email: profile.account.email,
1117
+ organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
1118
+ rateLimitTier: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
1119
+ };
1120
+ cachedOwner = { accessToken, owner };
1121
+ return { owner, ownerLookupError: null };
1122
+ } catch (error2) {
1123
+ return { owner: null, ownerLookupError: ownerLookupFailure(error2) };
1124
+ }
1125
+ }
1009
1126
  async function getClaudeUsage() {
1010
1127
  const credentials2 = readClaudeCliCredentials();
1011
1128
  if (!credentials2) {
@@ -1025,15 +1142,19 @@ async function getClaudeUsage() {
1025
1142
  Authorization: `Bearer ${credentials2.accessToken}`,
1026
1143
  "Content-Type": "application/json",
1027
1144
  "anthropic-version": "2023-06-01"
1028
- }
1145
+ },
1146
+ signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
1029
1147
  });
1030
1148
  if (!res.ok) {
1031
1149
  throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
1032
1150
  }
1033
1151
  const body = await res.json();
1152
+ const { owner, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
1034
1153
  return {
1035
1154
  fiveHour: toWindow(body.five_hour),
1036
- sevenDay: toWindow(body.seven_day)
1155
+ sevenDay: toWindow(body.seven_day),
1156
+ owner,
1157
+ ownerLookupError
1037
1158
  };
1038
1159
  }
1039
1160
 
@@ -1062,8 +1183,8 @@ async function claudeUsage() {
1062
1183
  }
1063
1184
 
1064
1185
  // src/commands/run.ts
1065
- import { homedir as homedir3 } from "os";
1066
- import { isAbsolute as isAbsolute2, join as join4, parse, resolve as resolvePath } from "path";
1186
+ import { homedir as homedir4 } from "os";
1187
+ import { isAbsolute as isAbsolute2, join as join7, parse, resolve as resolvePath } from "path";
1067
1188
  import chalk6 from "chalk";
1068
1189
 
1069
1190
  // ../../packages/types/src/agents/index.ts
@@ -1296,6 +1417,8 @@ var SEVERITY_BY_LEVEL = {
1296
1417
  error: "error"
1297
1418
  };
1298
1419
  var MAX_MESSAGE_LENGTH = 500;
1420
+ var MAX_METADATA_VALUE_LENGTH = 200;
1421
+ var MAX_METADATA_ENTRIES = 20;
1299
1422
  var TRUNCATION_MARKER = "\u2026";
1300
1423
  function redact(message) {
1301
1424
  return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
@@ -1304,6 +1427,24 @@ function truncate(message) {
1304
1427
  if (message.length <= MAX_MESSAGE_LENGTH) return message;
1305
1428
  return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
1306
1429
  }
1430
+ function sanitiseMetadata(metadata) {
1431
+ if (!metadata) return {};
1432
+ const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
1433
+ if (Object.keys(metadata).length > entries.length) {
1434
+ console.error(
1435
+ `[runner-activity-telemetry] dropped ${Object.keys(metadata).length - entries.length} metadata entries`
1436
+ );
1437
+ }
1438
+ const sanitised = [];
1439
+ for (const [key, value] of entries) {
1440
+ if (typeof value === "string") {
1441
+ sanitised.push([key, truncate(redact(value).slice(0, MAX_METADATA_VALUE_LENGTH))]);
1442
+ } else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
1443
+ sanitised.push([key, value]);
1444
+ }
1445
+ }
1446
+ return Object.fromEntries(sanitised);
1447
+ }
1307
1448
  var RATE_LIMIT_WINDOW_MS = 6e4;
1308
1449
  var RATE_LIMIT_MAX_EVENTS = 30;
1309
1450
  var windowStartedAt = 0;
@@ -1342,7 +1483,7 @@ function forwardRunnerActivity(entry, context) {
1342
1483
  logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
1343
1484
  severity: SEVERITY_BY_LEVEL[entry.level],
1344
1485
  message,
1345
- metadata: { source: "cli.run" },
1486
+ metadata: { ...sanitiseMetadata(entry.metadata), source: "cli.run" },
1346
1487
  agentId: context.agentId
1347
1488
  });
1348
1489
  } catch (err) {
@@ -1352,6 +1493,150 @@ function forwardRunnerActivity(entry, context) {
1352
1493
  }
1353
1494
  }
1354
1495
 
1496
+ // src/lib/opencode/session-db-recovery-report.ts
1497
+ import { readFileSync as readFileSync2, unlinkSync } from "fs";
1498
+ import { join as join2 } from "path";
1499
+ function sessionDbRecoveryReportPath(homeDir, env) {
1500
+ const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
1501
+ return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
1502
+ }
1503
+ function drainSessionDbRecoveryReport({
1504
+ homeDir,
1505
+ env
1506
+ }) {
1507
+ const path = sessionDbRecoveryReportPath(homeDir, env);
1508
+ let content;
1509
+ try {
1510
+ content = readFileSync2(path, "utf8");
1511
+ } catch (error2) {
1512
+ if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")
1513
+ return { path, records: [], skippedLines: 0, readError: null };
1514
+ const readError = error2 instanceof Error ? error2.message : String(error2);
1515
+ console.error(`[session-db-recovery-report] could not read ${path}: ${readError}`);
1516
+ return { path, records: [], skippedLines: 0, readError };
1517
+ }
1518
+ let skippedLines = 0;
1519
+ const records = content.split("\n").flatMap((line) => {
1520
+ if (!line.trim()) return [];
1521
+ try {
1522
+ const value = JSON.parse(line);
1523
+ if (!isSessionDbRecoveryRecord(value)) {
1524
+ skippedLines++;
1525
+ return [];
1526
+ }
1527
+ return [value];
1528
+ } catch (error2) {
1529
+ skippedLines++;
1530
+ console.error(
1531
+ `[session-db-recovery-report] skipped malformed record in ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
1532
+ );
1533
+ return [];
1534
+ }
1535
+ });
1536
+ return { path, records, skippedLines, readError: null };
1537
+ }
1538
+ function acknowledgeSessionDbRecoveryReport(path) {
1539
+ try {
1540
+ unlinkSync(path);
1541
+ } catch (error2) {
1542
+ if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
1543
+ console.error(
1544
+ `[session-db-recovery-report] could not remove ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
1545
+ );
1546
+ }
1547
+ }
1548
+ function buildSessionDbRecoveryActivity(record) {
1549
+ const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
1550
+ if (!level) return null;
1551
+ const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
1552
+ switch (record.outcome) {
1553
+ case "fresh_session_db":
1554
+ return {
1555
+ level,
1556
+ metadata: withoutContractFields(record),
1557
+ message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.`
1558
+ };
1559
+ case "restore_retried":
1560
+ return {
1561
+ level,
1562
+ metadata: withoutContractFields(record),
1563
+ message: "The initial session database restore failed; the runner will retry it. Monitor the runner backup for another restore failure."
1564
+ };
1565
+ case "replica_recovered":
1566
+ if (record.reason === "quarantine")
1567
+ return {
1568
+ level,
1569
+ metadata: withoutContractFields(record),
1570
+ message: `Session database recovery quarantined ${record.quarantined_objects ?? "unknown"} objects (${record.quarantined_bytes ?? "unknown"} bytes); ${record.quarantine_failed_objects ?? "unknown"} moves failed. Review the preserved backup at ${record.quarantine_destination ?? "an unknown destination"} before deleting it.`
1571
+ };
1572
+ if (record.reason === "prune")
1573
+ return {
1574
+ level,
1575
+ metadata: withoutContractFields(record),
1576
+ message: "Session database recovery discarded a damaged newest backup and retried. Sessions recorded after the previous backup point may be unavailable. Review the runner backup for another restore failure."
1577
+ };
1578
+ if (record.reason === "clear")
1579
+ return {
1580
+ level,
1581
+ metadata: withoutContractFields(record),
1582
+ message: "Session database recovery deleted the damaged backup and prior session history is unavailable. Review the runner backup configuration before relying on restored session history."
1583
+ };
1584
+ return null;
1585
+ case "history_rolled_back":
1586
+ return {
1587
+ level,
1588
+ metadata: withoutContractFields(record),
1589
+ message: `Session history was rolled back to verified restore point ${record.verified_restore_point ?? "unknown"}; everything after it is unavailable. Review the runner backup for another restore failure.`
1590
+ };
1591
+ case "restore_misconfigured":
1592
+ return {
1593
+ level,
1594
+ metadata: withoutContractFields(record),
1595
+ message: "This runner started with a fresh session database because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions."
1596
+ };
1597
+ case "session_db_boot_refused":
1598
+ return {
1599
+ level,
1600
+ metadata: withoutContractFields(record),
1601
+ message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
1602
+ };
1603
+ default:
1604
+ return null;
1605
+ }
1606
+ }
1607
+ function withoutContractFields(record) {
1608
+ const { v: _v, event: _event, ...metadata } = record;
1609
+ return metadata;
1610
+ }
1611
+ var OUTCOMES = /* @__PURE__ */ new Set([
1612
+ "replica_recovered",
1613
+ "restore_retried",
1614
+ "fresh_session_db",
1615
+ "history_rolled_back",
1616
+ "restore_misconfigured",
1617
+ "session_db_boot_refused"
1618
+ ]);
1619
+ var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
1620
+ var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
1621
+ var NUMBER_FIELDS = [
1622
+ "litestream_exit_code",
1623
+ "attempt",
1624
+ "replica_objects",
1625
+ "replica_bytes",
1626
+ "quarantined_objects",
1627
+ "quarantine_failed_objects",
1628
+ "quarantined_bytes",
1629
+ "restore_points_tried"
1630
+ ];
1631
+ var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"];
1632
+ function isSessionDbRecoveryRecord(value) {
1633
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
1634
+ const record = value;
1635
+ return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
1636
+ (field) => record[field] === null || typeof record[field] === "string"
1637
+ );
1638
+ }
1639
+
1355
1640
  // src/lib/opencode/health.ts
1356
1641
  async function checkOpenCodeHealth(port) {
1357
1642
  try {
@@ -1396,6 +1681,62 @@ function buildOpenCodeVersionWarning(version2) {
1396
1681
 
1397
1682
  // src/lib/opencode/process.ts
1398
1683
  import { execSync, spawn } from "child_process";
1684
+
1685
+ // src/lib/process-stop.ts
1686
+ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
1687
+ if (!child.pid) {
1688
+ return { outcome: "not-running", code: child.exitCode, signal: child.signalCode };
1689
+ }
1690
+ if (child.exitCode !== null || child.signalCode !== null) {
1691
+ return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
1692
+ }
1693
+ return new Promise((resolve3, reject) => {
1694
+ let forced = false;
1695
+ let settled = false;
1696
+ const timer = setTimeout(() => {
1697
+ forced = true;
1698
+ try {
1699
+ sendKill();
1700
+ } catch (error2) {
1701
+ if (error2.code === "ESRCH") {
1702
+ finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
1703
+ } else {
1704
+ fail(error2);
1705
+ }
1706
+ }
1707
+ }, timeoutMs);
1708
+ const finish = (result) => {
1709
+ if (settled) return;
1710
+ settled = true;
1711
+ clearTimeout(timer);
1712
+ child.removeListener("exit", onExit);
1713
+ resolve3(result);
1714
+ };
1715
+ const fail = (error2) => {
1716
+ if (settled) return;
1717
+ settled = true;
1718
+ clearTimeout(timer);
1719
+ child.removeListener("exit", onExit);
1720
+ reject(error2);
1721
+ };
1722
+ const onExit = (code, signal) => {
1723
+ finish({ outcome: forced ? "killed" : "exited", code, signal });
1724
+ };
1725
+ child.once("exit", onExit);
1726
+ try {
1727
+ sendTerm();
1728
+ } catch (error2) {
1729
+ if (error2.code === "ESRCH") {
1730
+ finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
1731
+ } else {
1732
+ fail(error2);
1733
+ }
1734
+ return;
1735
+ }
1736
+ });
1737
+ }
1738
+
1739
+ // src/lib/opencode/process.ts
1399
1740
  var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
1400
1741
  function getProcessCwd(pid) {
1401
1742
  const platform = process.platform;
@@ -1567,23 +1908,20 @@ async function startOpenCode(port) {
1567
1908
  });
1568
1909
  return child;
1569
1910
  }
1570
- function stopOpenCode(opencodeProcess) {
1571
- if (!opencodeProcess || !opencodeProcess.pid) {
1572
- return;
1573
- }
1574
- try {
1911
+ function stopOpenCodeAndWait(opencodeProcess, timeoutMs) {
1912
+ const sendSignal = (signal) => {
1575
1913
  if (process.platform === "win32") {
1576
- opencodeProcess.kill("SIGTERM");
1914
+ opencodeProcess.kill(signal);
1577
1915
  } else {
1578
- process.kill(-opencodeProcess.pid, "SIGTERM");
1916
+ process.kill(-opencodeProcess.pid, signal);
1579
1917
  }
1580
- } catch (err) {
1581
- if (err.code !== "ESRCH") {
1582
- console.warn(
1583
- `stopOpenCode: kill failed: ${err instanceof Error ? err.message : String(err)}`
1584
- );
1585
- }
1586
- }
1918
+ };
1919
+ return stopProcessAndWait(
1920
+ opencodeProcess,
1921
+ timeoutMs,
1922
+ () => sendSignal("SIGTERM"),
1923
+ () => sendSignal("SIGKILL")
1924
+ );
1587
1925
  }
1588
1926
 
1589
1927
  // src/lib/opencode/install.ts
@@ -1700,13 +2038,22 @@ function buildNoProviderWarning(hasProvider) {
1700
2038
  return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
1701
2039
  }
1702
2040
 
2041
+ // src/lib/http-timeout.ts
2042
+ var REQUEST_TIMEOUT_MS = 6e4;
2043
+ function withRequestTimeout(fetchImpl, timeoutMs) {
2044
+ return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
2045
+ }
2046
+
1703
2047
  // src/lib/opencode/session.ts
2048
+ function timedFetch(input, init) {
2049
+ return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
2050
+ }
1704
2051
  function opencodeBase(port) {
1705
2052
  return `http://127.0.0.1:${port}`;
1706
2053
  }
1707
2054
  async function getOpenCodeDirectory(port) {
1708
2055
  try {
1709
- const res = await fetch(`${opencodeBase(port)}/path`);
2056
+ const res = await timedFetch(`${opencodeBase(port)}/path`);
1710
2057
  if (!res.ok) return null;
1711
2058
  const body = await res.json();
1712
2059
  const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
@@ -1757,7 +2104,7 @@ function isAssistantInFlight(m) {
1757
2104
  }
1758
2105
  async function getSessionMessages(port, sessionId) {
1759
2106
  try {
1760
- const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
2107
+ const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
1761
2108
  if (!res.ok) return null;
1762
2109
  const body = await res.json();
1763
2110
  return Array.isArray(body) ? body : null;
@@ -1787,7 +2134,7 @@ function sessionLastActivityMs(session) {
1787
2134
  }
1788
2135
  async function listSessions(port) {
1789
2136
  try {
1790
- const res = await fetch(`${opencodeBase(port)}/session`);
2137
+ const res = await timedFetch(`${opencodeBase(port)}/session`);
1791
2138
  if (!res.ok) return null;
1792
2139
  const body = await res.json();
1793
2140
  return Array.isArray(body) ? body : null;
@@ -1797,7 +2144,7 @@ async function listSessions(port) {
1797
2144
  }
1798
2145
  async function deleteSession(port, id) {
1799
2146
  try {
1800
- const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
2147
+ const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
1801
2148
  return res.status >= 200 && res.status < 300;
1802
2149
  } catch {
1803
2150
  return false;
@@ -1805,7 +2152,7 @@ async function deleteSession(port, id) {
1805
2152
  }
1806
2153
  async function sessionExists(port, id) {
1807
2154
  try {
1808
- const res = await fetch(`${opencodeBase(port)}/session/${id}`);
2155
+ const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
1809
2156
  if (res.status >= 200 && res.status < 300) return true;
1810
2157
  if (res.status === 404) return false;
1811
2158
  return null;
@@ -1815,7 +2162,7 @@ async function sessionExists(port, id) {
1815
2162
  }
1816
2163
  async function getSessionStatuses(port) {
1817
2164
  try {
1818
- const res = await fetch(`${opencodeBase(port)}/session/status`);
2165
+ const res = await timedFetch(`${opencodeBase(port)}/session/status`);
1819
2166
  if (!res.ok) {
1820
2167
  console.error(
1821
2168
  `[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
@@ -1848,7 +2195,7 @@ async function createOpenCodeSession(port, directory) {
1848
2195
  if (directory && directory.trim()) {
1849
2196
  url.searchParams.set("directory", directory.trim());
1850
2197
  }
1851
- const response = await fetch(url, {
2198
+ const response = await timedFetch(url, {
1852
2199
  method: "POST",
1853
2200
  headers: { "Content-Type": "application/json" },
1854
2201
  body: JSON.stringify({})
@@ -1861,8 +2208,9 @@ async function createOpenCodeSession(port, directory) {
1861
2208
  return data.id;
1862
2209
  }
1863
2210
  async function getModelAttachmentCapability(port, model) {
2211
+ const { model: baseModel } = splitModelVariant(model);
1864
2212
  try {
1865
- const res = await fetch(`${opencodeBase(port)}/config/providers`);
2213
+ const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
1866
2214
  if (!res.ok) {
1867
2215
  console.error(
1868
2216
  `[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
@@ -1877,9 +2225,9 @@ async function getModelAttachmentCapability(port, model) {
1877
2225
  );
1878
2226
  return null;
1879
2227
  }
1880
- const slash = model ? model.indexOf("/") : -1;
1881
- const providerId = slash > 0 ? model.slice(0, slash) : void 0;
1882
- let modelId = slash > 0 ? model.slice(slash + 1) : void 0;
2228
+ const slash = baseModel ? baseModel.indexOf("/") : -1;
2229
+ const providerId = slash > 0 ? baseModel.slice(0, slash) : void 0;
2230
+ let modelId = slash > 0 ? baseModel.slice(slash + 1) : void 0;
1883
2231
  const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
1884
2232
  let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
1885
2233
  if (!provider && !providerId) {
@@ -1959,6 +2307,29 @@ async function buildFileParts(attachments, capable) {
1959
2307
  }
1960
2308
  return { parts, outcomes, capabilityUnknown };
1961
2309
  }
2310
+ function splitModelVariant(raw) {
2311
+ const value = raw?.trim();
2312
+ if (!value) return {};
2313
+ const hashIndex = value.indexOf("#");
2314
+ if (hashIndex === -1) return { model: value };
2315
+ const model = value.slice(0, hashIndex).trim() || void 0;
2316
+ const variant = value.slice(hashIndex + 1).trim() || void 0;
2317
+ return { model, variant };
2318
+ }
2319
+ function applyModelOptions(body, options) {
2320
+ if (options?.agent) body.agent = options.agent;
2321
+ const { model, variant } = splitModelVariant(options?.model);
2322
+ if (model) {
2323
+ const slashIndex = model.indexOf("/");
2324
+ if (slashIndex !== -1) {
2325
+ body.model = {
2326
+ providerID: model.substring(0, slashIndex),
2327
+ modelID: model.substring(slashIndex + 1)
2328
+ };
2329
+ }
2330
+ }
2331
+ if (variant) body.variant = variant;
2332
+ }
1962
2333
  function messageText(m) {
1963
2334
  if (!m || !Array.isArray(m.parts)) return "";
1964
2335
  return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
@@ -1983,26 +2354,18 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
1983
2354
  const body = {
1984
2355
  parts
1985
2356
  };
1986
- if (options?.agent) {
1987
- body.agent = options.agent;
1988
- }
1989
- if (options?.model) {
1990
- const slashIndex = options.model.indexOf("/");
1991
- if (slashIndex !== -1) {
1992
- body.model = {
1993
- providerID: options.model.substring(0, slashIndex),
1994
- modelID: options.model.substring(slashIndex + 1)
1995
- };
1996
- }
1997
- }
1998
- const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
2357
+ applyModelOptions(body, options);
2358
+ const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
1999
2359
  method: "POST",
2000
2360
  headers: { "Content-Type": "application/json" },
2001
2361
  body: JSON.stringify(body)
2002
2362
  });
2003
2363
  if (res.status < 200 || res.status >= 300) {
2004
2364
  const text = await res.text().catch(() => "");
2005
- throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
2365
+ const { variant } = splitModelVariant(options?.model);
2366
+ throw new Error(
2367
+ `OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}${variant ? ` (variant: ${variant})` : ""}`
2368
+ );
2006
2369
  }
2007
2370
  const READ_BACK_ATTEMPTS = 5;
2008
2371
  const READ_BACK_DELAY_MS = 150;
@@ -2157,7 +2520,7 @@ function isPreamblePinnedRunning(messages, userMessageId) {
2157
2520
  return completedOf(reply) != null && finishOf(reply) === "tool-calls";
2158
2521
  }
2159
2522
  function isB2AbandonmentConfirmed(params) {
2160
- return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
2523
+ return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false && params.rootOngoing === false;
2161
2524
  }
2162
2525
  function isAmbiguousTerminalFinish(m) {
2163
2526
  if (completedOf(m) == null) return false;
@@ -2170,7 +2533,7 @@ function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
2170
2533
  return isAmbiguousTerminalFinish(reply);
2171
2534
  }
2172
2535
  function isAmbiguousFinishResolved(params) {
2173
- return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
2536
+ return params.sessionOngoing === false || params.sessionOngoing !== true && params.pinnedForMs >= params.maxPinnedMs;
2174
2537
  }
2175
2538
  function messageError(messages, userMessageId) {
2176
2539
  const reply = findLastAssistantReplyFor(messages, userMessageId);
@@ -2244,7 +2607,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
2244
2607
  }
2245
2608
  async function hasAnyConfiguredProvider(port) {
2246
2609
  try {
2247
- const res = await fetch(`${opencodeBase(port)}/config/providers`);
2610
+ const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
2248
2611
  if (!res.ok) {
2249
2612
  console.error(
2250
2613
  `[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
@@ -2380,10 +2743,10 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
2380
2743
 
2381
2744
  // src/lib/opencode/session-db-size.ts
2382
2745
  import { statSync as statSync2 } from "fs";
2383
- import { join as join2 } from "path";
2746
+ import { join as join3 } from "path";
2384
2747
  var LARGE_DB_THRESHOLD_BYTES = 268435456;
2385
2748
  function statSessionDbBytes(homeDir) {
2386
- const dbPath = join2(homeDir, ".local", "share", "opencode", "opencode.db");
2749
+ const dbPath = join3(homeDir, ".local", "share", "opencode", "opencode.db");
2387
2750
  try {
2388
2751
  return statSync2(dbPath).size;
2389
2752
  } catch (err) {
@@ -2952,44 +3315,421 @@ function writeTunnelReadyMarker(path, agentId) {
2952
3315
  }
2953
3316
  }
2954
3317
 
3318
+ // src/lib/replication.ts
3319
+ import { spawn as spawn2 } from "child_process";
3320
+ function startSessionDbReplication(configPath) {
3321
+ return spawn2("litestream", ["replicate", "-config", configPath], {
3322
+ stdio: "inherit"
3323
+ });
3324
+ }
3325
+ async function stopSessionDbReplication(child, timeoutMs) {
3326
+ return stopProcessAndWait(
3327
+ child,
3328
+ timeoutMs,
3329
+ () => child.kill("SIGTERM"),
3330
+ () => child.kill("SIGKILL")
3331
+ );
3332
+ }
3333
+
3334
+ // src/lib/openai-usage.ts
3335
+ import { readFileSync as readFileSync3 } from "fs";
3336
+ import { homedir as homedir2 } from "os";
3337
+ import { join as join4 } from "path";
3338
+ var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
3339
+ var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
3340
+ var OpenAiUsageError = class extends Error {
3341
+ constructor(message, reason) {
3342
+ super(message);
3343
+ this.reason = reason;
3344
+ }
3345
+ };
3346
+ function isLocalCredentialProblem2(err) {
3347
+ return err instanceof OpenAiUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
3348
+ }
3349
+ function readOpenCodeChatGptCredentials() {
3350
+ try {
3351
+ const raw = readFileSync3(join4(homedir2(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
3352
+ let parsed;
3353
+ try {
3354
+ parsed = JSON.parse(raw);
3355
+ } catch {
3356
+ return null;
3357
+ }
3358
+ const entry = parsed.openai;
3359
+ if (entry?.type !== "oauth" || typeof entry.access !== "string" || !entry.access || typeof entry.expires !== "number") {
3360
+ return null;
3361
+ }
3362
+ return { accessToken: entry.access, expiresAt: entry.expires };
3363
+ } catch (err) {
3364
+ const code = err.code;
3365
+ if (code !== "ENOENT" && code !== "ENOTDIR") {
3366
+ console.warn(
3367
+ `readOpenCodeChatGptCredentials: reading auth.json failed (${code ?? "unknown"})`
3368
+ );
3369
+ }
3370
+ return null;
3371
+ }
3372
+ }
3373
+ function toWindow2(headers, name) {
3374
+ const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
3375
+ const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
3376
+ if (utilizationHeader == null || utilizationHeader === "" || windowMinutesHeader == null) {
3377
+ return null;
3378
+ }
3379
+ const utilization = Number(utilizationHeader);
3380
+ const windowMinutes = Number(windowMinutesHeader);
3381
+ if (!Number.isFinite(utilization) || !Number.isFinite(windowMinutes) || windowMinutes <= 0) {
3382
+ return null;
3383
+ }
3384
+ const resetAtHeader = headers.get(`x-codex-${name}-reset-at`);
3385
+ const resetSeconds = resetAtHeader == null || resetAtHeader === "" ? NaN : Number(resetAtHeader);
3386
+ const resetsAt = Number.isFinite(resetSeconds) ? new Date(resetSeconds * 1e3).toISOString() : null;
3387
+ return { utilization: Math.min(100, Math.max(0, utilization)), windowMinutes, resetsAt };
3388
+ }
3389
+ function parseCodexUsageHeaders(headers) {
3390
+ return {
3391
+ primary: toWindow2(headers, "primary"),
3392
+ secondary: toWindow2(headers, "secondary"),
3393
+ hasCredits: headers.has("x-codex-credits-has-credits") ? headers.get("x-codex-credits-has-credits")?.toLowerCase() === "true" : null,
3394
+ creditsUnlimited: headers.has("x-codex-credits-unlimited") ? headers.get("x-codex-credits-unlimited")?.toLowerCase() === "true" : null
3395
+ };
3396
+ }
3397
+ function normalizeProbeModel(model) {
3398
+ return model.endsWith("-fast") ? model.slice(0, -"-fast".length) : model;
3399
+ }
3400
+ async function resolveProbeModels(port) {
3401
+ try {
3402
+ const res = await withRequestTimeout(
3403
+ fetch,
3404
+ REQUEST_TIMEOUT_MS
3405
+ )(`${opencodeBase(port)}/config/providers`);
3406
+ if (!res.ok) {
3407
+ console.error(
3408
+ `[resolveProbeModels] GET /config/providers returned HTTP ${res.status} (port ${port})`
3409
+ );
3410
+ return [];
3411
+ }
3412
+ const body = await res.json();
3413
+ const provider = body?.providers?.find((candidate) => candidate?.id === "openai");
3414
+ if (!provider || !provider.models || typeof provider.models !== "object") return [];
3415
+ const candidates = [
3416
+ ...typeof body?.default?.openai === "string" ? [body.default.openai] : [],
3417
+ ...Object.keys(provider.models)
3418
+ ].map(normalizeProbeModel);
3419
+ return [...new Set(candidates)].slice(0, 4);
3420
+ } catch (err) {
3421
+ console.error(
3422
+ `[resolveProbeModels] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
3423
+ );
3424
+ return [];
3425
+ }
3426
+ }
3427
+ function hasPrimaryHeaders(headers) {
3428
+ return [
3429
+ "x-codex-primary-used-percent",
3430
+ "x-codex-primary-window-minutes",
3431
+ "x-codex-primary-reset-at"
3432
+ ].some((name) => headers.has(name));
3433
+ }
3434
+ async function getOpenAiUsage(port) {
3435
+ const credentials2 = readOpenCodeChatGptCredentials();
3436
+ if (!credentials2) {
3437
+ throw new OpenAiUsageError(
3438
+ "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
3439
+ "no_credentials"
3440
+ );
3441
+ }
3442
+ if (credentials2.expiresAt < Date.now()) {
3443
+ throw new OpenAiUsageError(
3444
+ "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
3445
+ "credentials_expired"
3446
+ );
3447
+ }
3448
+ const models = await resolveProbeModels(port);
3449
+ if (models.length === 0) {
3450
+ throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
3451
+ }
3452
+ let lastStatus;
3453
+ for (const model of models) {
3454
+ let res;
3455
+ try {
3456
+ res = await withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(CODEX_RESPONSES_URL, {
3457
+ method: "POST",
3458
+ headers: {
3459
+ Authorization: `Bearer ${credentials2.accessToken}`,
3460
+ "Content-Type": "application/json"
3461
+ },
3462
+ body: JSON.stringify({ model, store: false, stream: true })
3463
+ });
3464
+ } catch (err) {
3465
+ throw new OpenAiUsageError(
3466
+ `OpenAI usage probe request failed: ${err instanceof Error ? err.message : String(err)}.`,
3467
+ "request_failed"
3468
+ );
3469
+ }
3470
+ try {
3471
+ lastStatus = res.status;
3472
+ if (hasPrimaryHeaders(res.headers)) {
3473
+ const usage = parseCodexUsageHeaders(res.headers);
3474
+ if (!usage.primary && !usage.secondary) {
3475
+ throw new OpenAiUsageError(
3476
+ `OpenAI usage probe returned no usable window (HTTP ${res.status}).`,
3477
+ "no_usable_window"
3478
+ );
3479
+ }
3480
+ return usage;
3481
+ }
3482
+ if (res.status === 401) {
3483
+ throw new OpenAiUsageError(
3484
+ "ChatGPT credentials have expired (HTTP 401).",
3485
+ "credentials_expired"
3486
+ );
3487
+ }
3488
+ if (res.status === 403 || res.status === 429) {
3489
+ throw new OpenAiUsageError(
3490
+ `OpenAI usage probe was blocked (HTTP ${res.status}).`,
3491
+ "probe_blocked"
3492
+ );
3493
+ }
3494
+ } finally {
3495
+ await res.body?.cancel().catch(() => {
3496
+ });
3497
+ }
3498
+ }
3499
+ throw new OpenAiUsageError(
3500
+ `OpenAI usage probe failed: HTTP ${lastStatus ?? "unknown"}.`,
3501
+ "request_failed"
3502
+ );
3503
+ }
3504
+
3505
+ // src/lib/reporting-schedule.ts
3506
+ function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
3507
+ const jitterRangeMs = baseMs * jitterFraction;
3508
+ return baseMs - jitterRangeMs + random() * (2 * jitterRangeMs);
3509
+ }
3510
+ function firstReportDelayMs(random = Math.random) {
3511
+ return 5e3 + random() * 1e4;
3512
+ }
3513
+ var VALID_USAGE_REPORTING_MODES = ["auto", "on", "off"];
3514
+ function resolveUsageReportingMode(flagValue, env, names) {
3515
+ const raw = flagValue ?? env[names.envVar];
3516
+ if (raw === void 0 || raw === "") return { mode: "auto", warnings: [] };
3517
+ const normalized = raw.trim().toLowerCase();
3518
+ if (VALID_USAGE_REPORTING_MODES.includes(normalized)) {
3519
+ return { mode: normalized, warnings: [] };
3520
+ }
3521
+ const source = flagValue !== void 0 ? names.flagName : names.envVar;
3522
+ return {
3523
+ mode: "auto",
3524
+ warnings: [
3525
+ `Ignoring invalid ${source} "${raw}": expected one of ${VALID_USAGE_REPORTING_MODES.join(", ")}; using auto`
3526
+ ]
3527
+ };
3528
+ }
3529
+ var BASE_USAGE_REPORT_DELAY_MS = 10 * 6e4;
3530
+ var USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
3531
+ function usageReportDelayMs(random = Math.random) {
3532
+ return jitteredDelayMs(BASE_USAGE_REPORT_DELAY_MS, USAGE_REPORT_DELAY_JITTER_FRACTION, random);
3533
+ }
3534
+ var USAGE_REPORT_FAILURE_REESCALATION_TICKS = 6;
3535
+ function usageReportFailureLogLevel(consecutiveFailures) {
3536
+ return reportFailureLogLevel(consecutiveFailures, USAGE_REPORT_FAILURE_REESCALATION_TICKS);
3537
+ }
3538
+ function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
3539
+ return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
3540
+ }
3541
+ function failureStreakSuffix(consecutiveFailures) {
3542
+ return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
3543
+ }
3544
+
2955
3545
  // src/lib/claude-usage-reporting.ts
2956
- var VALID_MODES = ["auto", "on", "off"];
2957
3546
  function resolveClaudeUsageReportingMode(flagValue, env) {
2958
- const raw = flagValue ?? env.EVIDENT_CLAUDE_USAGE_REPORTING;
3547
+ return resolveUsageReportingMode(flagValue, env, {
3548
+ flagName: "--claude-usage-reporting",
3549
+ envVar: "EVIDENT_CLAUDE_USAGE_REPORTING"
3550
+ });
3551
+ }
3552
+ function nextReportDelayMs(random = Math.random) {
3553
+ return usageReportDelayMs(random);
3554
+ }
3555
+ var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
3556
+ function claudeUsageFailureLogLevel(consecutiveFailures) {
3557
+ return usageReportFailureLogLevel(consecutiveFailures);
3558
+ }
3559
+
3560
+ // src/lib/openai-usage-reporting.ts
3561
+ function resolveOpenAiUsageReportingMode(flagValue, env) {
3562
+ return resolveUsageReportingMode(flagValue, env, {
3563
+ flagName: "--openai-usage-reporting",
3564
+ envVar: "EVIDENT_OPENAI_USAGE_REPORTING"
3565
+ });
3566
+ }
3567
+
3568
+ // src/lib/resource-usage-reporting.ts
3569
+ var ENABLED_VALUES = /* @__PURE__ */ new Set(["on", "true", "1"]);
3570
+ var DISABLED_VALUES = /* @__PURE__ */ new Set(["off", "false", "0"]);
3571
+ function resolveResourceUsageReportingEnabled(flagValue, env) {
3572
+ if (flagValue === false) {
3573
+ return { enabled: false, warnings: [] };
3574
+ }
3575
+ const raw = env.EVIDENT_RESOURCE_USAGE_REPORTING;
2959
3576
  if (raw === void 0 || raw === "") {
2960
- return { mode: "auto", warnings: [] };
3577
+ return { enabled: true, warnings: [] };
2961
3578
  }
2962
3579
  const normalized = raw.trim().toLowerCase();
2963
- if (VALID_MODES.includes(normalized)) {
2964
- return { mode: normalized, warnings: [] };
3580
+ if (DISABLED_VALUES.has(normalized)) {
3581
+ return { enabled: false, warnings: [] };
3582
+ }
3583
+ if (ENABLED_VALUES.has(normalized)) {
3584
+ return { enabled: true, warnings: [] };
3585
+ }
3586
+ return {
3587
+ enabled: true,
3588
+ warnings: [
3589
+ `Ignoring invalid EVIDENT_RESOURCE_USAGE_REPORTING "${raw}": expected on or off; leaving reporting on`
3590
+ ]
3591
+ };
3592
+ }
3593
+
3594
+ // src/lib/resource-usage.ts
3595
+ import { cpus, totalmem, freemem } from "os";
3596
+ import { statfsSync as statfsSync2 } from "fs";
3597
+
3598
+ // src/lib/ecs-task-metadata.ts
3599
+ var ECS_METADATA_TIMEOUT_MS = 2e3;
3600
+ function parseEcsTaskLimits(payload) {
3601
+ if (typeof payload !== "object" || payload === null) return null;
3602
+ const limits = payload.Limits;
3603
+ if (typeof limits !== "object" || limits === null) return null;
3604
+ const cpu = limits.CPU;
3605
+ const memory = limits.Memory;
3606
+ if (typeof cpu !== "number" || !Number.isFinite(cpu) || cpu <= 0) return null;
3607
+ if (typeof memory !== "number" || !Number.isFinite(memory) || memory <= 0) return null;
3608
+ return {
3609
+ cpuCount: Math.max(1, Math.round(cpu)),
3610
+ memoryTotalBytes: memory * 1024 * 1024
3611
+ };
3612
+ }
3613
+ async function readEcsTaskLimits(env) {
3614
+ const uri = env.ECS_CONTAINER_METADATA_URI_V4;
3615
+ if (!uri) {
3616
+ return { limits: null };
3617
+ }
3618
+ const url = `${uri}/task`;
3619
+ try {
3620
+ const response = await fetch(url, { signal: AbortSignal.timeout(ECS_METADATA_TIMEOUT_MS) });
3621
+ if (!response.ok) {
3622
+ return {
3623
+ limits: null,
3624
+ warning: `ECS task metadata fetch (${url}) returned HTTP ${response.status}`
3625
+ };
3626
+ }
3627
+ const payload = await response.json();
3628
+ const limits = parseEcsTaskLimits(payload);
3629
+ if (limits === null) {
3630
+ return {
3631
+ limits: null,
3632
+ warning: `ECS task metadata fetch (${url}) returned an unexpected payload`
3633
+ };
3634
+ }
3635
+ return { limits };
3636
+ } catch (error2) {
3637
+ const message = error2 instanceof Error ? error2.message : String(error2);
3638
+ return { limits: null, warning: `ECS task metadata fetch (${url}) failed: ${message}` };
3639
+ }
3640
+ }
3641
+
3642
+ // src/lib/resource-usage.ts
3643
+ function readCpuSample() {
3644
+ let busyMs = 0;
3645
+ let idleMs = 0;
3646
+ for (const cpu of cpus()) {
3647
+ busyMs += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.irq;
3648
+ idleMs += cpu.times.idle;
2965
3649
  }
2966
- const source = flagValue !== void 0 ? "--claude-usage-reporting" : "EVIDENT_CLAUDE_USAGE_REPORTING";
2967
- return {
2968
- mode: "auto",
2969
- warnings: [
2970
- `Ignoring invalid ${source} "${raw}": expected one of ${VALID_MODES.join(", ")}; using auto`
2971
- ]
2972
- };
3650
+ return { busyMs, idleMs };
2973
3651
  }
2974
- var BASE_REPORT_DELAY_MS = 10 * 6e4;
2975
- var REPORT_DELAY_JITTER_FRACTION = 0.2;
2976
- function nextReportDelayMs(random = Math.random) {
2977
- const jitterRangeMs = BASE_REPORT_DELAY_MS * REPORT_DELAY_JITTER_FRACTION;
2978
- return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
3652
+ function cpuPercentBetween(previous, current) {
3653
+ const deltaBusy = current.busyMs - previous.busyMs;
3654
+ const deltaIdle = current.idleMs - previous.idleMs;
3655
+ const total = deltaBusy + deltaIdle;
3656
+ if (total === 0) return null;
3657
+ return Math.round((deltaBusy / total * 100 + Number.EPSILON) * 100) / 100;
2979
3658
  }
2980
- var FIRST_REPORT_DELAY_MS = 5e3 + Math.random() * 1e4;
2981
- var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
2982
- function claudeUsageFailureLogLevel(consecutiveFailures) {
2983
- return consecutiveFailures === 1 || consecutiveFailures % CLAUDE_USAGE_FAILURE_REESCALATION_TICKS === 0 ? "warn" : "debug";
3659
+ function clamp(value, min, max) {
3660
+ return Math.min(Math.max(value, min), max);
3661
+ }
3662
+ function round2(value) {
3663
+ return Math.round((value + Number.EPSILON) * 100) / 100;
3664
+ }
3665
+ function readDisk(homeDir) {
3666
+ try {
3667
+ const stats = statfsSync2(homeDir);
3668
+ return {
3669
+ totalBytes: stats.bsize * stats.blocks,
3670
+ freeBytes: stats.bsize * stats.bavail
3671
+ };
3672
+ } catch (error2) {
3673
+ const message = error2 instanceof Error ? error2.message : String(error2);
3674
+ return {
3675
+ totalBytes: null,
3676
+ freeBytes: null,
3677
+ warning: `Could not read disk usage for ${homeDir}: ${message}`
3678
+ };
3679
+ }
3680
+ }
3681
+ function createResourceUsageCollector(homeDir) {
3682
+ let previous = readCpuSample();
3683
+ return async () => {
3684
+ const current = readCpuSample();
3685
+ const hostCpuPercent = cpuPercentBetween(previous, current);
3686
+ const hostCpuCount = cpus().length;
3687
+ previous = current;
3688
+ const disk = readDisk(homeDir);
3689
+ const opencodeDbBytes = statSessionDbBytes(homeDir);
3690
+ const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
3691
+ const warnings = [];
3692
+ if (disk.warning) warnings.push(disk.warning);
3693
+ if (ecsWarning) warnings.push(ecsWarning);
3694
+ let cpuPercent = hostCpuPercent;
3695
+ let cpuCount = hostCpuCount;
3696
+ let memoryTotalBytes = totalmem();
3697
+ let memoryAvailableBytes = freemem();
3698
+ if (limits !== null) {
3699
+ cpuCount = limits.cpuCount;
3700
+ memoryTotalBytes = limits.memoryTotalBytes;
3701
+ memoryAvailableBytes = clamp(
3702
+ limits.memoryTotalBytes - (totalmem() - freemem()),
3703
+ 0,
3704
+ limits.memoryTotalBytes
3705
+ );
3706
+ cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
3707
+ }
3708
+ return {
3709
+ usage: {
3710
+ cpuPercent,
3711
+ cpuCount,
3712
+ memoryTotalBytes,
3713
+ memoryAvailableBytes,
3714
+ diskTotalBytes: disk.totalBytes,
3715
+ diskFreeBytes: disk.freeBytes,
3716
+ opencodeDbBytes
3717
+ },
3718
+ warnings
3719
+ };
3720
+ };
2984
3721
  }
2985
3722
 
2986
3723
  // src/lib/channels/driver.ts
2987
- import { homedir as homedir2 } from "os";
3724
+ import { homedir as homedir3 } from "os";
3725
+
3726
+ // src/lib/runner-file-sync.ts
3727
+ import { join as join6 } from "path";
2988
3728
 
2989
3729
  // src/lib/file-push.ts
2990
3730
  import { randomUUID } from "crypto";
2991
3731
  import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
2992
- import { basename, dirname as dirname3, isAbsolute, join as join3, relative, resolve as resolve2, sep } from "path";
3732
+ import { basename, dirname as dirname3, isAbsolute, join as join5, relative, resolve as resolve2, sep } from "path";
2993
3733
  var FILE_MODE = 384;
2994
3734
  var DIRECTORY_MODE = 448;
2995
3735
  async function writePushedFile(request) {
@@ -3022,7 +3762,7 @@ async function writePushedFile(request) {
3022
3762
  const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
3023
3763
  dirname3(candidate)
3024
3764
  );
3025
- const realTarget = join3(existingAncestor, ...missingSegments, basename(candidate));
3765
+ const realTarget = join5(existingAncestor, ...missingSegments, basename(candidate));
3026
3766
  const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
3027
3767
  if (allowedDirectory === null) {
3028
3768
  return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
@@ -3058,7 +3798,7 @@ function expandAndValidate(requestedPath, homeDir) {
3058
3798
  if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
3059
3799
  return null;
3060
3800
  }
3061
- const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join3(homeDir, requestedPath.slice(2)) : requestedPath;
3801
+ const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join5(homeDir, requestedPath.slice(2)) : requestedPath;
3062
3802
  if (expanded.split(/[/\\]/).includes("..")) {
3063
3803
  return null;
3064
3804
  }
@@ -3131,13 +3871,13 @@ function contains(realDirectory, realTarget) {
3131
3871
  async function createMissingDirectories(existingAncestor, missingSegments) {
3132
3872
  let current = existingAncestor;
3133
3873
  for (const segment of missingSegments) {
3134
- current = join3(current, segment);
3874
+ current = join5(current, segment);
3135
3875
  await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
3136
3876
  await chmod(current, DIRECTORY_MODE);
3137
3877
  }
3138
3878
  }
3139
3879
  async function writeAtomically(realTarget, content) {
3140
- const temporaryPath = join3(dirname3(realTarget), `.evident-push-${randomUUID()}.tmp`);
3880
+ const temporaryPath = join5(dirname3(realTarget), `.evident-push-${randomUUID()}.tmp`);
3141
3881
  let handle;
3142
3882
  try {
3143
3883
  handle = await open2(temporaryPath, "wx", FILE_MODE);
@@ -3179,17 +3919,28 @@ async function syncPendingRunnerFiles(options) {
3179
3919
  for (const id of options.ackFailures.keys()) {
3180
3920
  if (!pendingIds.has(id)) options.ackFailures.delete(id);
3181
3921
  }
3182
- if (pending.length === 0) return 0;
3922
+ if (pending.length === 0) {
3923
+ return { applied: 0, claudeCredentialApplied: false, opencodeAuthApplied: false };
3924
+ }
3183
3925
  options.log({
3184
3926
  level: "info",
3185
3927
  message: `Runner file sync: ${pending.length} file(s) queued for this runner`
3186
3928
  });
3187
3929
  let applied = 0;
3930
+ let claudeCredentialApplied = false;
3931
+ let opencodeAuthApplied = false;
3188
3932
  for (const file of pending) {
3189
3933
  if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
3190
- if (await applyOne(options, file)) applied += 1;
3934
+ const outcome = await applyOne(options, file);
3935
+ if (outcome.applied) applied += 1;
3936
+ if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
3937
+ if (outcome.opencodeAuthApplied) opencodeAuthApplied = true;
3191
3938
  }
3192
- return applied;
3939
+ return {
3940
+ applied,
3941
+ claudeCredentialApplied,
3942
+ opencodeAuthApplied
3943
+ };
3193
3944
  }
3194
3945
  async function listPendingFiles(options) {
3195
3946
  let res;
@@ -3250,6 +4001,19 @@ function asPendingFile(entry) {
3250
4001
  if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
3251
4002
  return { id, path, size };
3252
4003
  }
4004
+ var NOT_APPLIED = {
4005
+ applied: false,
4006
+ claudeCredentialApplied: false,
4007
+ opencodeAuthApplied: false
4008
+ };
4009
+ function isClaudeCredentialPath(requestedPath, homeDir) {
4010
+ const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
4011
+ return expanded === join6(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
4012
+ }
4013
+ function isOpenCodeAuthPath(requestedPath, homeDir) {
4014
+ const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
4015
+ return expanded === join6(homeDir, ...OPENCODE_AUTH_SEGMENTS);
4016
+ }
3253
4017
  async function applyOne(options, file) {
3254
4018
  const label = `${file.id.slice(0, 8)} (${file.path})`;
3255
4019
  if (options.allowedDirectories.length === 0) {
@@ -3258,7 +4022,7 @@ async function applyOne(options, file) {
3258
4022
  message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
3259
4023
  });
3260
4024
  await ack(options, file, "rejected", "file_sync_disabled");
3261
- return false;
4025
+ return NOT_APPLIED;
3262
4026
  }
3263
4027
  if (file.size > MAX_FILE_PUSH_BYTES) {
3264
4028
  options.log({
@@ -3266,12 +4030,12 @@ async function applyOne(options, file) {
3266
4030
  message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
3267
4031
  });
3268
4032
  await ack(options, file, "rejected", "file_too_large");
3269
- return false;
4033
+ return NOT_APPLIED;
3270
4034
  }
3271
4035
  const download = await downloadContent(options, file, label);
3272
4036
  if (!download.ok) {
3273
4037
  if (download.terminal) await ack(options, file, "rejected", download.code);
3274
- return false;
4038
+ return NOT_APPLIED;
3275
4039
  }
3276
4040
  let outcome;
3277
4041
  try {
@@ -3287,7 +4051,7 @@ async function applyOne(options, file) {
3287
4051
  message: `Runner file ${label} could not be written: ${describe(err)}`
3288
4052
  });
3289
4053
  await ack(options, file, "rejected", "write_failed");
3290
- return false;
4054
+ return NOT_APPLIED;
3291
4055
  }
3292
4056
  if (!outcome.ok) {
3293
4057
  options.log({
@@ -3295,14 +4059,18 @@ async function applyOne(options, file) {
3295
4059
  message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
3296
4060
  });
3297
4061
  await ack(options, file, "rejected", outcome.code);
3298
- return false;
4062
+ return NOT_APPLIED;
3299
4063
  }
3300
4064
  options.log({
3301
4065
  level: "info",
3302
4066
  message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
3303
4067
  });
3304
4068
  await ack(options, file, "applied");
3305
- return true;
4069
+ return {
4070
+ applied: true,
4071
+ claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir),
4072
+ opencodeAuthApplied: isOpenCodeAuthPath(file.path, options.homeDir)
4073
+ };
3306
4074
  }
3307
4075
  function durableDownloadCode(status2) {
3308
4076
  return status2 === 413 ? "file_too_large" : "write_failed";
@@ -3413,8 +4181,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
3413
4181
  var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
3414
4182
  var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
3415
4183
  var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
4184
+ var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
4185
+ var MAX_WATCHER_STALL_RESTARTS = 3;
4186
+ var MAX_RELEASED_OPENCODE_IDS = 256;
3416
4187
  var MAX_SUPERSEDED_CONVERSATIONS = 256;
3417
4188
  var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
4189
+ var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
4190
+ var MAX_WEDGED_CONVERSATIONS = 256;
3418
4191
  var ChannelAuthError = class extends Error {
3419
4192
  constructor(message) {
3420
4193
  super(message);
@@ -3458,6 +4231,8 @@ var ChannelDriver = class _ChannelDriver {
3458
4231
  fileSyncDirectories;
3459
4232
  homeDir;
3460
4233
  maxActiveSessions;
4234
+ watcherStallMs;
4235
+ wedgeWarningIntervalMs;
3461
4236
  /** Cache of conversationId → opencode sessionId. */
3462
4237
  sessions = /* @__PURE__ */ new Map();
3463
4238
  /**
@@ -3488,6 +4263,40 @@ var ChannelDriver = class _ChannelDriver {
3488
4263
  * bounded cost.
3489
4264
  */
3490
4265
  supersededSessions = /* @__PURE__ */ new Map();
4266
+ /**
4267
+ * Local re-drive fence for a message force-released by the stall watchdog
4268
+ * (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
4269
+ * see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
4270
+ * is `null` for exactly this shape (its `markProcessing` never landed), so
4271
+ * without a local record of the id the driver last knew, the next drain's
4272
+ * `if (message.opencode_message_id)` re-drive-fence check at
4273
+ * `processConversation` would not engage and it would blind-`prompt_async`
4274
+ * a turn that may still be running in opencode — the one duplicate-turn
4275
+ * hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
4276
+ * `processConversation` reads `message.opencode_message_id ?? this
4277
+ * .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
4278
+ * threads it into `resolveRedrive`, which asks opencode itself whether the
4279
+ * turn is still ongoing before ever dispatching.
4280
+ *
4281
+ * Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
4282
+ * `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
4283
+ * non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
4284
+ * dispatch) and at the top-level fresh-dispatch site, so it does not outlive
4285
+ * the row it was recorded for.
4286
+ */
4287
+ releasedOpencodeIds = /* @__PURE__ */ new Map();
4288
+ /**
4289
+ * Per-conversation throttle state for the #183 recurrence warning (#1618
4290
+ * WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
4291
+ * `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
4292
+ * log line and the `dispatch_wedged` signal to at most once per
4293
+ * `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
4294
+ * so the operator sees magnitude, not repetition. Cleared the moment the
4295
+ * conversation dispatches anything (a fresh wedge, if it recurs, is a new
4296
+ * incident). Bounded FIFO, mirroring `supersededSessions`
4297
+ * (`MAX_WEDGED_CONVERSATIONS`).
4298
+ */
4299
+ wedgeWarnings = /* @__PURE__ */ new Map();
3491
4300
  /**
3492
4301
  * Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
3493
4302
  * longer idempotent (no caller-supplied `messageID`), and its read-back picks
@@ -3706,6 +4515,15 @@ var ChannelDriver = class _ChannelDriver {
3706
4515
  * same trick `lastProxiedActivityAt` uses.
3707
4516
  */
3708
4517
  appliedFileCount = 0;
4518
+ /**
4519
+ * Generation counter, NOT a tally (#1656): advances by exactly one per sync
4520
+ * batch that applied the Claude CLI credential file, not by how many
4521
+ * credential files were in that batch. `run.ts` only ever tests inequality
4522
+ * against the value it saw last cycle, so magnitude is meaningless — keep it
4523
+ * that way rather than "fixing" it into a count.
4524
+ */
4525
+ claudeCredentialApplyCount = 0;
4526
+ opencodeAuthApplyCount = 0;
3709
4527
  /**
3710
4528
  * The currently-executing `drainPending()` promise, or null when idle. Lets a
3711
4529
  * graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
@@ -3730,15 +4548,20 @@ var ChannelDriver = class _ChannelDriver {
3730
4548
  this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
3731
4549
  this.log = config.log ?? (() => {
3732
4550
  });
3733
- this.fetchImpl = config.fetchImpl ?? fetch;
4551
+ this.fetchImpl = withRequestTimeout(
4552
+ config.fetchImpl ?? fetch,
4553
+ config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
4554
+ );
3734
4555
  this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
3735
4556
  this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
3736
4557
  this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
3737
4558
  this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
3738
4559
  this.now = config.now ?? (() => Date.now());
3739
4560
  this.fileSyncDirectories = config.fileSyncDirectories ?? [];
3740
- this.homeDir = config.homeDir ?? homedir2();
4561
+ this.homeDir = config.homeDir ?? homedir3();
3741
4562
  this.maxActiveSessions = config.maxActiveSessions;
4563
+ this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
4564
+ this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
3742
4565
  }
3743
4566
  /** The IPv4-loopback base URL for the local `opencode serve`. */
3744
4567
  get opencodeBase() {
@@ -3752,6 +4575,14 @@ var ChannelDriver = class _ChannelDriver {
3752
4575
  * @returns the number of messages NEWLY dispatched to opencode's native queue.
3753
4576
  */
3754
4577
  async drainPending() {
4578
+ try {
4579
+ this.reconcileWatchers();
4580
+ } catch (err) {
4581
+ this.log({
4582
+ level: "error",
4583
+ message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
4584
+ });
4585
+ }
3755
4586
  if (this.stopped) return 0;
3756
4587
  if (this.draining) return 0;
3757
4588
  this.draining = true;
@@ -3785,7 +4616,7 @@ var ChannelDriver = class _ChannelDriver {
3785
4616
  if (this.syncingFiles) return 0;
3786
4617
  this.syncingFiles = true;
3787
4618
  try {
3788
- const applied = await syncPendingRunnerFiles({
4619
+ const result = await syncPendingRunnerFiles({
3789
4620
  agentId: this.agentId,
3790
4621
  apiUrl: this.apiUrl,
3791
4622
  getAuthHeader: this.getAuthHeader,
@@ -3795,8 +4626,10 @@ var ChannelDriver = class _ChannelDriver {
3795
4626
  ackFailures: this.fileAckFailures,
3796
4627
  log: this.log
3797
4628
  });
3798
- this.appliedFileCount += applied;
3799
- return applied;
4629
+ this.appliedFileCount += result.applied;
4630
+ if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
4631
+ if (result.opencodeAuthApplied) this.opencodeAuthApplyCount += 1;
4632
+ return result.applied;
3800
4633
  } catch (err) {
3801
4634
  this.log({
3802
4635
  level: "error",
@@ -3887,12 +4720,25 @@ var ChannelDriver = class _ChannelDriver {
3887
4720
  * `appliedFiles` is monotonic so a pull that started AND finished between two
3888
4721
  * idle checks still shows up as an advance.
3889
4722
  *
4723
+ * A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
4724
+ * (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
4725
+ * reporting on it advancing, so an unrelated file sync can never disturb a
4726
+ * healthy reporting cadence (#1627) — it never even reaches that trigger, let
4727
+ * alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
4728
+ * whose consumer is idle-timeout suppression and must key on ANY file, not
4729
+ * just a Claude credential.
4730
+ *
3890
4731
  * CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
3891
4732
  * the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
3892
4733
  * samples afterwards reads `true` every single cycle and can never idle out.
3893
4734
  */
3894
4735
  fileSyncActivity() {
3895
- return { appliedFiles: this.appliedFileCount, inFlight: this.syncingFiles };
4736
+ return {
4737
+ appliedFiles: this.appliedFileCount,
4738
+ inFlight: this.syncingFiles,
4739
+ claudeCredentialApplies: this.claudeCredentialApplyCount,
4740
+ opencodeAuthApplyCount: this.opencodeAuthApplyCount
4741
+ };
3896
4742
  }
3897
4743
  /**
3898
4744
  * OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
@@ -4005,8 +4851,15 @@ var ChannelDriver = class _ChannelDriver {
4005
4851
  skippedAlreadyDispatched += 1;
4006
4852
  continue;
4007
4853
  }
4008
- if (message.opencode_message_id) {
4009
- const outcome = await this.resolveRedrive(conv, sessionId, message, sessionCreated);
4854
+ const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
4855
+ if (effectiveOpencodeMessageId) {
4856
+ const outcome = await this.resolveRedrive(
4857
+ conv,
4858
+ sessionId,
4859
+ message,
4860
+ sessionCreated,
4861
+ effectiveOpencodeMessageId
4862
+ );
4010
4863
  if (outcome === "abandoned") {
4011
4864
  continue;
4012
4865
  }
@@ -4117,21 +4970,102 @@ var ChannelDriver = class _ChannelDriver {
4117
4970
  }
4118
4971
  this.unconfirmedDispatchFailures.delete(message.id);
4119
4972
  this.dispatchNotStartedSignalled.delete(message.id);
4973
+ this.releasedOpencodeIds.delete(message.id);
4120
4974
  this.dispatched.add(message.id);
4121
4975
  this.registerInFlight(conv, sessionId, message, opencodeMessageId);
4122
4976
  dispatched += 1;
4123
4977
  void this.postSignal(conv.id, message.id, "dispatched");
4124
4978
  }
4125
4979
  if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
4126
- this.log({
4127
- level: "warn",
4128
- message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
4129
- conversation_id: conv.id
4130
- });
4980
+ this.reportWedgedConversation(conv, messages);
4981
+ } else if (dispatched > 0) {
4982
+ this.wedgeWarnings.delete(conv.id);
4131
4983
  }
4132
4984
  this.ensureWatcherRunning(sessionId);
4133
4985
  return dispatched;
4134
4986
  }
4987
+ /**
4988
+ * The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
4989
+ * escalated (#1618 WI-4). `messages` is the conversation's full pending list
4990
+ * on THIS tick — the caller has already confirmed every one of them is a
4991
+ * skip-because-already-`dispatched`, the exact signature of a message stuck
4992
+ * acknowledged-but-never-worked.
4993
+ *
4994
+ * Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
4995
+ * (52,843 occurrences observed in one incident) — burning the GLOBAL
4996
+ * 30-events/60s `runner-activity-telemetry.ts` budget that was itself
4997
+ * suppressing the diagnostics needed to debug the wedge. The `warn` log (and
4998
+ * the `dispatch_wedged` signal once the wedge has persisted past the same
4999
+ * interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
5000
+ * naming the consecutive-tick count so the operator sees magnitude rather
5001
+ * than repetition.
5002
+ *
5003
+ * Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
5004
+ * unconditionally on this same tick and is already recovering anything it
5005
+ * can see. This is reporting only — see `countUntrackedIds`'s doc for the
5006
+ * one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
5007
+ */
5008
+ reportWedgedConversation(conv, messages) {
5009
+ const now = this.now();
5010
+ const existing = this.wedgeWarnings.get(conv.id);
5011
+ const firstWedgedAt = existing?.firstWedgedAt ?? now;
5012
+ const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
5013
+ const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
5014
+ if (!dueForWarn) {
5015
+ this.wedgeWarnings.delete(conv.id);
5016
+ this.wedgeWarnings.set(conv.id, {
5017
+ firstWedgedAt,
5018
+ lastWarnedAt: existing.lastWarnedAt,
5019
+ consecutiveTicks
5020
+ });
5021
+ return;
5022
+ }
5023
+ const stuckForMs = now - firstWedgedAt;
5024
+ const untracked = this.countUntrackedIds(messages);
5025
+ this.log({
5026
+ level: "warn",
5027
+ message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
5028
+ conversation_id: conv.id
5029
+ });
5030
+ this.wedgeWarnings.delete(conv.id);
5031
+ this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
5032
+ while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
5033
+ const oldest = this.wedgeWarnings.keys().next().value;
5034
+ if (oldest === void 0) break;
5035
+ this.wedgeWarnings.delete(oldest);
5036
+ }
5037
+ if (stuckForMs >= this.wedgeWarningIntervalMs) {
5038
+ void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
5039
+ stuck_for_ms: stuckForMs,
5040
+ untracked
5041
+ });
5042
+ }
5043
+ }
5044
+ /**
5045
+ * How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
5046
+ * WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
5047
+ * `dispatched`/`inFlight` pairing invariant holds by construction across
5048
+ * every `dispatched.add` site (see its own doc comment), so `> 0` here means
5049
+ * that invariant has actually been violated for this conversation: there is
5050
+ * no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
5051
+ * `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
5052
+ * recovering. One pass over `this.watchers`, called only when the throttled
5053
+ * warning above is due to fire — not every tick.
5054
+ */
5055
+ countUntrackedIds(messages) {
5056
+ let untracked = 0;
5057
+ for (const message of messages) {
5058
+ let tracked = false;
5059
+ for (const watcher of this.watchers.values()) {
5060
+ if (watcher.inFlight.has(message.id)) {
5061
+ tracked = true;
5062
+ break;
5063
+ }
5064
+ }
5065
+ if (!tracked) untracked += 1;
5066
+ }
5067
+ return untracked;
5068
+ }
4135
5069
  /**
4136
5070
  * Poll a session's message list for the re-drive fence (#965), via the
4137
5071
  * INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
@@ -4200,9 +5134,16 @@ var ChannelDriver = class _ChannelDriver {
4200
5134
  * for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
4201
5135
  * other failure resolves to `unresolved` and is retried whole on the next
4202
5136
  * ~2s drain tick.
5137
+ *
5138
+ * `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
5139
+ * server `opencode_message_id` when present, else the stall watchdog's local
5140
+ * `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
5141
+ * `message` so every line below — and the signals this method posts —
5142
+ * keeps reporting the REAL server row; a shadow-copied `message` would
5143
+ * silently diverge from it.
4203
5144
  */
4204
- async resolveRedrive(conv, sessionId, message, sessionCreated) {
4205
- const ocId = message.opencode_message_id ?? null;
5145
+ async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
5146
+ const ocId = effectiveOpencodeMessageId;
4206
5147
  if (sessionCreated) {
4207
5148
  this.clearRedriveUnresolved(message.id);
4208
5149
  void this.postSignal(conv.id, message.id, "redrive_redispatched");
@@ -4432,7 +5373,13 @@ var ChannelDriver = class _ChannelDriver {
4432
5373
  }
4433
5374
  return "unresolved";
4434
5375
  }
4435
- /** Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved` outcome). */
5376
+ /**
5377
+ * Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
5378
+ * outcome) — including the stall watchdog's local re-drive fence (#1618): once
5379
+ * `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
5380
+ * a real server-side `opencode_message_id` again or is no longer pending, so
5381
+ * the fence entry is no longer needed.
5382
+ */
4436
5383
  clearRedriveUnresolved(messageId) {
4437
5384
  this.redriveUnresolvedSince.delete(messageId);
4438
5385
  this.redriveUnresolvedSignalled.delete(messageId);
@@ -4440,6 +5387,7 @@ var ChannelDriver = class _ChannelDriver {
4440
5387
  this.redriveOutcomeUnreportedSignalled.delete(messageId);
4441
5388
  this.redriveOutcomeFailingSince.delete(messageId);
4442
5389
  this.redriveOutcomeAbandonedSignalled.delete(messageId);
5390
+ this.releasedOpencodeIds.delete(messageId);
4443
5391
  }
4444
5392
  /**
4445
5393
  * #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
@@ -4594,6 +5542,21 @@ var ChannelDriver = class _ChannelDriver {
4594
5542
  this.supersededSessions.delete(oldest);
4595
5543
  }
4596
5544
  }
5545
+ /**
5546
+ * Record the local re-drive fence for a message force-released without
5547
+ * completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
5548
+ * `removeInFlight`, which is about to drop the `InFlightMessage` this reads
5549
+ * `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
5550
+ */
5551
+ recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
5552
+ this.releasedOpencodeIds.delete(evidentMessageId);
5553
+ this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
5554
+ while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
5555
+ const oldest = this.releasedOpencodeIds.keys().next().value;
5556
+ if (oldest === void 0) return;
5557
+ this.releasedOpencodeIds.delete(oldest);
5558
+ }
5559
+ }
4597
5560
  /** Whether `sessionId` is the session this conversation has abandoned (#553). */
4598
5561
  isSuperseded(conversationId, sessionId) {
4599
5562
  return this.supersededSessions.get(conversationId) === sessionId;
@@ -4635,6 +5598,17 @@ var ChannelDriver = class _ChannelDriver {
4635
5598
  message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
4636
5599
  conversation_id: conv.id
4637
5600
  });
5601
+ const watcher = this.watchers.get(bound);
5602
+ if (watcher) {
5603
+ for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
5604
+ this.recordReleasedOpencodeId(evidentMessageId, bound, inFlight.opencodeMessageId);
5605
+ this.removeInFlight(watcher, evidentMessageId);
5606
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
5607
+ recovery: "session_gone_released"
5608
+ });
5609
+ }
5610
+ this.watchers.delete(bound);
5611
+ }
4638
5612
  this.sessions.delete(conv.id);
4639
5613
  return { sessionId: await this.createAndBindSession(conv.id), created: true };
4640
5614
  }
@@ -4825,15 +5799,7 @@ var ChannelDriver = class _ChannelDriver {
4825
5799
  registerInFlight(conv, sessionId, message, opencodeMessageId) {
4826
5800
  let watcher = this.watchers.get(sessionId);
4827
5801
  if (!watcher) {
4828
- watcher = {
4829
- conv,
4830
- inFlight: /* @__PURE__ */ new Map(),
4831
- loop: null,
4832
- reportedQuestions: /* @__PURE__ */ new Set(),
4833
- reportedPermissions: /* @__PURE__ */ new Set(),
4834
- lastGoodPollAt: this.now(),
4835
- hadUsablePoll: false
4836
- };
5802
+ watcher = this.newSessionWatcher(conv);
4837
5803
  this.watchers.set(sessionId, watcher);
4838
5804
  }
4839
5805
  const now = this.now();
@@ -4859,11 +5825,33 @@ var ChannelDriver = class _ChannelDriver {
4859
5825
  deliveryDeadlineAnchored: false,
4860
5826
  b2PinnedSinceMs: 0,
4861
5827
  b2LastDescendantCheckMs: 0,
5828
+ b2RootOngoingHeldLogged: false,
4862
5829
  b2AbandonedSignalled: false,
4863
5830
  ambiguousPinnedSinceMs: 0,
4864
5831
  ambiguousResolved: false
4865
5832
  });
4866
5833
  }
5834
+ /**
5835
+ * Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
5836
+ * EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
5837
+ * misread as stalled by the very first reconciliation that sees it.
5838
+ */
5839
+ newSessionWatcher(conv) {
5840
+ const now = this.now();
5841
+ return {
5842
+ conv,
5843
+ inFlight: /* @__PURE__ */ new Map(),
5844
+ loop: null,
5845
+ reportedQuestions: /* @__PURE__ */ new Set(),
5846
+ reportedPermissions: /* @__PURE__ */ new Set(),
5847
+ lastGoodPollAt: now,
5848
+ hadUsablePoll: false,
5849
+ generation: 0,
5850
+ lastTickAt: now,
5851
+ lastObservedTickAt: now,
5852
+ consecutiveStallRestarts: 0
5853
+ };
5854
+ }
4867
5855
  /**
4868
5856
  * Register a RE-ADOPTED `processing` message with its session watcher
4869
5857
  * (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
@@ -4893,15 +5881,7 @@ var ChannelDriver = class _ChannelDriver {
4893
5881
  registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
4894
5882
  let watcher = this.watchers.get(sessionId);
4895
5883
  if (!watcher) {
4896
- watcher = {
4897
- conv,
4898
- inFlight: /* @__PURE__ */ new Map(),
4899
- loop: null,
4900
- reportedQuestions: /* @__PURE__ */ new Set(),
4901
- reportedPermissions: /* @__PURE__ */ new Set(),
4902
- lastGoodPollAt: this.now(),
4903
- hadUsablePoll: false
4904
- };
5884
+ watcher = this.newSessionWatcher(conv);
4905
5885
  this.watchers.set(sessionId, watcher);
4906
5886
  }
4907
5887
  watcher.inFlight.set(message.id, {
@@ -4940,17 +5920,116 @@ var ChannelDriver = class _ChannelDriver {
4940
5920
  deliveryDeadlineAnchored: false,
4941
5921
  b2PinnedSinceMs: 0,
4942
5922
  b2LastDescendantCheckMs: 0,
5923
+ b2RootOngoingHeldLogged: false,
4943
5924
  b2AbandonedSignalled: false,
4944
5925
  ambiguousPinnedSinceMs: 0,
4945
5926
  ambiguousResolved: false
4946
5927
  });
4947
5928
  }
5929
+ /**
5930
+ * Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
5931
+ * restarts any per-session watcher whose loop has exited or stopped ticking
5932
+ * — escalating to a bounded force-release only once
5933
+ * `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
5934
+ * it. Fully synchronous: it only inspects in-memory state and calls the
5935
+ * synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
5936
+ * run from the very top of `drainPending()` — ahead of the un-timed
5937
+ * `getPendingConversations()` await that would otherwise be able to disable
5938
+ * it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
5939
+ * `drainPending()` from being CALLED again at all, not just from finishing).
5940
+ *
5941
+ * Restarts the loop rather than releasing messages directly: a blind release
5942
+ * would let the next drain re-`prompt_async` a turn that may still be
5943
+ * running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
5944
+ * re-polls with each message's `opencodeMessageId` still in hand and lets
5945
+ * the existing, audited `!activelyRunning` give-up decide, same as it always
5946
+ * has.
5947
+ *
5948
+ * Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
5949
+ * that shape has no in-flight entry and therefore no `opencodeMessageId` to
5950
+ * fence a release with, so releasing it here would blind-re-POST a possibly-
5951
+ * running turn — and there is no conversation id in hand to signal with
5952
+ * either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
5953
+ * instead, where a conversation id already exists. If you find yourself
5954
+ * wanting to add a `dispatched` sweep here, don't — read the drain-wedge
5955
+ * plan's §3/D5 first.
5956
+ */
5957
+ reconcileWatchers() {
5958
+ const now = this.now();
5959
+ for (const [sessionId, watcher] of [...this.watchers]) {
5960
+ if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
5961
+ watcher.consecutiveStallRestarts = 0;
5962
+ }
5963
+ watcher.lastObservedTickAt = watcher.lastTickAt;
5964
+ if (watcher.inFlight.size === 0 && watcher.loop === null) {
5965
+ this.watchers.delete(sessionId);
5966
+ continue;
5967
+ }
5968
+ if (watcher.loop === null && watcher.inFlight.size > 0) {
5969
+ if (now - watcher.lastTickAt < this.watcherStallMs) continue;
5970
+ this.log({
5971
+ level: "warn",
5972
+ message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
5973
+ conversation_id: watcher.conv.id
5974
+ });
5975
+ this.ensureWatcherRunning(sessionId);
5976
+ for (const evidentMessageId of watcher.inFlight.keys()) {
5977
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
5978
+ recovery: "loop_exited"
5979
+ });
5980
+ }
5981
+ continue;
5982
+ }
5983
+ if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
5984
+ const stalledForMs = now - watcher.lastTickAt;
5985
+ watcher.consecutiveStallRestarts += 1;
5986
+ if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
5987
+ this.log({
5988
+ level: "error",
5989
+ message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
5990
+ conversation_id: watcher.conv.id
5991
+ });
5992
+ for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
5993
+ this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
5994
+ this.removeInFlight(watcher, evidentMessageId);
5995
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
5996
+ recovery: "unrecoverable_released"
5997
+ });
5998
+ }
5999
+ watcher.generation += 1;
6000
+ this.watchers.delete(sessionId);
6001
+ continue;
6002
+ }
6003
+ watcher.generation += 1;
6004
+ watcher.loop = null;
6005
+ watcher.lastGoodPollAt = now;
6006
+ watcher.lastTickAt = now;
6007
+ watcher.lastObservedTickAt = watcher.lastTickAt;
6008
+ this.ensureWatcherRunning(sessionId);
6009
+ this.log({
6010
+ level: "warn",
6011
+ message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
6012
+ conversation_id: watcher.conv.id
6013
+ });
6014
+ for (const evidentMessageId of watcher.inFlight.keys()) {
6015
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
6016
+ recovery: "loop_stalled"
6017
+ });
6018
+ }
6019
+ }
6020
+ }
6021
+ }
4948
6022
  /**
4949
6023
  * Start (but do NOT await) the per-session watcher loop if it has in-flight
4950
6024
  * work and is not already running. Single-flight per session. The loop is
4951
6025
  * tracked on the watcher and cleared when it settles; it never rejects (fully
4952
6026
  * guarded), so a failed poll/callback can never crash the run loop — the cron
4953
6027
  * stays as the safety net.
6028
+ *
6029
+ * The generation started here (#1618) is captured in the `.finally` closure
6030
+ * so a RETIRED loop settling late — after `reconcileWatchers` has already
6031
+ * restarted this watcher under a newer generation — can neither null the new
6032
+ * loop's handle nor delete a watcher that still has live work.
4954
6033
  */
4955
6034
  ensureWatcherRunning(sessionId) {
4956
6035
  const watcher = this.watchers.get(sessionId);
@@ -4960,7 +6039,9 @@ var ChannelDriver = class _ChannelDriver {
4960
6039
  this.watchers.delete(sessionId);
4961
6040
  return;
4962
6041
  }
4963
- const loop = this.runWatcherLoop(sessionId, watcher).finally(() => {
6042
+ const generation = watcher.generation;
6043
+ const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
6044
+ if (watcher.generation !== generation) return;
4964
6045
  watcher.loop = null;
4965
6046
  if (watcher.inFlight.size === 0) {
4966
6047
  this.watchers.delete(sessionId);
@@ -4980,11 +6061,25 @@ var ChannelDriver = class _ChannelDriver {
4980
6061
  * `source_message_id`;
4981
6062
  * 4. drops messages that completed or timed out from the in-flight set.
4982
6063
  * Exits when the in-flight set empties. Never throws.
6064
+ *
6065
+ * `generation` (#1618) is the incarnation this call was started under.
6066
+ * `reconcileWatchers` can restart a stalled loop by bumping
6067
+ * `watcher.generation` and starting a NEW `runWatcherLoop` over the same
6068
+ * `SessionWatcher` object — the stalled promise itself cannot be cancelled,
6069
+ * so this loop instead checks at the top of every iteration, right after
6070
+ * waking from `sleep`, and right before servicing any message, and quietly
6071
+ * retires (returns without touching anything) the moment it is no longer the
6072
+ * watcher's current generation. Retiring mid-tick can still let ONE
6073
+ * `serviceInFlightMessage` pass complete first — acceptable, since that
6074
+ * method contains no non-idempotent action.
4983
6075
  */
4984
- async runWatcherLoop(sessionId, watcher) {
6076
+ async runWatcherLoop(sessionId, watcher, generation) {
4985
6077
  try {
4986
6078
  while (watcher.inFlight.size > 0) {
6079
+ if (watcher.generation !== generation) return;
6080
+ watcher.lastTickAt = this.now();
4987
6081
  await this.sleep(this.pausedPollIntervalMs);
6082
+ if (watcher.generation !== generation) return;
4988
6083
  let messages = null;
4989
6084
  try {
4990
6085
  const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
@@ -5005,6 +6100,7 @@ var ChannelDriver = class _ChannelDriver {
5005
6100
  }
5006
6101
  }
5007
6102
  const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
6103
+ if (watcher.generation !== generation) return;
5008
6104
  for (const inFlight of [...watcher.inFlight.values()]) {
5009
6105
  await this.serviceInFlightMessage(
5010
6106
  sessionId,
@@ -5178,6 +6274,7 @@ var ChannelDriver = class _ChannelDriver {
5178
6274
  if (snapshotReadable) {
5179
6275
  inFlight.b2PinnedSinceMs = 0;
5180
6276
  inFlight.b2LastDescendantCheckMs = 0;
6277
+ inFlight.b2RootOngoingHeldLogged = false;
5181
6278
  inFlight.b2AbandonedSignalled = false;
5182
6279
  }
5183
6280
  } else {
@@ -5189,11 +6286,15 @@ var ChannelDriver = class _ChannelDriver {
5189
6286
  const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
5190
6287
  if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
5191
6288
  inFlight.b2LastDescendantCheckMs = this.now();
5192
- const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
6289
+ const [descendantOngoing, rootOngoing] = await Promise.all([
6290
+ this.isAnyDescendantSessionOngoing(sessionId),
6291
+ isSessionOngoing(this.port, sessionId)
6292
+ ]);
5193
6293
  if (isB2AbandonmentConfirmed({
5194
6294
  pinnedForMs,
5195
6295
  minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
5196
- descendantOngoing
6296
+ descendantOngoing,
6297
+ rootOngoing
5197
6298
  })) {
5198
6299
  inFlight.b2AbandonedSignalled = true;
5199
6300
  this.log({
@@ -5202,12 +6303,26 @@ var ChannelDriver = class _ChannelDriver {
5202
6303
  conversation_id: conv.id,
5203
6304
  message_id: id
5204
6305
  });
6306
+ const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
5205
6307
  void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
5206
- watched_for_ms: pinnedForMs
6308
+ watched_for_ms: pinnedForMs,
6309
+ finish: reply?.info?.finish ?? reply?.finish,
6310
+ ...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
6311
+ ...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
6312
+ opencode_message_id: inFlight.opencodeMessageId
5207
6313
  });
5208
6314
  await this.settleMessageDone(sessionId, watcher, inFlight, messages);
5209
6315
  return;
5210
6316
  }
6317
+ if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
6318
+ inFlight.b2RootOngoingHeldLogged = true;
6319
+ this.log({
6320
+ level: "warn",
6321
+ message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with root_ongoing=${rootOngoing} and descendant_ongoing=${descendantOngoing} \u2014 holding until OpenCode confirms the root is idle`,
6322
+ conversation_id: conv.id,
6323
+ message_id: id
6324
+ });
6325
+ }
5211
6326
  }
5212
6327
  }
5213
6328
  const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
@@ -6950,6 +8065,7 @@ var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_
6950
8065
  var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
6951
8066
  var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
6952
8067
  var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
8068
+ var CHILD_STOP_TIMEOUT_MS = 1e4;
6953
8069
  function resolveLogLevel(options) {
6954
8070
  const accepted = Object.keys(LOG_LEVELS);
6955
8071
  const validate = (value, source) => {
@@ -6980,7 +8096,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
6980
8096
  if (trimmed === "") {
6981
8097
  throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
6982
8098
  }
6983
- const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join4(homeDir, trimmed.slice(2)) : trimmed;
8099
+ const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join7(homeDir, trimmed.slice(2)) : trimmed;
6984
8100
  if (!isAbsolute2(expanded)) {
6985
8101
  throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
6986
8102
  }
@@ -7078,7 +8194,7 @@ function logActivity(state, entry) {
7078
8194
  const level = entry.level ?? (entry.type === "error" ? "error" : "info");
7079
8195
  if (!meetsThreshold(state, level)) return;
7080
8196
  forwardRunnerActivity(
7081
- { level, message: entry.message, error: entry.error },
8197
+ { level, message: entry.message, error: entry.error, metadata: entry.metadata },
7082
8198
  { agentId: state.agentId, authHeader: state.authHeader }
7083
8199
  );
7084
8200
  const fullEntry = {
@@ -7098,6 +8214,26 @@ function logActivity(state, entry) {
7098
8214
  }
7099
8215
  }
7100
8216
  }
8217
+ function reportSessionDbRecovery(state) {
8218
+ try {
8219
+ const report = drainSessionDbRecoveryReport({ homeDir: homedir4(), env: process.env });
8220
+ for (const record of report.records) {
8221
+ const activity = buildSessionDbRecoveryActivity(record);
8222
+ if (!activity) throw new Error("could not map session-DB recovery record");
8223
+ logActivity(state, {
8224
+ type: activity.level === "error" ? "error" : "info",
8225
+ level: activity.level,
8226
+ ...activity.level === "error" ? { error: activity.message } : { message: activity.message },
8227
+ metadata: activity.metadata
8228
+ });
8229
+ }
8230
+ acknowledgeSessionDbRecoveryReport(report.path);
8231
+ } catch (error2) {
8232
+ console.error(
8233
+ `[run] could not report session-DB recovery activity: ${error2 instanceof Error ? error2.message : String(error2)}`
8234
+ );
8235
+ }
8236
+ }
7101
8237
  function displayStatus(state) {
7102
8238
  if (!state.interactive) return;
7103
8239
  const attempt = state.connection?.reconnectAttempt ?? 0;
@@ -7187,6 +8323,8 @@ async function driveChannels(state, driver) {
7187
8323
  let unreachableMs = 0;
7188
8324
  let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
7189
8325
  let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
8326
+ let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
8327
+ let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
7190
8328
  while (state.running) {
7191
8329
  const cycleStartedAtMs = performance.now();
7192
8330
  let idleThisCycle = false;
@@ -7210,11 +8348,19 @@ async function driveChannels(state, driver) {
7210
8348
  state.messageCount += processed;
7211
8349
  const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
7212
8350
  lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
7213
- const appliedFiles = driver.fileSyncActivity().appliedFiles;
8351
+ const fileActivitySnapshot = driver.fileSyncActivity();
8352
+ const appliedFiles = fileActivitySnapshot.appliedFiles;
7214
8353
  const filesApplied = appliedFiles !== lastSeenAppliedFiles;
7215
8354
  const fileActivity = carriedOverFileSync || filesApplied;
7216
8355
  lastSeenAppliedFiles = appliedFiles;
7217
- if (filesApplied) state.claudeUsageRearm?.();
8356
+ const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
8357
+ const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
8358
+ lastSeenClaudeApplies = claudeCredentialApplies;
8359
+ if (claudeCredentialApplied) state.claudeUsageRearm?.();
8360
+ const opencodeAuthApplies = fileActivitySnapshot.opencodeAuthApplyCount;
8361
+ const opencodeAuthApplied = opencodeAuthApplies !== lastSeenOpencodeAuthApplies;
8362
+ lastSeenOpencodeAuthApplies = opencodeAuthApplies;
8363
+ if (opencodeAuthApplied) state.openaiUsageRearm?.();
7218
8364
  if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
7219
8365
  idlePolls = 0;
7220
8366
  idleMs = 0;
@@ -7283,7 +8429,7 @@ async function driveChannels(state, driver) {
7283
8429
  var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
7284
8430
  var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
7285
8431
  function sessionDbPath() {
7286
- return join4(homedir3(), ".local", "share", "opencode", "opencode.db");
8432
+ return join7(homedir4(), ".local", "share", "opencode", "opencode.db");
7287
8433
  }
7288
8434
  async function runSweep(state, driver, config) {
7289
8435
  const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
@@ -7366,7 +8512,7 @@ function scheduleSessionCleanup(state, driver, options) {
7366
8512
  for (const warning2 of config.warnings) {
7367
8513
  logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
7368
8514
  }
7369
- const dbBytes = statSessionDbBytes(homedir3());
8515
+ const dbBytes = statSessionDbBytes(homedir4());
7370
8516
  void (async () => {
7371
8517
  const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
7372
8518
  const sizeWarning = buildSessionStoreSizeWarning({
@@ -7394,95 +8540,110 @@ function scheduleSessionCleanup(state, driver, options) {
7394
8540
  );
7395
8541
  state.sessionCleanupTimers.push(interval, firstSweep);
7396
8542
  }
7397
- function claudeUsageFailureStreakSuffix(consecutiveFailures) {
7398
- return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
7399
- }
7400
- function scheduleClaudeUsageReporting(state, options) {
7401
- const { mode, warnings } = resolveClaudeUsageReportingMode(
7402
- options.claudeUsageReporting,
7403
- process.env
7404
- );
8543
+ function scheduleUsageReporting(state, params) {
8544
+ const { mode, warnings } = params.resolved;
7405
8545
  for (const warning2 of warnings) {
7406
8546
  logActivity(state, {
7407
8547
  type: "info",
7408
8548
  level: "warn",
7409
- message: `Claude usage reporting: ${warning2}`
8549
+ message: `${params.label} usage reporting: ${warning2}`
7410
8550
  });
7411
8551
  }
7412
8552
  if (mode === "off") {
7413
8553
  logActivity(state, {
7414
8554
  type: "info",
7415
8555
  level: "debug",
7416
- message: "Claude usage reporting is off (--claude-usage-reporting off)"
8556
+ message: `${params.label} usage reporting is off (${params.offFlagHint})`
7417
8557
  });
7418
8558
  return null;
7419
8559
  }
7420
8560
  let consecutiveFailures = 0;
7421
- let armed = false;
8561
+ let phase = "dormant";
7422
8562
  let rearmRequested = false;
8563
+ const armProbe = () => {
8564
+ phase = "probe-pending";
8565
+ params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
8566
+ };
7423
8567
  const scheduleNextTick = () => {
7424
- armed = true;
7425
- rearmRequested = false;
7426
- state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
8568
+ if (rearmRequested) {
8569
+ rearmRequested = false;
8570
+ armProbe();
8571
+ return;
8572
+ }
8573
+ phase = "steady-pending";
8574
+ params.setTimer(setTimeout(() => void tick(false), params.nextDelayMs()));
7427
8575
  };
7428
8576
  const rearm = () => {
7429
- if (armed) {
7430
- rearmRequested = true;
7431
- return;
8577
+ switch (phase) {
8578
+ case "tick-in-flight":
8579
+ rearmRequested = true;
8580
+ return;
8581
+ case "probe-pending":
8582
+ return;
8583
+ case "steady-pending":
8584
+ if (params.getTimer()) {
8585
+ clearTimeout(params.getTimer());
8586
+ params.setTimer(null);
8587
+ }
8588
+ rearmRequested = false;
8589
+ armProbe();
8590
+ return;
8591
+ case "dormant":
8592
+ rearmRequested = false;
8593
+ armProbe();
8594
+ return;
7432
8595
  }
7433
- rearmRequested = false;
7434
- armed = true;
7435
- state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
7436
8596
  };
7437
8597
  const tick = async (isProbe) => {
8598
+ phase = "tick-in-flight";
7438
8599
  try {
7439
- const usage = await getClaudeUsage();
7440
- const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
8600
+ const usage = await params.fetchUsage();
8601
+ const result = await params.report(usage);
7441
8602
  if (result.ok) {
7442
8603
  if (consecutiveFailures > 0) {
7443
8604
  logActivity(state, {
7444
8605
  type: "info",
7445
8606
  level: "info",
7446
- message: "Claude usage reporting recovered"
8607
+ message: `${params.label} usage reporting recovered`
7447
8608
  });
7448
8609
  }
7449
8610
  consecutiveFailures = 0;
7450
8611
  logActivity(state, {
7451
8612
  type: "info",
7452
8613
  level: "debug",
7453
- message: "Reported Claude usage to Evident"
8614
+ message: `Reported ${params.label} usage to Evident`
7454
8615
  });
7455
8616
  } else {
7456
8617
  consecutiveFailures++;
7457
8618
  logActivity(state, {
7458
8619
  type: "info",
7459
- level: claudeUsageFailureLogLevel(consecutiveFailures),
7460
- message: `Failed to report Claude usage: ${result.error}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
8620
+ level: params.failureLogLevel(consecutiveFailures),
8621
+ message: `Failed to report ${params.label} usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
7461
8622
  });
7462
8623
  }
7463
8624
  scheduleNextTick();
7464
8625
  } catch (error2) {
7465
- if (error2 instanceof ClaudeUsageError && isLocalCredentialProblem(error2)) {
8626
+ if (params.isLocalCredentialProblem(error2)) {
7466
8627
  if (mode === "on") {
7467
8628
  logActivity(state, {
7468
8629
  type: "info",
7469
8630
  level: "warn",
7470
- message: "Claude usage reporting is forced on but no usable Claude Code login was found \u2014 run `claude` to sign in; reporting will keep retrying"
8631
+ message: `${params.label} usage reporting is forced on but no usable login was found \u2014 ${params.forcedOnHint}; reporting will keep retrying`
7471
8632
  });
7472
8633
  scheduleNextTick();
7473
8634
  } else if (isProbe) {
7474
8635
  logActivity(state, {
7475
8636
  type: "info",
7476
8637
  level: "debug",
7477
- message: `Claude usage reporting: ${error2.message}`
8638
+ message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
7478
8639
  });
7479
- armed = false;
8640
+ phase = "dormant";
7480
8641
  if (rearmRequested) rearm();
7481
8642
  } else {
7482
8643
  logActivity(state, {
7483
8644
  type: "info",
7484
8645
  level: "debug",
7485
- message: `Claude usage reporting: ${error2.message}`
8646
+ message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
7486
8647
  });
7487
8648
  scheduleNextTick();
7488
8649
  }
@@ -7491,17 +8652,128 @@ function scheduleClaudeUsageReporting(state, options) {
7491
8652
  const message = error2 instanceof Error ? error2.message : String(error2);
7492
8653
  logActivity(state, {
7493
8654
  type: "info",
7494
- level: claudeUsageFailureLogLevel(consecutiveFailures),
7495
- message: `Claude usage reporting failed: ${message}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
8655
+ level: params.failureLogLevel(consecutiveFailures),
8656
+ message: `${params.label} usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
7496
8657
  });
7497
8658
  scheduleNextTick();
7498
8659
  }
7499
8660
  }
7500
8661
  };
7501
- armed = true;
7502
- state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
8662
+ armProbe();
7503
8663
  return rearm;
7504
8664
  }
8665
+ function scheduleClaudeUsageReporting(state, options) {
8666
+ return scheduleUsageReporting(state, {
8667
+ label: "Claude",
8668
+ resolved: resolveClaudeUsageReportingMode(options.claudeUsageReporting, process.env),
8669
+ offFlagHint: "--claude-usage-reporting off",
8670
+ getTimer: () => state.claudeUsageTimer,
8671
+ setTimer: (timer) => {
8672
+ state.claudeUsageTimer = timer;
8673
+ },
8674
+ fetchUsage: async () => {
8675
+ const usage = await getClaudeUsage();
8676
+ if (usage.ownerLookupError) {
8677
+ logActivity(state, {
8678
+ type: "info",
8679
+ level: "debug",
8680
+ message: `Claude usage owner lookup failed: ${usage.ownerLookupError}`
8681
+ });
8682
+ }
8683
+ return usage;
8684
+ },
8685
+ report: (usage) => reportClaudeUsage(state.agentId, state.authHeader, usage),
8686
+ isLocalCredentialProblem,
8687
+ forcedOnHint: "run `claude` to sign in",
8688
+ firstDelayMs: () => FIRST_REPORT_DELAY_MS,
8689
+ nextDelayMs: nextReportDelayMs,
8690
+ failureLogLevel: claudeUsageFailureLogLevel
8691
+ });
8692
+ }
8693
+ var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
8694
+ var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
8695
+ var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
8696
+ function scheduleResourceUsageReporting(state, options) {
8697
+ const { enabled, warnings } = resolveResourceUsageReportingEnabled(
8698
+ options.resourceUsageReporting,
8699
+ process.env
8700
+ );
8701
+ for (const warning2 of warnings) {
8702
+ logActivity(state, {
8703
+ type: "info",
8704
+ level: "warn",
8705
+ message: `Resource usage reporting: ${warning2}`
8706
+ });
8707
+ }
8708
+ if (!enabled) {
8709
+ logActivity(state, {
8710
+ type: "info",
8711
+ level: "debug",
8712
+ message: "Resource usage reporting is off (--no-resource-usage-reporting)"
8713
+ });
8714
+ return;
8715
+ }
8716
+ const collect = createResourceUsageCollector(homedir4());
8717
+ let consecutiveFailures = 0;
8718
+ const tick = async () => {
8719
+ try {
8720
+ const { usage, warnings: collectWarnings } = await collect();
8721
+ for (const warning2 of collectWarnings) {
8722
+ logActivity(state, {
8723
+ type: "info",
8724
+ level: "debug",
8725
+ message: `Resource usage collection: ${warning2}`
8726
+ });
8727
+ }
8728
+ const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
8729
+ if (result.ok) {
8730
+ if (consecutiveFailures > 0) {
8731
+ logActivity(state, {
8732
+ type: "info",
8733
+ level: "info",
8734
+ message: "Resource usage reporting recovered"
8735
+ });
8736
+ }
8737
+ consecutiveFailures = 0;
8738
+ logActivity(state, {
8739
+ type: "info",
8740
+ level: "debug",
8741
+ message: "Reported resource usage to Evident"
8742
+ });
8743
+ } else {
8744
+ consecutiveFailures++;
8745
+ logActivity(state, {
8746
+ type: "info",
8747
+ level: reportFailureLogLevel(
8748
+ consecutiveFailures,
8749
+ RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
8750
+ ),
8751
+ message: `Failed to report resource usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
8752
+ });
8753
+ }
8754
+ } catch (error2) {
8755
+ consecutiveFailures++;
8756
+ const message = error2 instanceof Error ? error2.message : String(error2);
8757
+ logActivity(state, {
8758
+ type: "info",
8759
+ level: reportFailureLogLevel(
8760
+ consecutiveFailures,
8761
+ RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
8762
+ ),
8763
+ message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
8764
+ });
8765
+ } finally {
8766
+ state.resourceUsageTimer = setTimeout(
8767
+ () => void tick(),
8768
+ jitteredDelayMs(
8769
+ RESOURCE_USAGE_BASE_REPORT_DELAY_MS,
8770
+ RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION
8771
+ )
8772
+ );
8773
+ }
8774
+ };
8775
+ state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
8776
+ }
7505
8777
  async function notifyOffline(state) {
7506
8778
  if (!state.agentId || !state.authHeader) return;
7507
8779
  if (!state.connected) {
@@ -7542,6 +8814,15 @@ async function cleanup(state, opts = {}) {
7542
8814
  state.claudeUsageTimer = null;
7543
8815
  }
7544
8816
  state.claudeUsageRearm = null;
8817
+ if (state.openaiUsageTimer) {
8818
+ clearTimeout(state.openaiUsageTimer);
8819
+ state.openaiUsageTimer = null;
8820
+ }
8821
+ state.openaiUsageRearm = null;
8822
+ if (state.resourceUsageTimer) {
8823
+ clearTimeout(state.resourceUsageTimer);
8824
+ state.resourceUsageTimer = null;
8825
+ }
7545
8826
  if (opts.graceful && state.channelDriver) {
7546
8827
  state.channelDriver.stop();
7547
8828
  log2(state, "Draining in-flight channel work before shutdown...");
@@ -7572,15 +8853,31 @@ async function cleanup(state, opts = {}) {
7572
8853
  }
7573
8854
  if (state.opencodeProcess) {
7574
8855
  const opencodeProcess = state.opencodeProcess;
7575
- await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
8856
+ const result = await timeShutdownPhase(
8857
+ state,
8858
+ durations,
8859
+ "opencode_stop",
8860
+ () => stopOpenCodeAndWait(opencodeProcess, CHILD_STOP_TIMEOUT_MS)
8861
+ );
7576
8862
  if (state.interactive) {
7577
- logActivity(state, { type: "info", message: "Stopped OpenCode process" });
8863
+ logActivity(state, { type: "info", message: `Stopped OpenCode process (${result.outcome})` });
7578
8864
  displayStatus(state);
7579
8865
  } else {
7580
- log2(state, "Stopped OpenCode process");
8866
+ log2(state, `Stopped OpenCode process (${result.outcome})`);
7581
8867
  }
7582
8868
  state.opencodeProcess = null;
7583
8869
  }
8870
+ if (state.litestreamProcess) {
8871
+ const litestreamProcess = state.litestreamProcess;
8872
+ const result = await timeShutdownPhase(
8873
+ state,
8874
+ durations,
8875
+ "litestream_stop",
8876
+ () => stopSessionDbReplication(litestreamProcess, CHILD_STOP_TIMEOUT_MS)
8877
+ );
8878
+ log2(state, `Stopped litestream replication (${result.outcome})`);
8879
+ state.litestreamProcess = null;
8880
+ }
7584
8881
  return durations;
7585
8882
  }
7586
8883
  async function run(options) {
@@ -7589,7 +8886,7 @@ async function run(options) {
7589
8886
  let fileSyncDirectories;
7590
8887
  try {
7591
8888
  logLevel = resolveLogLevel(options);
7592
- fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir3());
8889
+ fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir4());
7593
8890
  } catch (error2) {
7594
8891
  const message = error2 instanceof Error ? error2.message : String(error2);
7595
8892
  if (options.json) {
@@ -7614,6 +8911,7 @@ async function run(options) {
7614
8911
  opencodeConnected: false,
7615
8912
  opencodeVersion: null,
7616
8913
  opencodeProcess: null,
8914
+ litestreamProcess: null,
7617
8915
  connection: null,
7618
8916
  channelDriver: null,
7619
8917
  running: true,
@@ -7624,6 +8922,9 @@ async function run(options) {
7624
8922
  sessionCleanupTimers: [],
7625
8923
  claudeUsageTimer: null,
7626
8924
  claudeUsageRearm: null,
8925
+ openaiUsageTimer: null,
8926
+ openaiUsageRearm: null,
8927
+ resourceUsageTimer: null,
7627
8928
  authHeader: ""
7628
8929
  };
7629
8930
  setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
@@ -7818,6 +9119,7 @@ async function run(options) {
7818
9119
  } else {
7819
9120
  log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
7820
9121
  }
9122
+ reportSessionDbRecovery(state);
7821
9123
  const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
7822
9124
  for (const warning2 of opencodeStartTimeoutWarnings) {
7823
9125
  logActivity(state, { type: "info", level: "warn", message: warning2 });
@@ -7874,6 +9176,40 @@ async function run(options) {
7874
9176
  ocSpinner?.fail(error2.message);
7875
9177
  throw error2;
7876
9178
  }
9179
+ if (options.litestreamConfig) {
9180
+ const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
9181
+ state.litestreamProcess = litestreamProcess;
9182
+ let failureHandled = false;
9183
+ const failRunForReplication = (message) => {
9184
+ if (failureHandled || state.shuttingDown || !state.running) return;
9185
+ failureHandled = true;
9186
+ state.shuttingDown = true;
9187
+ logActivity(state, { type: "error", error: message });
9188
+ if (state.interactive) displayStatus(state);
9189
+ void (async () => {
9190
+ try {
9191
+ await cleanup(state);
9192
+ await shutdownTelemetry();
9193
+ } catch (error2) {
9194
+ console.error(
9195
+ `[run] replication failure cleanup failed: ${error2 instanceof Error ? error2.message : String(error2)}`
9196
+ );
9197
+ }
9198
+ process.exit(1);
9199
+ })();
9200
+ };
9201
+ litestreamProcess.on("exit", (code, signal) => {
9202
+ failRunForReplication(
9203
+ `Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
9204
+ );
9205
+ });
9206
+ litestreamProcess.on("error", (error2) => {
9207
+ failRunForReplication(
9208
+ `Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
9209
+ );
9210
+ });
9211
+ log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
9212
+ }
7877
9213
  const tunnelSpinner = interactive && !state.json ? ora3("Connecting tunnel...").start() : null;
7878
9214
  const channelDriver = new ChannelDriver({
7879
9215
  agentId: state.agentId,
@@ -7885,7 +9221,7 @@ async function run(options) {
7885
9221
  // #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
7886
9222
  // REJECTED with `file_sync_disabled` on the ack, not silently ignored.
7887
9223
  fileSyncDirectories,
7888
- homeDir: homedir3(),
9224
+ homeDir: homedir4(),
7889
9225
  maxActiveSessions,
7890
9226
  log: (entry) => (
7891
9227
  // Thread the driver's real level straight through so `debug`/`warn`
@@ -8023,6 +9359,23 @@ async function run(options) {
8023
9359
  }
8024
9360
  scheduleSessionCleanup(state, channelDriver, options);
8025
9361
  state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
9362
+ state.openaiUsageRearm = scheduleUsageReporting(state, {
9363
+ label: "OpenAI",
9364
+ resolved: resolveOpenAiUsageReportingMode(options.openaiUsageReporting, process.env),
9365
+ offFlagHint: "--openai-usage-reporting off",
9366
+ getTimer: () => state.openaiUsageTimer,
9367
+ setTimer: (timer) => {
9368
+ state.openaiUsageTimer = timer;
9369
+ },
9370
+ fetchUsage: () => getOpenAiUsage(state.port),
9371
+ report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
9372
+ isLocalCredentialProblem: isLocalCredentialProblem2,
9373
+ forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
9374
+ firstDelayMs: firstReportDelayMs,
9375
+ nextDelayMs: usageReportDelayMs,
9376
+ failureLogLevel: usageReportFailureLogLevel
9377
+ });
9378
+ scheduleResourceUsageReporting(state, options);
8026
9379
  if (!interactive || state.json) {
8027
9380
  log2(state, "Driving channel messages...");
8028
9381
  }
@@ -8103,6 +9456,12 @@ program.command("run").description("Connect to Evident and process messages").op
8103
9456
  ).option(
8104
9457
  "--claude-usage-reporting <mode>",
8105
9458
  "Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
9459
+ ).option(
9460
+ "--openai-usage-reporting <mode>",
9461
+ "Report OpenAI plan usage to Evident: auto | on | off (default: auto). Env: EVIDENT_OPENAI_USAGE_REPORTING"
9462
+ ).option(
9463
+ "--no-resource-usage-reporting",
9464
+ "Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
8106
9465
  ).option(
8107
9466
  "--enable-file-sync-to <dir>",
8108
9467
  "Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
@@ -8111,6 +9470,9 @@ program.command("run").description("Connect to Evident and process messages").op
8111
9470
  ).option(
8112
9471
  "--tunnel-ready-file <path>",
8113
9472
  "Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
9473
+ ).option(
9474
+ "--litestream-config <path>",
9475
+ "Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
8114
9476
  ).action(
8115
9477
  (options) => {
8116
9478
  run({
@@ -8135,10 +9497,15 @@ program.command("run").description("Connect to Evident and process messages").op
8135
9497
  // Raw string — the resolver in run.ts single-sources parsing
8136
9498
  // (resolveClaudeUsageReportingMode).
8137
9499
  claudeUsageReporting: options.claudeUsageReporting,
9500
+ openaiUsageReporting: options.openaiUsageReporting,
9501
+ // Raw value — resolution is single-sourced in run.ts's
9502
+ // resolveResourceUsageReportingEnabled.
9503
+ resourceUsageReporting: options.resourceUsageReporting,
8138
9504
  // Raw values — expansion/validation is single-sourced in run.ts's
8139
9505
  // resolveFileSyncDirectories.
8140
9506
  enableFileSyncTo: options.enableFileSyncTo,
8141
- tunnelReadyFile: options.tunnelReadyFile
9507
+ tunnelReadyFile: options.tunnelReadyFile,
9508
+ litestreamConfig: options.litestreamConfig
8142
9509
  });
8143
9510
  }
8144
9511
  );