@evident-ai/cli 3.3.1-dev.a18f9b8 → 3.3.1-dev.b5645b1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -746,6 +746,32 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
746
746
  return { ok: false, error: describeBestEffortError(error2) };
747
747
  }
748
748
  }
749
+ async function reportResourceUsage(agentId, authHeader, usage) {
750
+ try {
751
+ const apiUrl = getApiUrlConfig();
752
+ const response = await fetch(`${apiUrl}/runners/${agentId}/resource-usage`, {
753
+ method: "POST",
754
+ headers: { Authorization: authHeader, "Content-Type": "application/json" },
755
+ body: JSON.stringify({
756
+ cpu_percent: usage.cpuPercent,
757
+ cpu_count: usage.cpuCount,
758
+ memory_total_bytes: usage.memoryTotalBytes,
759
+ memory_available_bytes: usage.memoryAvailableBytes
760
+ }),
761
+ signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
762
+ });
763
+ if (!response.ok) {
764
+ const serverMessage = await readErrorMessage(response);
765
+ return {
766
+ ok: false,
767
+ error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
768
+ };
769
+ }
770
+ return { ok: true };
771
+ } catch (error2) {
772
+ return { ok: false, error: describeBestEffortError(error2) };
773
+ }
774
+ }
749
775
  async function getAgentInfo(agentId, authHeader) {
750
776
  const apiUrl = getApiUrlConfig();
751
777
  try {
@@ -934,6 +960,7 @@ import { homedir } from "os";
934
960
  import { join } from "path";
935
961
  var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
936
962
  var KEYCHAIN_SERVICE = "Claude Code-credentials";
963
+ var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
937
964
  function parseClaudeCliCredentials(raw) {
938
965
  let parsed;
939
966
  try {
@@ -967,7 +994,7 @@ function readClaudeCliCredentials() {
967
994
  }
968
995
  }
969
996
  try {
970
- const raw = readFileSync(join(homedir(), ".claude", ".credentials.json"), "utf-8");
997
+ const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
971
998
  return parseClaudeCliCredentials(raw);
972
999
  } catch (err) {
973
1000
  const code = err.code;
@@ -1063,7 +1090,7 @@ async function claudeUsage() {
1063
1090
 
1064
1091
  // src/commands/run.ts
1065
1092
  import { homedir as homedir3 } from "os";
1066
- import { isAbsolute as isAbsolute2, join as join4, parse, resolve as resolvePath } from "path";
1093
+ import { isAbsolute as isAbsolute2, join as join5, parse, resolve as resolvePath } from "path";
1067
1094
  import chalk6 from "chalk";
1068
1095
 
1069
1096
  // ../../packages/types/src/agents/index.ts
@@ -1700,13 +1727,22 @@ function buildNoProviderWarning(hasProvider) {
1700
1727
  return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
1701
1728
  }
1702
1729
 
1730
+ // src/lib/http-timeout.ts
1731
+ var REQUEST_TIMEOUT_MS = 6e4;
1732
+ function withRequestTimeout(fetchImpl, timeoutMs) {
1733
+ return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
1734
+ }
1735
+
1703
1736
  // src/lib/opencode/session.ts
1737
+ function timedFetch(input, init) {
1738
+ return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
1739
+ }
1704
1740
  function opencodeBase(port) {
1705
1741
  return `http://127.0.0.1:${port}`;
1706
1742
  }
1707
1743
  async function getOpenCodeDirectory(port) {
1708
1744
  try {
1709
- const res = await fetch(`${opencodeBase(port)}/path`);
1745
+ const res = await timedFetch(`${opencodeBase(port)}/path`);
1710
1746
  if (!res.ok) return null;
1711
1747
  const body = await res.json();
1712
1748
  const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
@@ -1757,7 +1793,7 @@ function isAssistantInFlight(m) {
1757
1793
  }
1758
1794
  async function getSessionMessages(port, sessionId) {
1759
1795
  try {
1760
- const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
1796
+ const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
1761
1797
  if (!res.ok) return null;
1762
1798
  const body = await res.json();
1763
1799
  return Array.isArray(body) ? body : null;
@@ -1787,7 +1823,7 @@ function sessionLastActivityMs(session) {
1787
1823
  }
1788
1824
  async function listSessions(port) {
1789
1825
  try {
1790
- const res = await fetch(`${opencodeBase(port)}/session`);
1826
+ const res = await timedFetch(`${opencodeBase(port)}/session`);
1791
1827
  if (!res.ok) return null;
1792
1828
  const body = await res.json();
1793
1829
  return Array.isArray(body) ? body : null;
@@ -1797,7 +1833,7 @@ async function listSessions(port) {
1797
1833
  }
1798
1834
  async function deleteSession(port, id) {
1799
1835
  try {
1800
- const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
1836
+ const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
1801
1837
  return res.status >= 200 && res.status < 300;
1802
1838
  } catch {
1803
1839
  return false;
@@ -1805,7 +1841,7 @@ async function deleteSession(port, id) {
1805
1841
  }
1806
1842
  async function sessionExists(port, id) {
1807
1843
  try {
1808
- const res = await fetch(`${opencodeBase(port)}/session/${id}`);
1844
+ const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
1809
1845
  if (res.status >= 200 && res.status < 300) return true;
1810
1846
  if (res.status === 404) return false;
1811
1847
  return null;
@@ -1815,7 +1851,7 @@ async function sessionExists(port, id) {
1815
1851
  }
1816
1852
  async function getSessionStatuses(port) {
1817
1853
  try {
1818
- const res = await fetch(`${opencodeBase(port)}/session/status`);
1854
+ const res = await timedFetch(`${opencodeBase(port)}/session/status`);
1819
1855
  if (!res.ok) {
1820
1856
  console.error(
1821
1857
  `[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
@@ -1848,7 +1884,7 @@ async function createOpenCodeSession(port, directory) {
1848
1884
  if (directory && directory.trim()) {
1849
1885
  url.searchParams.set("directory", directory.trim());
1850
1886
  }
1851
- const response = await fetch(url, {
1887
+ const response = await timedFetch(url, {
1852
1888
  method: "POST",
1853
1889
  headers: { "Content-Type": "application/json" },
1854
1890
  body: JSON.stringify({})
@@ -1862,7 +1898,7 @@ async function createOpenCodeSession(port, directory) {
1862
1898
  }
1863
1899
  async function getModelAttachmentCapability(port, model) {
1864
1900
  try {
1865
- const res = await fetch(`${opencodeBase(port)}/config/providers`);
1901
+ const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
1866
1902
  if (!res.ok) {
1867
1903
  console.error(
1868
1904
  `[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
@@ -1995,7 +2031,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
1995
2031
  };
1996
2032
  }
1997
2033
  }
1998
- const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
2034
+ const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
1999
2035
  method: "POST",
2000
2036
  headers: { "Content-Type": "application/json" },
2001
2037
  body: JSON.stringify(body)
@@ -2244,7 +2280,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
2244
2280
  }
2245
2281
  async function hasAnyConfiguredProvider(port) {
2246
2282
  try {
2247
- const res = await fetch(`${opencodeBase(port)}/config/providers`);
2283
+ const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
2248
2284
  if (!res.ok) {
2249
2285
  console.error(
2250
2286
  `[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
@@ -2952,6 +2988,21 @@ function writeTunnelReadyMarker(path, agentId) {
2952
2988
  }
2953
2989
  }
2954
2990
 
2991
+ // src/lib/reporting-schedule.ts
2992
+ function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
2993
+ const jitterRangeMs = baseMs * jitterFraction;
2994
+ return baseMs - jitterRangeMs + random() * (2 * jitterRangeMs);
2995
+ }
2996
+ function firstReportDelayMs(random = Math.random) {
2997
+ return 5e3 + random() * 1e4;
2998
+ }
2999
+ function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
3000
+ return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
3001
+ }
3002
+ function failureStreakSuffix(consecutiveFailures) {
3003
+ return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
3004
+ }
3005
+
2955
3006
  // src/lib/claude-usage-reporting.ts
2956
3007
  var VALID_MODES = ["auto", "on", "off"];
2957
3008
  function resolveClaudeUsageReportingMode(flagValue, env) {
@@ -2974,18 +3025,79 @@ function resolveClaudeUsageReportingMode(flagValue, env) {
2974
3025
  var BASE_REPORT_DELAY_MS = 10 * 6e4;
2975
3026
  var REPORT_DELAY_JITTER_FRACTION = 0.2;
2976
3027
  function nextReportDelayMs(random = Math.random) {
2977
- const jitterRangeMs = BASE_REPORT_DELAY_MS * REPORT_DELAY_JITTER_FRACTION;
2978
- return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
3028
+ return jitteredDelayMs(BASE_REPORT_DELAY_MS, REPORT_DELAY_JITTER_FRACTION, random);
2979
3029
  }
2980
- var FIRST_REPORT_DELAY_MS = 5e3 + Math.random() * 1e4;
3030
+ var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
2981
3031
  var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
2982
3032
  function claudeUsageFailureLogLevel(consecutiveFailures) {
2983
- return consecutiveFailures === 1 || consecutiveFailures % CLAUDE_USAGE_FAILURE_REESCALATION_TICKS === 0 ? "warn" : "debug";
3033
+ return reportFailureLogLevel(consecutiveFailures, CLAUDE_USAGE_FAILURE_REESCALATION_TICKS);
3034
+ }
3035
+
3036
+ // src/lib/resource-usage-reporting.ts
3037
+ var ENABLED_VALUES = /* @__PURE__ */ new Set(["on", "true", "1"]);
3038
+ var DISABLED_VALUES = /* @__PURE__ */ new Set(["off", "false", "0"]);
3039
+ function resolveResourceUsageReportingEnabled(flagValue, env) {
3040
+ if (flagValue === false) {
3041
+ return { enabled: false, warnings: [] };
3042
+ }
3043
+ const raw = env.EVIDENT_RESOURCE_USAGE_REPORTING;
3044
+ if (raw === void 0 || raw === "") {
3045
+ return { enabled: true, warnings: [] };
3046
+ }
3047
+ const normalized = raw.trim().toLowerCase();
3048
+ if (DISABLED_VALUES.has(normalized)) {
3049
+ return { enabled: false, warnings: [] };
3050
+ }
3051
+ if (ENABLED_VALUES.has(normalized)) {
3052
+ return { enabled: true, warnings: [] };
3053
+ }
3054
+ return {
3055
+ enabled: true,
3056
+ warnings: [
3057
+ `Ignoring invalid EVIDENT_RESOURCE_USAGE_REPORTING "${raw}": expected on or off; leaving reporting on`
3058
+ ]
3059
+ };
3060
+ }
3061
+
3062
+ // src/lib/resource-usage.ts
3063
+ import { cpus, totalmem, freemem } from "os";
3064
+ function readCpuSample() {
3065
+ let busyMs = 0;
3066
+ let idleMs = 0;
3067
+ for (const cpu of cpus()) {
3068
+ busyMs += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.irq;
3069
+ idleMs += cpu.times.idle;
3070
+ }
3071
+ return { busyMs, idleMs };
3072
+ }
3073
+ function cpuPercentBetween(previous, current) {
3074
+ const deltaBusy = current.busyMs - previous.busyMs;
3075
+ const deltaIdle = current.idleMs - previous.idleMs;
3076
+ const total = deltaBusy + deltaIdle;
3077
+ if (total === 0) return null;
3078
+ return Math.round((deltaBusy / total * 100 + Number.EPSILON) * 100) / 100;
3079
+ }
3080
+ function createResourceUsageCollector() {
3081
+ let previous = readCpuSample();
3082
+ return () => {
3083
+ const current = readCpuSample();
3084
+ const cpuPercent = cpuPercentBetween(previous, current);
3085
+ previous = current;
3086
+ return {
3087
+ cpuPercent,
3088
+ cpuCount: cpus().length,
3089
+ memoryTotalBytes: totalmem(),
3090
+ memoryAvailableBytes: freemem()
3091
+ };
3092
+ };
2984
3093
  }
2985
3094
 
2986
3095
  // src/lib/channels/driver.ts
2987
3096
  import { homedir as homedir2 } from "os";
2988
3097
 
3098
+ // src/lib/runner-file-sync.ts
3099
+ import { join as join4 } from "path";
3100
+
2989
3101
  // src/lib/file-push.ts
2990
3102
  import { randomUUID } from "crypto";
2991
3103
  import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
@@ -3179,17 +3291,20 @@ async function syncPendingRunnerFiles(options) {
3179
3291
  for (const id of options.ackFailures.keys()) {
3180
3292
  if (!pendingIds.has(id)) options.ackFailures.delete(id);
3181
3293
  }
3182
- if (pending.length === 0) return 0;
3294
+ if (pending.length === 0) return { applied: 0, claudeCredentialApplied: false };
3183
3295
  options.log({
3184
3296
  level: "info",
3185
3297
  message: `Runner file sync: ${pending.length} file(s) queued for this runner`
3186
3298
  });
3187
3299
  let applied = 0;
3300
+ let claudeCredentialApplied = false;
3188
3301
  for (const file of pending) {
3189
3302
  if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
3190
- if (await applyOne(options, file)) applied += 1;
3303
+ const outcome = await applyOne(options, file);
3304
+ if (outcome.applied) applied += 1;
3305
+ if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
3191
3306
  }
3192
- return applied;
3307
+ return { applied, claudeCredentialApplied };
3193
3308
  }
3194
3309
  async function listPendingFiles(options) {
3195
3310
  let res;
@@ -3250,6 +3365,11 @@ function asPendingFile(entry) {
3250
3365
  if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
3251
3366
  return { id, path, size };
3252
3367
  }
3368
+ var NOT_APPLIED = { applied: false, claudeCredentialApplied: false };
3369
+ function isClaudeCredentialPath(requestedPath, homeDir) {
3370
+ const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join4(homeDir, requestedPath.slice(2)) : requestedPath;
3371
+ return expanded === join4(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
3372
+ }
3253
3373
  async function applyOne(options, file) {
3254
3374
  const label = `${file.id.slice(0, 8)} (${file.path})`;
3255
3375
  if (options.allowedDirectories.length === 0) {
@@ -3258,7 +3378,7 @@ async function applyOne(options, file) {
3258
3378
  message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
3259
3379
  });
3260
3380
  await ack(options, file, "rejected", "file_sync_disabled");
3261
- return false;
3381
+ return NOT_APPLIED;
3262
3382
  }
3263
3383
  if (file.size > MAX_FILE_PUSH_BYTES) {
3264
3384
  options.log({
@@ -3266,12 +3386,12 @@ async function applyOne(options, file) {
3266
3386
  message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
3267
3387
  });
3268
3388
  await ack(options, file, "rejected", "file_too_large");
3269
- return false;
3389
+ return NOT_APPLIED;
3270
3390
  }
3271
3391
  const download = await downloadContent(options, file, label);
3272
3392
  if (!download.ok) {
3273
3393
  if (download.terminal) await ack(options, file, "rejected", download.code);
3274
- return false;
3394
+ return NOT_APPLIED;
3275
3395
  }
3276
3396
  let outcome;
3277
3397
  try {
@@ -3287,7 +3407,7 @@ async function applyOne(options, file) {
3287
3407
  message: `Runner file ${label} could not be written: ${describe(err)}`
3288
3408
  });
3289
3409
  await ack(options, file, "rejected", "write_failed");
3290
- return false;
3410
+ return NOT_APPLIED;
3291
3411
  }
3292
3412
  if (!outcome.ok) {
3293
3413
  options.log({
@@ -3295,14 +3415,17 @@ async function applyOne(options, file) {
3295
3415
  message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
3296
3416
  });
3297
3417
  await ack(options, file, "rejected", outcome.code);
3298
- return false;
3418
+ return NOT_APPLIED;
3299
3419
  }
3300
3420
  options.log({
3301
3421
  level: "info",
3302
3422
  message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
3303
3423
  });
3304
3424
  await ack(options, file, "applied");
3305
- return true;
3425
+ return {
3426
+ applied: true,
3427
+ claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir)
3428
+ };
3306
3429
  }
3307
3430
  function durableDownloadCode(status2) {
3308
3431
  return status2 === 413 ? "file_too_large" : "write_failed";
@@ -3413,8 +3536,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
3413
3536
  var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
3414
3537
  var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
3415
3538
  var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
3539
+ var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
3540
+ var MAX_WATCHER_STALL_RESTARTS = 3;
3541
+ var MAX_RELEASED_OPENCODE_IDS = 256;
3416
3542
  var MAX_SUPERSEDED_CONVERSATIONS = 256;
3417
3543
  var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
3544
+ var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
3545
+ var MAX_WEDGED_CONVERSATIONS = 256;
3418
3546
  var ChannelAuthError = class extends Error {
3419
3547
  constructor(message) {
3420
3548
  super(message);
@@ -3458,6 +3586,8 @@ var ChannelDriver = class _ChannelDriver {
3458
3586
  fileSyncDirectories;
3459
3587
  homeDir;
3460
3588
  maxActiveSessions;
3589
+ watcherStallMs;
3590
+ wedgeWarningIntervalMs;
3461
3591
  /** Cache of conversationId → opencode sessionId. */
3462
3592
  sessions = /* @__PURE__ */ new Map();
3463
3593
  /**
@@ -3488,6 +3618,40 @@ var ChannelDriver = class _ChannelDriver {
3488
3618
  * bounded cost.
3489
3619
  */
3490
3620
  supersededSessions = /* @__PURE__ */ new Map();
3621
+ /**
3622
+ * Local re-drive fence for a message force-released by the stall watchdog
3623
+ * (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
3624
+ * see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
3625
+ * is `null` for exactly this shape (its `markProcessing` never landed), so
3626
+ * without a local record of the id the driver last knew, the next drain's
3627
+ * `if (message.opencode_message_id)` re-drive-fence check at
3628
+ * `processConversation` would not engage and it would blind-`prompt_async`
3629
+ * a turn that may still be running in opencode — the one duplicate-turn
3630
+ * hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
3631
+ * `processConversation` reads `message.opencode_message_id ?? this
3632
+ * .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
3633
+ * threads it into `resolveRedrive`, which asks opencode itself whether the
3634
+ * turn is still ongoing before ever dispatching.
3635
+ *
3636
+ * Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
3637
+ * `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
3638
+ * non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
3639
+ * dispatch) and at the top-level fresh-dispatch site, so it does not outlive
3640
+ * the row it was recorded for.
3641
+ */
3642
+ releasedOpencodeIds = /* @__PURE__ */ new Map();
3643
+ /**
3644
+ * Per-conversation throttle state for the #183 recurrence warning (#1618
3645
+ * WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
3646
+ * `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
3647
+ * log line and the `dispatch_wedged` signal to at most once per
3648
+ * `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
3649
+ * so the operator sees magnitude, not repetition. Cleared the moment the
3650
+ * conversation dispatches anything (a fresh wedge, if it recurs, is a new
3651
+ * incident). Bounded FIFO, mirroring `supersededSessions`
3652
+ * (`MAX_WEDGED_CONVERSATIONS`).
3653
+ */
3654
+ wedgeWarnings = /* @__PURE__ */ new Map();
3491
3655
  /**
3492
3656
  * Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
3493
3657
  * longer idempotent (no caller-supplied `messageID`), and its read-back picks
@@ -3706,6 +3870,14 @@ var ChannelDriver = class _ChannelDriver {
3706
3870
  * same trick `lastProxiedActivityAt` uses.
3707
3871
  */
3708
3872
  appliedFileCount = 0;
3873
+ /**
3874
+ * Generation counter, NOT a tally (#1656): advances by exactly one per sync
3875
+ * batch that applied the Claude CLI credential file, not by how many
3876
+ * credential files were in that batch. `run.ts` only ever tests inequality
3877
+ * against the value it saw last cycle, so magnitude is meaningless — keep it
3878
+ * that way rather than "fixing" it into a count.
3879
+ */
3880
+ claudeCredentialApplyCount = 0;
3709
3881
  /**
3710
3882
  * The currently-executing `drainPending()` promise, or null when idle. Lets a
3711
3883
  * graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
@@ -3730,7 +3902,10 @@ var ChannelDriver = class _ChannelDriver {
3730
3902
  this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
3731
3903
  this.log = config.log ?? (() => {
3732
3904
  });
3733
- this.fetchImpl = config.fetchImpl ?? fetch;
3905
+ this.fetchImpl = withRequestTimeout(
3906
+ config.fetchImpl ?? fetch,
3907
+ config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
3908
+ );
3734
3909
  this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
3735
3910
  this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
3736
3911
  this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
@@ -3739,6 +3914,8 @@ var ChannelDriver = class _ChannelDriver {
3739
3914
  this.fileSyncDirectories = config.fileSyncDirectories ?? [];
3740
3915
  this.homeDir = config.homeDir ?? homedir2();
3741
3916
  this.maxActiveSessions = config.maxActiveSessions;
3917
+ this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
3918
+ this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
3742
3919
  }
3743
3920
  /** The IPv4-loopback base URL for the local `opencode serve`. */
3744
3921
  get opencodeBase() {
@@ -3752,6 +3929,14 @@ var ChannelDriver = class _ChannelDriver {
3752
3929
  * @returns the number of messages NEWLY dispatched to opencode's native queue.
3753
3930
  */
3754
3931
  async drainPending() {
3932
+ try {
3933
+ this.reconcileWatchers();
3934
+ } catch (err) {
3935
+ this.log({
3936
+ level: "error",
3937
+ message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
3938
+ });
3939
+ }
3755
3940
  if (this.stopped) return 0;
3756
3941
  if (this.draining) return 0;
3757
3942
  this.draining = true;
@@ -3785,7 +3970,7 @@ var ChannelDriver = class _ChannelDriver {
3785
3970
  if (this.syncingFiles) return 0;
3786
3971
  this.syncingFiles = true;
3787
3972
  try {
3788
- const applied = await syncPendingRunnerFiles({
3973
+ const result = await syncPendingRunnerFiles({
3789
3974
  agentId: this.agentId,
3790
3975
  apiUrl: this.apiUrl,
3791
3976
  getAuthHeader: this.getAuthHeader,
@@ -3795,8 +3980,9 @@ var ChannelDriver = class _ChannelDriver {
3795
3980
  ackFailures: this.fileAckFailures,
3796
3981
  log: this.log
3797
3982
  });
3798
- this.appliedFileCount += applied;
3799
- return applied;
3983
+ this.appliedFileCount += result.applied;
3984
+ if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
3985
+ return result.applied;
3800
3986
  } catch (err) {
3801
3987
  this.log({
3802
3988
  level: "error",
@@ -3887,12 +4073,24 @@ var ChannelDriver = class _ChannelDriver {
3887
4073
  * `appliedFiles` is monotonic so a pull that started AND finished between two
3888
4074
  * idle checks still shows up as an advance.
3889
4075
  *
4076
+ * A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
4077
+ * (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
4078
+ * reporting on it advancing, so an unrelated file sync can never disturb a
4079
+ * healthy reporting cadence (#1627) — it never even reaches that trigger, let
4080
+ * alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
4081
+ * whose consumer is idle-timeout suppression and must key on ANY file, not
4082
+ * just a Claude credential.
4083
+ *
3890
4084
  * CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
3891
4085
  * the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
3892
4086
  * samples afterwards reads `true` every single cycle and can never idle out.
3893
4087
  */
3894
4088
  fileSyncActivity() {
3895
- return { appliedFiles: this.appliedFileCount, inFlight: this.syncingFiles };
4089
+ return {
4090
+ appliedFiles: this.appliedFileCount,
4091
+ inFlight: this.syncingFiles,
4092
+ claudeCredentialApplies: this.claudeCredentialApplyCount
4093
+ };
3896
4094
  }
3897
4095
  /**
3898
4096
  * OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
@@ -4005,8 +4203,15 @@ var ChannelDriver = class _ChannelDriver {
4005
4203
  skippedAlreadyDispatched += 1;
4006
4204
  continue;
4007
4205
  }
4008
- if (message.opencode_message_id) {
4009
- const outcome = await this.resolveRedrive(conv, sessionId, message, sessionCreated);
4206
+ const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
4207
+ if (effectiveOpencodeMessageId) {
4208
+ const outcome = await this.resolveRedrive(
4209
+ conv,
4210
+ sessionId,
4211
+ message,
4212
+ sessionCreated,
4213
+ effectiveOpencodeMessageId
4214
+ );
4010
4215
  if (outcome === "abandoned") {
4011
4216
  continue;
4012
4217
  }
@@ -4117,21 +4322,102 @@ var ChannelDriver = class _ChannelDriver {
4117
4322
  }
4118
4323
  this.unconfirmedDispatchFailures.delete(message.id);
4119
4324
  this.dispatchNotStartedSignalled.delete(message.id);
4325
+ this.releasedOpencodeIds.delete(message.id);
4120
4326
  this.dispatched.add(message.id);
4121
4327
  this.registerInFlight(conv, sessionId, message, opencodeMessageId);
4122
4328
  dispatched += 1;
4123
4329
  void this.postSignal(conv.id, message.id, "dispatched");
4124
4330
  }
4125
4331
  if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
4126
- this.log({
4127
- level: "warn",
4128
- message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
4129
- conversation_id: conv.id
4130
- });
4332
+ this.reportWedgedConversation(conv, messages);
4333
+ } else if (dispatched > 0) {
4334
+ this.wedgeWarnings.delete(conv.id);
4131
4335
  }
4132
4336
  this.ensureWatcherRunning(sessionId);
4133
4337
  return dispatched;
4134
4338
  }
4339
+ /**
4340
+ * The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
4341
+ * escalated (#1618 WI-4). `messages` is the conversation's full pending list
4342
+ * on THIS tick — the caller has already confirmed every one of them is a
4343
+ * skip-because-already-`dispatched`, the exact signature of a message stuck
4344
+ * acknowledged-but-never-worked.
4345
+ *
4346
+ * Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
4347
+ * (52,843 occurrences observed in one incident) — burning the GLOBAL
4348
+ * 30-events/60s `runner-activity-telemetry.ts` budget that was itself
4349
+ * suppressing the diagnostics needed to debug the wedge. The `warn` log (and
4350
+ * the `dispatch_wedged` signal once the wedge has persisted past the same
4351
+ * interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
4352
+ * naming the consecutive-tick count so the operator sees magnitude rather
4353
+ * than repetition.
4354
+ *
4355
+ * Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
4356
+ * unconditionally on this same tick and is already recovering anything it
4357
+ * can see. This is reporting only — see `countUntrackedIds`'s doc for the
4358
+ * one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
4359
+ */
4360
+ reportWedgedConversation(conv, messages) {
4361
+ const now = this.now();
4362
+ const existing = this.wedgeWarnings.get(conv.id);
4363
+ const firstWedgedAt = existing?.firstWedgedAt ?? now;
4364
+ const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
4365
+ const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
4366
+ if (!dueForWarn) {
4367
+ this.wedgeWarnings.delete(conv.id);
4368
+ this.wedgeWarnings.set(conv.id, {
4369
+ firstWedgedAt,
4370
+ lastWarnedAt: existing.lastWarnedAt,
4371
+ consecutiveTicks
4372
+ });
4373
+ return;
4374
+ }
4375
+ const stuckForMs = now - firstWedgedAt;
4376
+ const untracked = this.countUntrackedIds(messages);
4377
+ this.log({
4378
+ level: "warn",
4379
+ message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
4380
+ conversation_id: conv.id
4381
+ });
4382
+ this.wedgeWarnings.delete(conv.id);
4383
+ this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
4384
+ while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
4385
+ const oldest = this.wedgeWarnings.keys().next().value;
4386
+ if (oldest === void 0) break;
4387
+ this.wedgeWarnings.delete(oldest);
4388
+ }
4389
+ if (stuckForMs >= this.wedgeWarningIntervalMs) {
4390
+ void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
4391
+ stuck_for_ms: stuckForMs,
4392
+ untracked
4393
+ });
4394
+ }
4395
+ }
4396
+ /**
4397
+ * How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
4398
+ * WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
4399
+ * `dispatched`/`inFlight` pairing invariant holds by construction across
4400
+ * every `dispatched.add` site (see its own doc comment), so `> 0` here means
4401
+ * that invariant has actually been violated for this conversation: there is
4402
+ * no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
4403
+ * `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
4404
+ * recovering. One pass over `this.watchers`, called only when the throttled
4405
+ * warning above is due to fire — not every tick.
4406
+ */
4407
+ countUntrackedIds(messages) {
4408
+ let untracked = 0;
4409
+ for (const message of messages) {
4410
+ let tracked = false;
4411
+ for (const watcher of this.watchers.values()) {
4412
+ if (watcher.inFlight.has(message.id)) {
4413
+ tracked = true;
4414
+ break;
4415
+ }
4416
+ }
4417
+ if (!tracked) untracked += 1;
4418
+ }
4419
+ return untracked;
4420
+ }
4135
4421
  /**
4136
4422
  * Poll a session's message list for the re-drive fence (#965), via the
4137
4423
  * INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
@@ -4200,9 +4486,16 @@ var ChannelDriver = class _ChannelDriver {
4200
4486
  * for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
4201
4487
  * other failure resolves to `unresolved` and is retried whole on the next
4202
4488
  * ~2s drain tick.
4489
+ *
4490
+ * `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
4491
+ * server `opencode_message_id` when present, else the stall watchdog's local
4492
+ * `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
4493
+ * `message` so every line below — and the signals this method posts —
4494
+ * keeps reporting the REAL server row; a shadow-copied `message` would
4495
+ * silently diverge from it.
4203
4496
  */
4204
- async resolveRedrive(conv, sessionId, message, sessionCreated) {
4205
- const ocId = message.opencode_message_id ?? null;
4497
+ async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
4498
+ const ocId = effectiveOpencodeMessageId;
4206
4499
  if (sessionCreated) {
4207
4500
  this.clearRedriveUnresolved(message.id);
4208
4501
  void this.postSignal(conv.id, message.id, "redrive_redispatched");
@@ -4432,7 +4725,13 @@ var ChannelDriver = class _ChannelDriver {
4432
4725
  }
4433
4726
  return "unresolved";
4434
4727
  }
4435
- /** Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved` outcome). */
4728
+ /**
4729
+ * Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
4730
+ * outcome) — including the stall watchdog's local re-drive fence (#1618): once
4731
+ * `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
4732
+ * a real server-side `opencode_message_id` again or is no longer pending, so
4733
+ * the fence entry is no longer needed.
4734
+ */
4436
4735
  clearRedriveUnresolved(messageId) {
4437
4736
  this.redriveUnresolvedSince.delete(messageId);
4438
4737
  this.redriveUnresolvedSignalled.delete(messageId);
@@ -4440,6 +4739,7 @@ var ChannelDriver = class _ChannelDriver {
4440
4739
  this.redriveOutcomeUnreportedSignalled.delete(messageId);
4441
4740
  this.redriveOutcomeFailingSince.delete(messageId);
4442
4741
  this.redriveOutcomeAbandonedSignalled.delete(messageId);
4742
+ this.releasedOpencodeIds.delete(messageId);
4443
4743
  }
4444
4744
  /**
4445
4745
  * #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
@@ -4594,6 +4894,21 @@ var ChannelDriver = class _ChannelDriver {
4594
4894
  this.supersededSessions.delete(oldest);
4595
4895
  }
4596
4896
  }
4897
+ /**
4898
+ * Record the local re-drive fence for a message force-released without
4899
+ * completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
4900
+ * `removeInFlight`, which is about to drop the `InFlightMessage` this reads
4901
+ * `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
4902
+ */
4903
+ recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
4904
+ this.releasedOpencodeIds.delete(evidentMessageId);
4905
+ this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
4906
+ while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
4907
+ const oldest = this.releasedOpencodeIds.keys().next().value;
4908
+ if (oldest === void 0) return;
4909
+ this.releasedOpencodeIds.delete(oldest);
4910
+ }
4911
+ }
4597
4912
  /** Whether `sessionId` is the session this conversation has abandoned (#553). */
4598
4913
  isSuperseded(conversationId, sessionId) {
4599
4914
  return this.supersededSessions.get(conversationId) === sessionId;
@@ -4635,6 +4950,17 @@ var ChannelDriver = class _ChannelDriver {
4635
4950
  message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
4636
4951
  conversation_id: conv.id
4637
4952
  });
4953
+ const watcher = this.watchers.get(bound);
4954
+ if (watcher) {
4955
+ for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
4956
+ this.recordReleasedOpencodeId(evidentMessageId, bound, inFlight.opencodeMessageId);
4957
+ this.removeInFlight(watcher, evidentMessageId);
4958
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
4959
+ recovery: "session_gone_released"
4960
+ });
4961
+ }
4962
+ this.watchers.delete(bound);
4963
+ }
4638
4964
  this.sessions.delete(conv.id);
4639
4965
  return { sessionId: await this.createAndBindSession(conv.id), created: true };
4640
4966
  }
@@ -4825,15 +5151,7 @@ var ChannelDriver = class _ChannelDriver {
4825
5151
  registerInFlight(conv, sessionId, message, opencodeMessageId) {
4826
5152
  let watcher = this.watchers.get(sessionId);
4827
5153
  if (!watcher) {
4828
- watcher = {
4829
- conv,
4830
- inFlight: /* @__PURE__ */ new Map(),
4831
- loop: null,
4832
- reportedQuestions: /* @__PURE__ */ new Set(),
4833
- reportedPermissions: /* @__PURE__ */ new Set(),
4834
- lastGoodPollAt: this.now(),
4835
- hadUsablePoll: false
4836
- };
5154
+ watcher = this.newSessionWatcher(conv);
4837
5155
  this.watchers.set(sessionId, watcher);
4838
5156
  }
4839
5157
  const now = this.now();
@@ -4864,6 +5182,27 @@ var ChannelDriver = class _ChannelDriver {
4864
5182
  ambiguousResolved: false
4865
5183
  });
4866
5184
  }
5185
+ /**
5186
+ * Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
5187
+ * EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
5188
+ * misread as stalled by the very first reconciliation that sees it.
5189
+ */
5190
+ newSessionWatcher(conv) {
5191
+ const now = this.now();
5192
+ return {
5193
+ conv,
5194
+ inFlight: /* @__PURE__ */ new Map(),
5195
+ loop: null,
5196
+ reportedQuestions: /* @__PURE__ */ new Set(),
5197
+ reportedPermissions: /* @__PURE__ */ new Set(),
5198
+ lastGoodPollAt: now,
5199
+ hadUsablePoll: false,
5200
+ generation: 0,
5201
+ lastTickAt: now,
5202
+ lastObservedTickAt: now,
5203
+ consecutiveStallRestarts: 0
5204
+ };
5205
+ }
4867
5206
  /**
4868
5207
  * Register a RE-ADOPTED `processing` message with its session watcher
4869
5208
  * (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
@@ -4893,15 +5232,7 @@ var ChannelDriver = class _ChannelDriver {
4893
5232
  registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
4894
5233
  let watcher = this.watchers.get(sessionId);
4895
5234
  if (!watcher) {
4896
- watcher = {
4897
- conv,
4898
- inFlight: /* @__PURE__ */ new Map(),
4899
- loop: null,
4900
- reportedQuestions: /* @__PURE__ */ new Set(),
4901
- reportedPermissions: /* @__PURE__ */ new Set(),
4902
- lastGoodPollAt: this.now(),
4903
- hadUsablePoll: false
4904
- };
5235
+ watcher = this.newSessionWatcher(conv);
4905
5236
  this.watchers.set(sessionId, watcher);
4906
5237
  }
4907
5238
  watcher.inFlight.set(message.id, {
@@ -4945,12 +5276,110 @@ var ChannelDriver = class _ChannelDriver {
4945
5276
  ambiguousResolved: false
4946
5277
  });
4947
5278
  }
5279
+ /**
5280
+ * Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
5281
+ * restarts any per-session watcher whose loop has exited or stopped ticking
5282
+ * — escalating to a bounded force-release only once
5283
+ * `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
5284
+ * it. Fully synchronous: it only inspects in-memory state and calls the
5285
+ * synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
5286
+ * run from the very top of `drainPending()` — ahead of the un-timed
5287
+ * `getPendingConversations()` await that would otherwise be able to disable
5288
+ * it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
5289
+ * `drainPending()` from being CALLED again at all, not just from finishing).
5290
+ *
5291
+ * Restarts the loop rather than releasing messages directly: a blind release
5292
+ * would let the next drain re-`prompt_async` a turn that may still be
5293
+ * running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
5294
+ * re-polls with each message's `opencodeMessageId` still in hand and lets
5295
+ * the existing, audited `!activelyRunning` give-up decide, same as it always
5296
+ * has.
5297
+ *
5298
+ * Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
5299
+ * that shape has no in-flight entry and therefore no `opencodeMessageId` to
5300
+ * fence a release with, so releasing it here would blind-re-POST a possibly-
5301
+ * running turn — and there is no conversation id in hand to signal with
5302
+ * either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
5303
+ * instead, where a conversation id already exists. If you find yourself
5304
+ * wanting to add a `dispatched` sweep here, don't — read the drain-wedge
5305
+ * plan's §3/D5 first.
5306
+ */
5307
+ reconcileWatchers() {
5308
+ const now = this.now();
5309
+ for (const [sessionId, watcher] of [...this.watchers]) {
5310
+ if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
5311
+ watcher.consecutiveStallRestarts = 0;
5312
+ }
5313
+ watcher.lastObservedTickAt = watcher.lastTickAt;
5314
+ if (watcher.inFlight.size === 0 && watcher.loop === null) {
5315
+ this.watchers.delete(sessionId);
5316
+ continue;
5317
+ }
5318
+ if (watcher.loop === null && watcher.inFlight.size > 0) {
5319
+ if (now - watcher.lastTickAt < this.watcherStallMs) continue;
5320
+ this.log({
5321
+ level: "warn",
5322
+ message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
5323
+ conversation_id: watcher.conv.id
5324
+ });
5325
+ this.ensureWatcherRunning(sessionId);
5326
+ for (const evidentMessageId of watcher.inFlight.keys()) {
5327
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
5328
+ recovery: "loop_exited"
5329
+ });
5330
+ }
5331
+ continue;
5332
+ }
5333
+ if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
5334
+ const stalledForMs = now - watcher.lastTickAt;
5335
+ watcher.consecutiveStallRestarts += 1;
5336
+ if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
5337
+ this.log({
5338
+ level: "error",
5339
+ message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
5340
+ conversation_id: watcher.conv.id
5341
+ });
5342
+ for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
5343
+ this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
5344
+ this.removeInFlight(watcher, evidentMessageId);
5345
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
5346
+ recovery: "unrecoverable_released"
5347
+ });
5348
+ }
5349
+ watcher.generation += 1;
5350
+ this.watchers.delete(sessionId);
5351
+ continue;
5352
+ }
5353
+ watcher.generation += 1;
5354
+ watcher.loop = null;
5355
+ watcher.lastGoodPollAt = now;
5356
+ watcher.lastTickAt = now;
5357
+ watcher.lastObservedTickAt = watcher.lastTickAt;
5358
+ this.ensureWatcherRunning(sessionId);
5359
+ this.log({
5360
+ level: "warn",
5361
+ message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
5362
+ conversation_id: watcher.conv.id
5363
+ });
5364
+ for (const evidentMessageId of watcher.inFlight.keys()) {
5365
+ void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
5366
+ recovery: "loop_stalled"
5367
+ });
5368
+ }
5369
+ }
5370
+ }
5371
+ }
4948
5372
  /**
4949
5373
  * Start (but do NOT await) the per-session watcher loop if it has in-flight
4950
5374
  * work and is not already running. Single-flight per session. The loop is
4951
5375
  * tracked on the watcher and cleared when it settles; it never rejects (fully
4952
5376
  * guarded), so a failed poll/callback can never crash the run loop — the cron
4953
5377
  * stays as the safety net.
5378
+ *
5379
+ * The generation started here (#1618) is captured in the `.finally` closure
5380
+ * so a RETIRED loop settling late — after `reconcileWatchers` has already
5381
+ * restarted this watcher under a newer generation — can neither null the new
5382
+ * loop's handle nor delete a watcher that still has live work.
4954
5383
  */
4955
5384
  ensureWatcherRunning(sessionId) {
4956
5385
  const watcher = this.watchers.get(sessionId);
@@ -4960,7 +5389,9 @@ var ChannelDriver = class _ChannelDriver {
4960
5389
  this.watchers.delete(sessionId);
4961
5390
  return;
4962
5391
  }
4963
- const loop = this.runWatcherLoop(sessionId, watcher).finally(() => {
5392
+ const generation = watcher.generation;
5393
+ const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
5394
+ if (watcher.generation !== generation) return;
4964
5395
  watcher.loop = null;
4965
5396
  if (watcher.inFlight.size === 0) {
4966
5397
  this.watchers.delete(sessionId);
@@ -4980,11 +5411,25 @@ var ChannelDriver = class _ChannelDriver {
4980
5411
  * `source_message_id`;
4981
5412
  * 4. drops messages that completed or timed out from the in-flight set.
4982
5413
  * Exits when the in-flight set empties. Never throws.
5414
+ *
5415
+ * `generation` (#1618) is the incarnation this call was started under.
5416
+ * `reconcileWatchers` can restart a stalled loop by bumping
5417
+ * `watcher.generation` and starting a NEW `runWatcherLoop` over the same
5418
+ * `SessionWatcher` object — the stalled promise itself cannot be cancelled,
5419
+ * so this loop instead checks at the top of every iteration, right after
5420
+ * waking from `sleep`, and right before servicing any message, and quietly
5421
+ * retires (returns without touching anything) the moment it is no longer the
5422
+ * watcher's current generation. Retiring mid-tick can still let ONE
5423
+ * `serviceInFlightMessage` pass complete first — acceptable, since that
5424
+ * method contains no non-idempotent action.
4983
5425
  */
4984
- async runWatcherLoop(sessionId, watcher) {
5426
+ async runWatcherLoop(sessionId, watcher, generation) {
4985
5427
  try {
4986
5428
  while (watcher.inFlight.size > 0) {
5429
+ if (watcher.generation !== generation) return;
5430
+ watcher.lastTickAt = this.now();
4987
5431
  await this.sleep(this.pausedPollIntervalMs);
5432
+ if (watcher.generation !== generation) return;
4988
5433
  let messages = null;
4989
5434
  try {
4990
5435
  const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
@@ -5005,6 +5450,7 @@ var ChannelDriver = class _ChannelDriver {
5005
5450
  }
5006
5451
  }
5007
5452
  const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
5453
+ if (watcher.generation !== generation) return;
5008
5454
  for (const inFlight of [...watcher.inFlight.values()]) {
5009
5455
  await this.serviceInFlightMessage(
5010
5456
  sessionId,
@@ -6980,7 +7426,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
6980
7426
  if (trimmed === "") {
6981
7427
  throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
6982
7428
  }
6983
- const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join4(homeDir, trimmed.slice(2)) : trimmed;
7429
+ const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join5(homeDir, trimmed.slice(2)) : trimmed;
6984
7430
  if (!isAbsolute2(expanded)) {
6985
7431
  throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
6986
7432
  }
@@ -7187,6 +7633,7 @@ async function driveChannels(state, driver) {
7187
7633
  let unreachableMs = 0;
7188
7634
  let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
7189
7635
  let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
7636
+ let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
7190
7637
  while (state.running) {
7191
7638
  const cycleStartedAtMs = performance.now();
7192
7639
  let idleThisCycle = false;
@@ -7210,11 +7657,15 @@ async function driveChannels(state, driver) {
7210
7657
  state.messageCount += processed;
7211
7658
  const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
7212
7659
  lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
7213
- const appliedFiles = driver.fileSyncActivity().appliedFiles;
7660
+ const fileActivitySnapshot = driver.fileSyncActivity();
7661
+ const appliedFiles = fileActivitySnapshot.appliedFiles;
7214
7662
  const filesApplied = appliedFiles !== lastSeenAppliedFiles;
7215
7663
  const fileActivity = carriedOverFileSync || filesApplied;
7216
7664
  lastSeenAppliedFiles = appliedFiles;
7217
- if (filesApplied) state.claudeUsageRearm?.();
7665
+ const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
7666
+ const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
7667
+ lastSeenClaudeApplies = claudeCredentialApplies;
7668
+ if (claudeCredentialApplied) state.claudeUsageRearm?.();
7218
7669
  if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
7219
7670
  idlePolls = 0;
7220
7671
  idleMs = 0;
@@ -7283,7 +7734,7 @@ async function driveChannels(state, driver) {
7283
7734
  var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
7284
7735
  var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
7285
7736
  function sessionDbPath() {
7286
- return join4(homedir3(), ".local", "share", "opencode", "opencode.db");
7737
+ return join5(homedir3(), ".local", "share", "opencode", "opencode.db");
7287
7738
  }
7288
7739
  async function runSweep(state, driver, config) {
7289
7740
  const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
@@ -7394,9 +7845,6 @@ function scheduleSessionCleanup(state, driver, options) {
7394
7845
  );
7395
7846
  state.sessionCleanupTimers.push(interval, firstSweep);
7396
7847
  }
7397
- function claudeUsageFailureStreakSuffix(consecutiveFailures) {
7398
- return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
7399
- }
7400
7848
  function scheduleClaudeUsageReporting(state, options) {
7401
7849
  const { mode, warnings } = resolveClaudeUsageReportingMode(
7402
7850
  options.claudeUsageReporting,
@@ -7418,23 +7866,44 @@ function scheduleClaudeUsageReporting(state, options) {
7418
7866
  return null;
7419
7867
  }
7420
7868
  let consecutiveFailures = 0;
7421
- let armed = false;
7869
+ let phase = "dormant";
7422
7870
  let rearmRequested = false;
7871
+ const armProbe = () => {
7872
+ phase = "probe-pending";
7873
+ state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
7874
+ };
7423
7875
  const scheduleNextTick = () => {
7424
- armed = true;
7425
- rearmRequested = false;
7876
+ if (rearmRequested) {
7877
+ rearmRequested = false;
7878
+ armProbe();
7879
+ return;
7880
+ }
7881
+ phase = "steady-pending";
7426
7882
  state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
7427
7883
  };
7428
7884
  const rearm = () => {
7429
- if (armed) {
7430
- rearmRequested = true;
7431
- return;
7885
+ switch (phase) {
7886
+ case "tick-in-flight":
7887
+ rearmRequested = true;
7888
+ return;
7889
+ case "probe-pending":
7890
+ return;
7891
+ case "steady-pending":
7892
+ if (state.claudeUsageTimer) {
7893
+ clearTimeout(state.claudeUsageTimer);
7894
+ state.claudeUsageTimer = null;
7895
+ }
7896
+ rearmRequested = false;
7897
+ armProbe();
7898
+ return;
7899
+ case "dormant":
7900
+ rearmRequested = false;
7901
+ armProbe();
7902
+ return;
7432
7903
  }
7433
- rearmRequested = false;
7434
- armed = true;
7435
- state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
7436
7904
  };
7437
7905
  const tick = async (isProbe) => {
7906
+ phase = "tick-in-flight";
7438
7907
  try {
7439
7908
  const usage = await getClaudeUsage();
7440
7909
  const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
@@ -7457,7 +7926,7 @@ function scheduleClaudeUsageReporting(state, options) {
7457
7926
  logActivity(state, {
7458
7927
  type: "info",
7459
7928
  level: claudeUsageFailureLogLevel(consecutiveFailures),
7460
- message: `Failed to report Claude usage: ${result.error}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
7929
+ message: `Failed to report Claude usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
7461
7930
  });
7462
7931
  }
7463
7932
  scheduleNextTick();
@@ -7476,7 +7945,7 @@ function scheduleClaudeUsageReporting(state, options) {
7476
7945
  level: "debug",
7477
7946
  message: `Claude usage reporting: ${error2.message}`
7478
7947
  });
7479
- armed = false;
7948
+ phase = "dormant";
7480
7949
  if (rearmRequested) rearm();
7481
7950
  } else {
7482
7951
  logActivity(state, {
@@ -7492,16 +7961,92 @@ function scheduleClaudeUsageReporting(state, options) {
7492
7961
  logActivity(state, {
7493
7962
  type: "info",
7494
7963
  level: claudeUsageFailureLogLevel(consecutiveFailures),
7495
- message: `Claude usage reporting failed: ${message}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
7964
+ message: `Claude usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
7496
7965
  });
7497
7966
  scheduleNextTick();
7498
7967
  }
7499
7968
  }
7500
7969
  };
7501
- armed = true;
7502
- state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
7970
+ armProbe();
7503
7971
  return rearm;
7504
7972
  }
7973
+ var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
7974
+ var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
7975
+ var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
7976
+ function scheduleResourceUsageReporting(state, options) {
7977
+ const { enabled, warnings } = resolveResourceUsageReportingEnabled(
7978
+ options.resourceUsageReporting,
7979
+ process.env
7980
+ );
7981
+ for (const warning2 of warnings) {
7982
+ logActivity(state, {
7983
+ type: "info",
7984
+ level: "warn",
7985
+ message: `Resource usage reporting: ${warning2}`
7986
+ });
7987
+ }
7988
+ if (!enabled) {
7989
+ logActivity(state, {
7990
+ type: "info",
7991
+ level: "debug",
7992
+ message: "Resource usage reporting is off (--no-resource-usage-reporting)"
7993
+ });
7994
+ return;
7995
+ }
7996
+ const collect = createResourceUsageCollector();
7997
+ let consecutiveFailures = 0;
7998
+ const tick = async () => {
7999
+ try {
8000
+ const usage = collect();
8001
+ const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
8002
+ if (result.ok) {
8003
+ if (consecutiveFailures > 0) {
8004
+ logActivity(state, {
8005
+ type: "info",
8006
+ level: "info",
8007
+ message: "Resource usage reporting recovered"
8008
+ });
8009
+ }
8010
+ consecutiveFailures = 0;
8011
+ logActivity(state, {
8012
+ type: "info",
8013
+ level: "debug",
8014
+ message: "Reported resource usage to Evident"
8015
+ });
8016
+ } else {
8017
+ consecutiveFailures++;
8018
+ logActivity(state, {
8019
+ type: "info",
8020
+ level: reportFailureLogLevel(
8021
+ consecutiveFailures,
8022
+ RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
8023
+ ),
8024
+ message: `Failed to report resource usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
8025
+ });
8026
+ }
8027
+ } catch (error2) {
8028
+ consecutiveFailures++;
8029
+ const message = error2 instanceof Error ? error2.message : String(error2);
8030
+ logActivity(state, {
8031
+ type: "info",
8032
+ level: reportFailureLogLevel(
8033
+ consecutiveFailures,
8034
+ RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
8035
+ ),
8036
+ message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
8037
+ });
8038
+ } finally {
8039
+ state.resourceUsageTimer = setTimeout(
8040
+ () => void tick(),
8041
+ jitteredDelayMs(
8042
+ RESOURCE_USAGE_BASE_REPORT_DELAY_MS,
8043
+ RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION
8044
+ )
8045
+ );
8046
+ }
8047
+ };
8048
+ state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
8049
+ }
7505
8050
  async function notifyOffline(state) {
7506
8051
  if (!state.agentId || !state.authHeader) return;
7507
8052
  if (!state.connected) {
@@ -7542,6 +8087,10 @@ async function cleanup(state, opts = {}) {
7542
8087
  state.claudeUsageTimer = null;
7543
8088
  }
7544
8089
  state.claudeUsageRearm = null;
8090
+ if (state.resourceUsageTimer) {
8091
+ clearTimeout(state.resourceUsageTimer);
8092
+ state.resourceUsageTimer = null;
8093
+ }
7545
8094
  if (opts.graceful && state.channelDriver) {
7546
8095
  state.channelDriver.stop();
7547
8096
  log2(state, "Draining in-flight channel work before shutdown...");
@@ -7624,6 +8173,7 @@ async function run(options) {
7624
8173
  sessionCleanupTimers: [],
7625
8174
  claudeUsageTimer: null,
7626
8175
  claudeUsageRearm: null,
8176
+ resourceUsageTimer: null,
7627
8177
  authHeader: ""
7628
8178
  };
7629
8179
  setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
@@ -8023,6 +8573,7 @@ async function run(options) {
8023
8573
  }
8024
8574
  scheduleSessionCleanup(state, channelDriver, options);
8025
8575
  state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
8576
+ scheduleResourceUsageReporting(state, options);
8026
8577
  if (!interactive || state.json) {
8027
8578
  log2(state, "Driving channel messages...");
8028
8579
  }
@@ -8103,6 +8654,9 @@ program.command("run").description("Connect to Evident and process messages").op
8103
8654
  ).option(
8104
8655
  "--claude-usage-reporting <mode>",
8105
8656
  "Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
8657
+ ).option(
8658
+ "--no-resource-usage-reporting",
8659
+ "Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
8106
8660
  ).option(
8107
8661
  "--enable-file-sync-to <dir>",
8108
8662
  "Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
@@ -8135,6 +8689,9 @@ program.command("run").description("Connect to Evident and process messages").op
8135
8689
  // Raw string — the resolver in run.ts single-sources parsing
8136
8690
  // (resolveClaudeUsageReportingMode).
8137
8691
  claudeUsageReporting: options.claudeUsageReporting,
8692
+ // Raw value — resolution is single-sourced in run.ts's
8693
+ // resolveResourceUsageReportingEnabled.
8694
+ resourceUsageReporting: options.resourceUsageReporting,
8138
8695
  // Raw values — expansion/validation is single-sourced in run.ts's
8139
8696
  // resolveFileSyncDirectories.
8140
8697
  enableFileSyncTo: options.enableFileSyncTo,