@evident-ai/cli 3.1.1-dev.8e7adc6 → 3.1.1-dev.8e82a28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -997,6 +997,9 @@ import { homedir as homedir3 } from "os";
997
997
  import { isAbsolute as isAbsolute2, join as join3, parse, resolve as resolvePath } from "path";
998
998
  import chalk6 from "chalk";
999
999
 
1000
+ // ../../packages/types/src/agents/index.ts
1001
+ var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
1002
+
1000
1003
  // ../../packages/types/src/telemetry/index.ts
1001
1004
  var TelemetryEventTypes = {
1002
1005
  // Agent activity events (shown in web UI activity log)
@@ -2086,6 +2089,21 @@ function messageError(messages, userMessageId) {
2086
2089
  }
2087
2090
  return "The agent run failed.";
2088
2091
  }
2092
+ function isAbortedTerminalReply(messages, userMessageId) {
2093
+ const reply = findLastAssistantReplyFor(messages, userMessageId);
2094
+ const error2 = errorOf(reply);
2095
+ if (error2 == null) return false;
2096
+ if (typeof error2 === "string") return error2.trim() === "Aborted";
2097
+ if (typeof error2 === "object") {
2098
+ const e = error2;
2099
+ if (e.name === "MessageAbortedError") return true;
2100
+ if (e.name === "AbortError") return true;
2101
+ const dataMessage = e.data?.message;
2102
+ const rendered = typeof dataMessage === "string" ? dataMessage : typeof e.message === "string" ? e.message : null;
2103
+ return rendered != null && rendered.trim() === "Aborted";
2104
+ }
2105
+ return false;
2106
+ }
2089
2107
  function messageFailure(messages, userMessageId) {
2090
2108
  const reply = findLastAssistantReplyFor(messages, userMessageId);
2091
2109
  const error2 = errorOf(reply);
@@ -2688,6 +2706,10 @@ function nextReportDelayMs(random = Math.random) {
2688
2706
  return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
2689
2707
  }
2690
2708
  var FIRST_REPORT_DELAY_MS = 5e3 + Math.random() * 1e4;
2709
+ var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
2710
+ function claudeUsageFailureLogLevel(consecutiveFailures) {
2711
+ return consecutiveFailures === 1 || consecutiveFailures % CLAUDE_USAGE_FAILURE_REESCALATION_TICKS === 0 ? "warn" : "debug";
2712
+ }
2691
2713
 
2692
2714
  // src/lib/channels/driver.ts
2693
2715
  import { homedir as homedir2 } from "os";
@@ -3119,6 +3141,7 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
3119
3141
  var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
3120
3142
  var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
3121
3143
  var MAX_SUPERSEDED_CONVERSATIONS = 256;
3144
+ var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
3122
3145
  var ChannelAuthError = class extends Error {
3123
3146
  constructor(message) {
3124
3147
  super(message);
@@ -3141,6 +3164,10 @@ function backoffDelay(attempt, policy) {
3141
3164
  function isRetryableStatus(status2) {
3142
3165
  return status2 === 429 || status2 >= 500 && status2 <= 599;
3143
3166
  }
3167
+ var VOLATILE_BODY_FIELD_PATTERN = /("(?:ref|requestId|request_id|traceId|trace_id)"\s*:\s*)"[^"]*"/gi;
3168
+ function normalizeRedrivePollFailureBody(body) {
3169
+ return body.replace(VOLATILE_BODY_FIELD_PATTERN, '$1"<redacted>"').replace(/\s+/g, " ").trim().slice(0, 200);
3170
+ }
3144
3171
  var ChannelDriver = class _ChannelDriver {
3145
3172
  agentId;
3146
3173
  port;
@@ -3254,6 +3281,43 @@ var ChannelDriver = class _ChannelDriver {
3254
3281
  * processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
3255
3282
  */
3256
3283
  readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
3284
+ /**
3285
+ * "Already emitted `redrive_unresolved` for this row" (#965). Mirrors
3286
+ * `readoptPollUnresolvedSignalled`: `resolveRedrive`'s `unresolved` leaf recurs
3287
+ * every ~2s drain until opencode's status becomes readable, but the
3288
+ * server-visible signal is an OUTCOME, so it fires at most once per row. Cleared
3289
+ * on any non-`unresolved` outcome so the set cannot grow beyond the currently
3290
+ * unresolvable rows.
3291
+ */
3292
+ redriveUnresolvedSignalled = /* @__PURE__ */ new Set();
3293
+ /**
3294
+ * First `now()` a `pending` row's re-drive was observed `unresolved` (#965). A
3295
+ * `pending` row is invisible to every cron arm (all require `status =
3296
+ * 'processing'`), so an indefinitely-`unresolved` row would be stranded with
3297
+ * nothing driving it. Once `now - since >= pausedMaxWaitMs`, `resolveRedrive`
3298
+ * takes `dispatch` instead of `unresolved` (reusing the existing knob — see
3299
+ * ADR-0047's own "unreachable ⇒ bounded" rule). Cleared on any other outcome.
3300
+ */
3301
+ redriveUnresolvedSince = /* @__PURE__ */ new Map();
3302
+ /**
3303
+ * Consecutive-identical-poll-failure streak for the re-drive fence (#1348),
3304
+ * keyed by Evident **message id** (not session) so `clearRedriveUnresolved`
3305
+ * can drop it with the other two trackers and it cannot leak. `sessionId` is
3306
+ * carried inside the entry, not the key: a session change is a different
3307
+ * situation and resets the streak, which gives the `(sessionId, message.id)`
3308
+ * pairing #1348 asks for without a composite map key.
3309
+ */
3310
+ redrivePollFailures = /* @__PURE__ */ new Map();
3311
+ /**
3312
+ * "Already emitted `redrive_outcome_unreported` for THIS (message, outcome)
3313
+ * streak" (Class B, #1340: the runner DECIDED reattach/settle/fail_permanent
3314
+ * but its own PATCH to record it failed — distinct from Class A's
3315
+ * `redrive_poll_failed`, where opencode itself can't be observed). Keyed by
3316
+ * message id, valued by the outcome currently failing to report, so a
3317
+ * change of outcome starts a fresh signal. Cleared by
3318
+ * `clearRedriveUnresolved` the instant either PATCH succeeds.
3319
+ */
3320
+ redriveOutcomeUnreportedSignalled = /* @__PURE__ */ new Map();
3257
3321
  /**
3258
3322
  * "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
3259
3323
  * Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
@@ -3593,6 +3657,12 @@ var ChannelDriver = class _ChannelDriver {
3593
3657
  skippedAlreadyDispatched += 1;
3594
3658
  continue;
3595
3659
  }
3660
+ if (message.opencode_message_id) {
3661
+ const outcome = await this.resolveRedrive(conv, sessionId, message, refusedSessionId);
3662
+ if (outcome !== "dispatch") {
3663
+ break;
3664
+ }
3665
+ }
3596
3666
  const options = {
3597
3667
  agent: message.opencode_agent ?? void 0,
3598
3668
  model: message.opencode_model ?? void 0
@@ -3682,6 +3752,338 @@ var ChannelDriver = class _ChannelDriver {
3682
3752
  this.ensureWatcherRunning(sessionId);
3683
3753
  return dispatched;
3684
3754
  }
3755
+ /**
3756
+ * Poll a session's message list for the re-drive fence (#965), via the
3757
+ * INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
3758
+ * hits the global `fetch` and would bypass the same override every other
3759
+ * opencode poll in this file respects. Mirrors `readoptProcessing`'s own
3760
+ * snapshot fetch (`:3081-3111`).
3761
+ *
3762
+ * Returns `{ ok: true, messages }` on a readable snapshot, or
3763
+ * `{ ok: false, signature }` on failure — `signature` is a string that
3764
+ * repeats across attempts for the SAME underlying fault (used by the
3765
+ * consecutive-identical-failure bound, #1348), or `null` for a thrown
3766
+ * exception, which is NOT countable toward that bound (a network blip / an
3767
+ * opencode restart also throws identically every tick, and must keep
3768
+ * retrying unbounded rather than ever being treated as permanent).
3769
+ */
3770
+ async pollSessionMessagesForRedrive(conv, message, sessionId) {
3771
+ try {
3772
+ const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
3773
+ if (!res.ok) {
3774
+ const rawBody = await res.text();
3775
+ const normalized = normalizeRedrivePollFailureBody(rawBody);
3776
+ this.log({
3777
+ level: "warn",
3778
+ message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status}${normalized ? `: ${normalized}` : ""} \u2014 treating as unreadable this tick`,
3779
+ conversation_id: conv.id,
3780
+ message_id: message.id
3781
+ });
3782
+ return { ok: false, signature: `HTTP ${res.status}${normalized ? `: ${normalized}` : ""}` };
3783
+ }
3784
+ const body = await res.json();
3785
+ if (!Array.isArray(body)) {
3786
+ this.log({
3787
+ level: "warn",
3788
+ message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned a non-array message body \u2014 treating as unreadable this tick`,
3789
+ conversation_id: conv.id,
3790
+ message_id: message.id
3791
+ });
3792
+ return { ok: false, signature: "non-array message body" };
3793
+ }
3794
+ return { ok: true, messages: body };
3795
+ } catch (err) {
3796
+ this.log({
3797
+ level: "warn",
3798
+ message: `Re-drive: failed to poll session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
3799
+ conversation_id: conv.id,
3800
+ message_id: message.id
3801
+ });
3802
+ return { ok: false, signature: null };
3803
+ }
3804
+ }
3805
+ /**
3806
+ * The re-drive fence for a `pending` row that already carries a stored
3807
+ * `opencode_message_id` (#965) — i.e. it has already been handed to opencode at
3808
+ * least once (see the invariant at `QueuedMessage.opencode_message_id`'s doc).
3809
+ * The lifecycle cron can falsely reclaim a `processing` row back to `pending`
3810
+ * mid-turn (a 5-minute liveness-staleness check racing a still-running turn);
3811
+ * without this fence the drain loop would re-`prompt_async` the SAME turn a
3812
+ * second time against live GitHub state. Mirrors `readoptOne`'s job for the
3813
+ * `processing` re-adopt path, but simpler: no b1/b2 preamble cross-check is
3814
+ * needed here because `refusedSessionId` already handles the one case
3815
+ * (#553 abandoned session) that path exists for.
3816
+ *
3817
+ * Only `ChannelAuthError` propagates. A poll that fails identically
3818
+ * `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row reports the message
3819
+ * failed instead of retrying it (#1348) — SEPARATE from, not a replacement
3820
+ * for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
3821
+ * other failure resolves to `unresolved` and is retried whole on the next
3822
+ * ~2s drain tick.
3823
+ */
3824
+ async resolveRedrive(conv, sessionId, message, refusedSessionId) {
3825
+ const ocId = message.opencode_message_id ?? null;
3826
+ if (refusedSessionId) {
3827
+ this.clearRedriveUnresolved(message.id);
3828
+ void this.postSignal(conv.id, message.id, "redrive_redispatched");
3829
+ return "dispatch";
3830
+ }
3831
+ const polled = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
3832
+ if (!polled.ok) {
3833
+ const streak = this.recordRedrivePollFailure(message.id, sessionId, polled.signature);
3834
+ if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES && polled.signature !== null) {
3835
+ return this.failRedrivePollPermanent(conv, sessionId, message, polled.signature, streak);
3836
+ }
3837
+ return this.resolveRedriveUnresolved(conv, message);
3838
+ }
3839
+ this.redrivePollFailures.delete(message.id);
3840
+ const messages = polled.messages;
3841
+ if (messages.length === 0) {
3842
+ return this.resolveRedriveUnresolved(conv, message);
3843
+ }
3844
+ const state = messageRunState(messages, ocId ?? "");
3845
+ if (state === "failed" && isAbortedTerminalReply(messages, ocId ?? "")) {
3846
+ const ongoing = await isSessionOngoing(this.port, sessionId);
3847
+ if (ongoing === false) {
3848
+ this.log({
3849
+ level: "info",
3850
+ message: `Re-drive: message ${message.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
3851
+ conversation_id: conv.id,
3852
+ message_id: message.id
3853
+ });
3854
+ this.clearRedriveUnresolved(message.id);
3855
+ void this.postSignal(conv.id, message.id, "redrive_redispatched");
3856
+ return "dispatch";
3857
+ }
3858
+ }
3859
+ if (state === "done" || state === "failed") {
3860
+ return this.settleRedrive(conv, sessionId, message, ocId, messages, state);
3861
+ }
3862
+ if (state === "running" || state === "queued") {
3863
+ const ongoing = await isSessionOngoing(this.port, sessionId);
3864
+ if (ongoing === true) {
3865
+ return this.reattachRedrive(conv, sessionId, message, ocId);
3866
+ }
3867
+ if (ongoing === false) {
3868
+ this.clearRedriveUnresolved(message.id);
3869
+ void this.postSignal(conv.id, message.id, "redrive_redispatched");
3870
+ return "dispatch";
3871
+ }
3872
+ return this.resolveRedriveUnresolved(conv, message);
3873
+ }
3874
+ this.clearRedriveUnresolved(message.id);
3875
+ void this.postSignal(conv.id, message.id, "redrive_redispatched");
3876
+ return "dispatch";
3877
+ }
3878
+ /**
3879
+ * The `reattached` outcome (Task 3.3): the prior turn is STILL ONGOING per
3880
+ * opencode's own status map — undo the false reclaim instead of starting a
3881
+ * second turn.
3882
+ */
3883
+ async reattachRedrive(conv, sessionId, message, ocId) {
3884
+ let anchorMs;
3885
+ const parsed = message.processing_started_at ? Date.parse(message.processing_started_at) : NaN;
3886
+ if (!Number.isNaN(parsed)) {
3887
+ anchorMs = parsed;
3888
+ } else {
3889
+ anchorMs = this.now();
3890
+ this.log({
3891
+ level: "error",
3892
+ message: `Re-drive: message ${message.id.slice(0, 8)} has null/unparseable processing_started_at (${String(message.processing_started_at)}) \u2014 anchoring the watcher's absolute-age ceiling to now (defensive)`,
3893
+ conversation_id: conv.id,
3894
+ message_id: message.id
3895
+ });
3896
+ }
3897
+ const title = await this.resolveSessionTitle(sessionId, conv.id);
3898
+ try {
3899
+ await this.markProcessing(conv.id, message.id, sessionId, ocId, title);
3900
+ } catch (err) {
3901
+ if (err instanceof ChannelAuthError) throw err;
3902
+ this.log({
3903
+ level: "warn",
3904
+ message: `Re-drive: failed to restore message ${message.id.slice(0, 8)} to processing (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
3905
+ conversation_id: conv.id,
3906
+ message_id: message.id
3907
+ });
3908
+ this.signalRedriveOutcomeUnreported(conv, message, "reattach");
3909
+ return "unresolved";
3910
+ }
3911
+ this.clearRedriveUnresolved(message.id);
3912
+ this.registerReadopted(conv, sessionId, message, ocId ?? "", anchorMs);
3913
+ this.dispatched.add(message.id);
3914
+ this.readopted.add(message.id);
3915
+ this.ensureWatcherRunning(sessionId);
3916
+ const watchedForMs = this.now() - anchorMs;
3917
+ void this.postSignal(conv.id, message.id, "redrive_reattached", {
3918
+ watched_for_ms: watchedForMs
3919
+ });
3920
+ this.log({
3921
+ level: "warn",
3922
+ message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) was wrongly reclaimed to pending while its turn was still running (watched ${watchedForMs}ms) \u2014 restored to processing instead of re-dispatching`,
3923
+ conversation_id: conv.id,
3924
+ message_id: message.id
3925
+ });
3926
+ return "reattached";
3927
+ }
3928
+ /**
3929
+ * The `settled` outcome (Task 3.2): the prior turn already finished (or
3930
+ * errored) while nobody was watching — deliver/report it instead of re-running.
3931
+ * Mirrors `readoptOne`'s `done`/`failed` branches' error discipline, simplified
3932
+ * (no `doneUndeliverable` park: a terminal PATCH failure here just retries next
3933
+ * drain, same as any other non-auth failure). The restart-abort carve-out that
3934
+ * keeps the two in step for `failed` lives in the caller (`resolveRedrive`, #1310),
3935
+ * so a row reaching this `failed` branch is a GENUINE failure.
3936
+ */
3937
+ async settleRedrive(conv, sessionId, message, ocId, messages, state) {
3938
+ try {
3939
+ if (state === "done") {
3940
+ const title = await this.resolveSessionTitle(sessionId, conv.id);
3941
+ const usage = messageUsage(messages, ocId ?? "");
3942
+ this.log({
3943
+ level: "info",
3944
+ message: `Re-drive: message ${message.id.slice(0, 8)} completed while its row was wrongly reclaimed to pending \u2014 marking done instead of re-dispatching`,
3945
+ conversation_id: conv.id,
3946
+ message_id: message.id
3947
+ });
3948
+ await this.markDone(conv.id, message.id, sessionId, ocId, title, usage);
3949
+ } else {
3950
+ const error2 = messageError(messages, ocId ?? "") ?? void 0;
3951
+ const usage = messageUsage(messages, ocId ?? "");
3952
+ const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
3953
+ this.log({
3954
+ level: "error",
3955
+ message: `Re-drive: message ${message.id.slice(0, 8)} errored while its row was wrongly reclaimed to pending \u2014 marking failed instead of re-dispatching: ${error2 ?? "(no error text)"}`,
3956
+ conversation_id: conv.id,
3957
+ message_id: message.id
3958
+ });
3959
+ await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
3960
+ }
3961
+ } catch (err) {
3962
+ if (err instanceof ChannelAuthError) throw err;
3963
+ this.log({
3964
+ level: "warn",
3965
+ message: `Re-drive: failed to report message ${message.id.slice(0, 8)} ${state} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
3966
+ conversation_id: conv.id,
3967
+ message_id: message.id
3968
+ });
3969
+ this.signalRedriveOutcomeUnreported(conv, message, "settle");
3970
+ return "unresolved";
3971
+ }
3972
+ this.clearRedriveUnresolved(message.id);
3973
+ void this.postSignal(conv.id, message.id, "redrive_settled");
3974
+ return "settled";
3975
+ }
3976
+ /**
3977
+ * The permanent-failure outcome (#1348): the fence's own poll of this session
3978
+ * failed with the SAME opencode-answered signature
3979
+ * `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row — a transient blip
3980
+ * would have varied or eventually cleared (see `pollSessionMessagesForRedrive`
3981
+ * and `recordRedrivePollFailure`), so this is a durable fault (e.g. #1345's
3982
+ * corrupted opencode session) rather than something worth retrying forever.
3983
+ * Mirrors `settleRedrive`'s error discipline: no `usage`/`failure` args to
3984
+ * `markFailed` (no opencode snapshot to extract them from — this poll never
3985
+ * got a readable one).
3986
+ */
3987
+ async failRedrivePollPermanent(conv, sessionId, message, signature, streak) {
3988
+ this.log({
3989
+ level: "error",
3990
+ message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) failed to poll with the identical signature "${signature}" ${streak} times in a row \u2014 reporting the message failed instead of retrying forever`,
3991
+ conversation_id: conv.id,
3992
+ message_id: message.id
3993
+ });
3994
+ try {
3995
+ await this.markFailed(
3996
+ conv.id,
3997
+ message.id,
3998
+ sessionId,
3999
+ `The runner could not read this conversation's state from OpenCode (${signature}). The same failure repeated ${streak} times in a row, so the message was not retried further.`
4000
+ );
4001
+ } catch (err) {
4002
+ if (err instanceof ChannelAuthError) throw err;
4003
+ this.log({
4004
+ level: "warn",
4005
+ message: `Re-drive: failed to report message ${message.id.slice(0, 8)} permanently failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
4006
+ conversation_id: conv.id,
4007
+ message_id: message.id
4008
+ });
4009
+ this.signalRedriveOutcomeUnreported(conv, message, "fail_permanent");
4010
+ return "unresolved";
4011
+ }
4012
+ this.clearRedriveUnresolved(message.id);
4013
+ void this.postSignal(conv.id, message.id, "redrive_poll_failed");
4014
+ return "settled";
4015
+ }
4016
+ /**
4017
+ * The bounded `unresolved` outcome (Task 3.4): opencode's state could not be
4018
+ * observed (snapshot unreadable/empty, or `isSessionOngoing` returned `null`).
4019
+ * A `pending` row is invisible to every cron arm (all require `status =
4020
+ * 'processing'`), so an indefinitely-`unresolved` row would be stranded with
4021
+ * nothing driving it — bound it to the existing `pausedMaxWaitMs` window
4022
+ * (reusing the knob, not a new constant) and take `dispatch` once elapsed.
4023
+ */
4024
+ resolveRedriveUnresolved(conv, message) {
4025
+ const now = this.now();
4026
+ const since = this.redriveUnresolvedSince.get(message.id);
4027
+ if (since !== void 0 && now - since >= this.pausedMaxWaitMs) {
4028
+ this.clearRedriveUnresolved(message.id);
4029
+ void this.postSignal(conv.id, message.id, "redrive_redispatched");
4030
+ return "dispatch";
4031
+ }
4032
+ if (since === void 0) {
4033
+ this.redriveUnresolvedSince.set(message.id, now);
4034
+ }
4035
+ if (!this.redriveUnresolvedSignalled.has(message.id)) {
4036
+ this.redriveUnresolvedSignalled.add(message.id);
4037
+ void this.postSignal(conv.id, message.id, "redrive_unresolved");
4038
+ }
4039
+ return "unresolved";
4040
+ }
4041
+ /** Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved` outcome). */
4042
+ clearRedriveUnresolved(messageId) {
4043
+ this.redriveUnresolvedSince.delete(messageId);
4044
+ this.redriveUnresolvedSignalled.delete(messageId);
4045
+ this.redrivePollFailures.delete(messageId);
4046
+ this.redriveOutcomeUnreportedSignalled.delete(messageId);
4047
+ }
4048
+ /**
4049
+ * Class B (#1340): the runner DECIDED an outcome (reattach/settle/fail_permanent)
4050
+ * but its own PATCH to record it failed. Deliberately NOT bounded like
4051
+ * `resolveRedriveUnresolved`, for two load-bearing reasons: (1) the only
4052
+ * remedy is a PATCH to the server — the very call that is failing, so a bound
4053
+ * would act by retrying it; (2) this class is self-clearing by construction
4054
+ * (the next successful PATCH resolves it via `clearRedriveUnresolved`),
4055
+ * unlike #1340's deterministic, permanent shape. Fires at most once per
4056
+ * (message, outcome) streak.
4057
+ */
4058
+ signalRedriveOutcomeUnreported(conv, message, outcome) {
4059
+ if (this.redriveOutcomeUnreportedSignalled.get(message.id) === outcome) return;
4060
+ this.redriveOutcomeUnreportedSignalled.set(message.id, outcome);
4061
+ void this.postSignal(conv.id, message.id, "redrive_outcome_unreported", {
4062
+ attempted_outcome: outcome
4063
+ });
4064
+ }
4065
+ /**
4066
+ * Record one poll outcome toward the re-drive fence's consecutive-identical-
4067
+ * failure streak (#1348) and return the resulting count. `signature === null`
4068
+ * (a thrown exception, H1) always clears the streak and returns `0` — it is
4069
+ * never countable. Otherwise the streak continues only when BOTH the session
4070
+ * and the signature match the previous failure; anything else (a different
4071
+ * session, or the same session failing a DIFFERENT way) starts a fresh streak
4072
+ * at `1`.
4073
+ */
4074
+ recordRedrivePollFailure(messageId, sessionId, signature) {
4075
+ if (signature === null) {
4076
+ this.redrivePollFailures.delete(messageId);
4077
+ return 0;
4078
+ }
4079
+ const existing = this.redrivePollFailures.get(messageId);
4080
+ if (existing && existing.sessionId === sessionId && existing.signature === signature) {
4081
+ existing.count += 1;
4082
+ return existing.count;
4083
+ }
4084
+ this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
4085
+ return 1;
4086
+ }
3685
4087
  /**
3686
4088
  * Record that `sessionId` is no longer a valid binding for `conversationId`
3687
4089
  * (#553). Keyed by conversation and hard-capped, so it cannot grow with the
@@ -3965,9 +4367,10 @@ var ChannelDriver = class _ChannelDriver {
3965
4367
  * opencode reports ACTIVELY `running` is watched to completion (its liveness
3966
4368
  * heartbeat keeps the cron off its row), while a re-adopted turn that is paused
3967
4369
  * awaiting a human — or queued/unreachable — is still bounded by `deadline` and
3968
- * handed to the cron. The old "the `deadline` must settle before the ~15-min
3969
- * cron or they double-drive" reasoning is superseded: liveness now settles the
3970
- * actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
4370
+ * handed to the cron. Real invariant (#965): the cron MAY reclaim a row this
4371
+ * runner still holds; a reclaimed row that already ran is never re-dispatched
4372
+ * while opencode reports its turn ongoing (readopt's own gate here, and the
4373
+ * `pending`-row re-drive fence, `resolveRedrive`). `dispatchedAt` stays `now`
3971
4374
  * (only the appear-guard uses it).
3972
4375
  *
3973
4376
  * `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
@@ -4527,7 +4930,10 @@ var ChannelDriver = class _ChannelDriver {
4527
4930
  * re-dispatched (at most once, see `forceReadoptRun`):
4528
4931
  * - `done` → `markDone` now (guarded like the watcher's done branch);
4529
4932
  * - `failed` → `markFailed` with the surfaced error (issue #182), so an
4530
- * errored turn is reported failed on restart, NOT re-dispatched;
4933
+ * errored turn is reported failed on restart, NOT re-dispatched
4934
+ * EXCEPT a restart-ABORTED turn under a not-ongoing session,
4935
+ * which is a restart orphan wearing a terminal error and is
4936
+ * re-dispatched instead (issue #1310, see the branch below);
4531
4937
  * - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
4532
4938
  * tracking the stored id so the reply correlates by it;
4533
4939
  * - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
@@ -4591,7 +4997,16 @@ var ChannelDriver = class _ChannelDriver {
4591
4997
  void this.postSignal(row.conversation_id, row.id, "readopt_done");
4592
4998
  return;
4593
4999
  }
4594
- if (state === "failed") {
5000
+ const restartAborted = state === "failed" && sessionOngoing === false && isAbortedTerminalReply(messages, ocId ?? "");
5001
+ if (restartAborted) {
5002
+ this.log({
5003
+ level: "info",
5004
+ message: `Re-adopt: message ${row.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
5005
+ conversation_id: row.conversation_id,
5006
+ message_id: row.id
5007
+ });
5008
+ }
5009
+ if (state === "failed" && !restartAborted) {
4595
5010
  const error2 = messageError(messages, ocId ?? "") ?? void 0;
4596
5011
  const usage = messageUsage(messages, ocId ?? "");
4597
5012
  const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
@@ -4869,7 +5284,8 @@ var ChannelDriver = class _ChannelDriver {
4869
5284
  opencode_model: row.opencode_model,
4870
5285
  source_message_id: row.source_message_id,
4871
5286
  slack_user_id: row.slack_user_id,
4872
- attachments: row.attachments ?? null
5287
+ attachments: row.attachments ?? null,
5288
+ opencode_message_id: row.opencode_message_id
4873
5289
  };
4874
5290
  }
4875
5291
  /**
@@ -6121,10 +6537,15 @@ async function handleAuthError(state, error2) {
6121
6537
  }
6122
6538
  async function driveChannels(state, driver) {
6123
6539
  let idlePolls = 0;
6540
+ let idleMs = 0;
6124
6541
  let consecutiveDrainFailures = 0;
6542
+ let unreachableMs = 0;
6125
6543
  let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
6126
6544
  let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
6127
6545
  while (state.running) {
6546
+ const cycleStartedAtMs = performance.now();
6547
+ let idleThisCycle = false;
6548
+ let unreachableThisCycle = false;
6128
6549
  if (state.connection?.reconnecting && state.connection.reconnectPromise) {
6129
6550
  logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
6130
6551
  if (state.interactive) displayStatus(state);
@@ -6140,17 +6561,22 @@ async function driveChannels(state, driver) {
6140
6561
  try {
6141
6562
  const processed = await driver.drainPending();
6142
6563
  consecutiveDrainFailures = 0;
6564
+ unreachableMs = 0;
6143
6565
  state.messageCount += processed;
6144
6566
  const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
6145
6567
  lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
6146
6568
  const appliedFiles = driver.fileSyncActivity().appliedFiles;
6147
- const fileActivity = carriedOverFileSync || appliedFiles !== lastSeenAppliedFiles;
6569
+ const filesApplied = appliedFiles !== lastSeenAppliedFiles;
6570
+ const fileActivity = carriedOverFileSync || filesApplied;
6148
6571
  lastSeenAppliedFiles = appliedFiles;
6572
+ if (filesApplied) state.claudeUsageRearm?.();
6149
6573
  if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
6150
6574
  idlePolls = 0;
6575
+ idleMs = 0;
6151
6576
  if (processed > 0 && state.interactive) displayStatus(state);
6152
6577
  } else if (state.idleTimeout !== null) {
6153
6578
  idlePolls++;
6579
+ idleThisCycle = true;
6154
6580
  if (idlePolls === 1) {
6155
6581
  logActivity(state, {
6156
6582
  type: "info",
@@ -6176,8 +6602,10 @@ async function driveChannels(state, driver) {
6176
6602
  if (state.interactive) displayStatus(state);
6177
6603
  if (driver.hasInFlightWatchers()) {
6178
6604
  consecutiveDrainFailures = 0;
6605
+ unreachableMs = 0;
6179
6606
  } else if (state.idleTimeout !== null) {
6180
6607
  consecutiveDrainFailures++;
6608
+ unreachableThisCycle = true;
6181
6609
  if (consecutiveDrainFailures === 1) {
6182
6610
  logActivity(state, {
6183
6611
  type: "info",
@@ -6188,25 +6616,22 @@ async function driveChannels(state, driver) {
6188
6616
  }
6189
6617
  }
6190
6618
  await new Promise((resolve3) => setTimeout(resolve3, CHANNEL_POLL_INTERVAL_MS));
6191
- if (state.idleTimeout !== null && consecutiveDrainFailures >= 2) {
6192
- const unreachableMs = consecutiveDrainFailures * CHANNEL_POLL_INTERVAL_MS;
6193
- if (unreachableMs > state.idleTimeout * 1e3) {
6194
- logActivity(state, {
6195
- type: "info",
6196
- level: "warn",
6197
- message: `Exiting: could not reach Evident for ${consecutiveDrainFailures} consecutive polls (${Math.round(unreachableMs / 1e3)}s)`
6198
- });
6199
- if (state.interactive) displayStatus(state);
6200
- break;
6201
- }
6619
+ const cycleMs = performance.now() - cycleStartedAtMs;
6620
+ if (idleThisCycle) idleMs += cycleMs;
6621
+ if (unreachableThisCycle) unreachableMs += cycleMs;
6622
+ if (state.idleTimeout !== null && consecutiveDrainFailures >= 2 && unreachableMs > state.idleTimeout * 1e3) {
6623
+ logActivity(state, {
6624
+ type: "info",
6625
+ level: "warn",
6626
+ message: `Exiting: could not reach Evident for ${consecutiveDrainFailures} consecutive polls (${Math.round(unreachableMs / 1e3)}s)`
6627
+ });
6628
+ if (state.interactive) displayStatus(state);
6629
+ break;
6202
6630
  }
6203
- if (state.idleTimeout !== null && idlePolls >= 2) {
6204
- const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
6205
- if (idleMs > state.idleTimeout * 1e3) {
6206
- logActivity(state, { type: "info", message: "Idle timeout reached" });
6207
- if (state.interactive) displayStatus(state);
6208
- break;
6209
- }
6631
+ if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
6632
+ logActivity(state, { type: "info", message: "Idle timeout reached" });
6633
+ if (state.interactive) displayStatus(state);
6634
+ break;
6210
6635
  }
6211
6636
  }
6212
6637
  }
@@ -6285,6 +6710,9 @@ function scheduleSessionCleanup(state, driver, options) {
6285
6710
  );
6286
6711
  state.sessionCleanupTimers.push(interval, firstSweep);
6287
6712
  }
6713
+ function claudeUsageFailureStreakSuffix(consecutiveFailures) {
6714
+ return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
6715
+ }
6288
6716
  function scheduleClaudeUsageReporting(state, options) {
6289
6717
  const { mode, warnings } = resolveClaudeUsageReportingMode(
6290
6718
  options.claudeUsageReporting,
@@ -6303,13 +6731,26 @@ function scheduleClaudeUsageReporting(state, options) {
6303
6731
  level: "debug",
6304
6732
  message: "Claude usage reporting is off (--claude-usage-reporting off)"
6305
6733
  });
6306
- return;
6734
+ return null;
6307
6735
  }
6308
6736
  let consecutiveFailures = 0;
6737
+ let armed = false;
6738
+ let rearmRequested = false;
6309
6739
  const scheduleNextTick = () => {
6740
+ armed = true;
6741
+ rearmRequested = false;
6310
6742
  state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
6311
6743
  };
6312
- const tick = async (isFirst) => {
6744
+ const rearm = () => {
6745
+ if (armed) {
6746
+ rearmRequested = true;
6747
+ return;
6748
+ }
6749
+ rearmRequested = false;
6750
+ armed = true;
6751
+ state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
6752
+ };
6753
+ const tick = async (isProbe) => {
6313
6754
  try {
6314
6755
  const usage = await getClaudeUsage();
6315
6756
  const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
@@ -6331,8 +6772,8 @@ function scheduleClaudeUsageReporting(state, options) {
6331
6772
  consecutiveFailures++;
6332
6773
  logActivity(state, {
6333
6774
  type: "info",
6334
- level: consecutiveFailures === 1 ? "warn" : "debug",
6335
- message: `Failed to report Claude usage: ${result.error}`
6775
+ level: claudeUsageFailureLogLevel(consecutiveFailures),
6776
+ message: `Failed to report Claude usage: ${result.error}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
6336
6777
  });
6337
6778
  }
6338
6779
  scheduleNextTick();
@@ -6345,12 +6786,14 @@ function scheduleClaudeUsageReporting(state, options) {
6345
6786
  message: "Claude usage reporting is forced on but no usable Claude Code login was found \u2014 run `claude` to sign in; reporting will keep retrying"
6346
6787
  });
6347
6788
  scheduleNextTick();
6348
- } else if (isFirst) {
6789
+ } else if (isProbe) {
6349
6790
  logActivity(state, {
6350
6791
  type: "info",
6351
6792
  level: "debug",
6352
6793
  message: `Claude usage reporting: ${error2.message}`
6353
6794
  });
6795
+ armed = false;
6796
+ if (rearmRequested) rearm();
6354
6797
  } else {
6355
6798
  logActivity(state, {
6356
6799
  type: "info",
@@ -6364,14 +6807,16 @@ function scheduleClaudeUsageReporting(state, options) {
6364
6807
  const message = error2 instanceof Error ? error2.message : String(error2);
6365
6808
  logActivity(state, {
6366
6809
  type: "info",
6367
- level: consecutiveFailures === 1 ? "warn" : "debug",
6368
- message: `Claude usage reporting failed: ${message}`
6810
+ level: claudeUsageFailureLogLevel(consecutiveFailures),
6811
+ message: `Claude usage reporting failed: ${message}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
6369
6812
  });
6370
6813
  scheduleNextTick();
6371
6814
  }
6372
6815
  }
6373
6816
  };
6817
+ armed = true;
6374
6818
  state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
6819
+ return rearm;
6375
6820
  }
6376
6821
  async function notifyOffline(state) {
6377
6822
  if (!state.agentId || !state.authHeader) return;
@@ -6412,6 +6857,7 @@ async function cleanup(state, opts = {}) {
6412
6857
  clearTimeout(state.claudeUsageTimer);
6413
6858
  state.claudeUsageTimer = null;
6414
6859
  }
6860
+ state.claudeUsageRearm = null;
6415
6861
  if (opts.graceful && state.channelDriver) {
6416
6862
  state.channelDriver.stop();
6417
6863
  log2(state, "Draining in-flight channel work before shutdown...");
@@ -6493,6 +6939,7 @@ async function run(options) {
6493
6939
  lastProxiedActivityAt: null,
6494
6940
  sessionCleanupTimers: [],
6495
6941
  claudeUsageTimer: null,
6942
+ claudeUsageRearm: null,
6496
6943
  authHeader: ""
6497
6944
  };
6498
6945
  setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
@@ -6572,6 +7019,7 @@ async function run(options) {
6572
7019
  console.log(chalk6.dim("Or run `evident login` for interactive authentication"));
6573
7020
  blank();
6574
7021
  process.exit(1);
7022
+ return;
6575
7023
  }
6576
7024
  blank();
6577
7025
  console.log(chalk6.yellow("You are not logged in to Evident."));
@@ -6616,6 +7064,7 @@ async function run(options) {
6616
7064
  } else {
6617
7065
  printError(resolved.error || "Failed to resolve runner ID from key");
6618
7066
  process.exit(1);
7067
+ return;
6619
7068
  }
6620
7069
  } else {
6621
7070
  printError(
@@ -6629,6 +7078,7 @@ async function run(options) {
6629
7078
  );
6630
7079
  blank();
6631
7080
  process.exit(1);
7081
+ return;
6632
7082
  }
6633
7083
  }
6634
7084
  telemetry.info(
@@ -6876,7 +7326,7 @@ async function run(options) {
6876
7326
  throw error2;
6877
7327
  }
6878
7328
  scheduleSessionCleanup(state, channelDriver, options);
6879
- scheduleClaudeUsageReporting(state, options);
7329
+ state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
6880
7330
  if (!interactive || state.json) {
6881
7331
  log2(state, "Driving channel messages...");
6882
7332
  }
@@ -6961,7 +7411,7 @@ program.command("run").description("Connect to Evident and process messages").op
6961
7411
  []
6962
7412
  ).option(
6963
7413
  "--tunnel-ready-file <path>",
6964
- "Path to write once the tunnel is connected (set by the MicroVM hooks; unused on a developer machine)"
7414
+ "Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
6965
7415
  ).action(
6966
7416
  (options) => {
6967
7417
  run({