@evident-ai/cli 3.3.1-dev.d288a4b → 3.3.1-dev.dd703c3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +433 -72
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -934,6 +934,7 @@ import { homedir } from "os";
|
|
|
934
934
|
import { join } from "path";
|
|
935
935
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
936
936
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
937
|
+
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
937
938
|
function parseClaudeCliCredentials(raw) {
|
|
938
939
|
let parsed;
|
|
939
940
|
try {
|
|
@@ -967,7 +968,7 @@ function readClaudeCliCredentials() {
|
|
|
967
968
|
}
|
|
968
969
|
}
|
|
969
970
|
try {
|
|
970
|
-
const raw = readFileSync(join(homedir(),
|
|
971
|
+
const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
|
|
971
972
|
return parseClaudeCliCredentials(raw);
|
|
972
973
|
} catch (err) {
|
|
973
974
|
const code = err.code;
|
|
@@ -1063,7 +1064,7 @@ async function claudeUsage() {
|
|
|
1063
1064
|
|
|
1064
1065
|
// src/commands/run.ts
|
|
1065
1066
|
import { homedir as homedir3 } from "os";
|
|
1066
|
-
import { isAbsolute as isAbsolute2, join as
|
|
1067
|
+
import { isAbsolute as isAbsolute2, join as join5, parse, resolve as resolvePath } from "path";
|
|
1067
1068
|
import chalk6 from "chalk";
|
|
1068
1069
|
|
|
1069
1070
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1700,13 +1701,22 @@ function buildNoProviderWarning(hasProvider) {
|
|
|
1700
1701
|
return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
|
|
1701
1702
|
}
|
|
1702
1703
|
|
|
1704
|
+
// src/lib/http-timeout.ts
|
|
1705
|
+
var REQUEST_TIMEOUT_MS = 6e4;
|
|
1706
|
+
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1707
|
+
return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1703
1710
|
// src/lib/opencode/session.ts
|
|
1711
|
+
function timedFetch(input, init) {
|
|
1712
|
+
return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
|
|
1713
|
+
}
|
|
1704
1714
|
function opencodeBase(port) {
|
|
1705
1715
|
return `http://127.0.0.1:${port}`;
|
|
1706
1716
|
}
|
|
1707
1717
|
async function getOpenCodeDirectory(port) {
|
|
1708
1718
|
try {
|
|
1709
|
-
const res = await
|
|
1719
|
+
const res = await timedFetch(`${opencodeBase(port)}/path`);
|
|
1710
1720
|
if (!res.ok) return null;
|
|
1711
1721
|
const body = await res.json();
|
|
1712
1722
|
const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
|
|
@@ -1757,7 +1767,7 @@ function isAssistantInFlight(m) {
|
|
|
1757
1767
|
}
|
|
1758
1768
|
async function getSessionMessages(port, sessionId) {
|
|
1759
1769
|
try {
|
|
1760
|
-
const res = await
|
|
1770
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1761
1771
|
if (!res.ok) return null;
|
|
1762
1772
|
const body = await res.json();
|
|
1763
1773
|
return Array.isArray(body) ? body : null;
|
|
@@ -1787,7 +1797,7 @@ function sessionLastActivityMs(session) {
|
|
|
1787
1797
|
}
|
|
1788
1798
|
async function listSessions(port) {
|
|
1789
1799
|
try {
|
|
1790
|
-
const res = await
|
|
1800
|
+
const res = await timedFetch(`${opencodeBase(port)}/session`);
|
|
1791
1801
|
if (!res.ok) return null;
|
|
1792
1802
|
const body = await res.json();
|
|
1793
1803
|
return Array.isArray(body) ? body : null;
|
|
@@ -1797,7 +1807,7 @@ async function listSessions(port) {
|
|
|
1797
1807
|
}
|
|
1798
1808
|
async function deleteSession(port, id) {
|
|
1799
1809
|
try {
|
|
1800
|
-
const res = await
|
|
1810
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1801
1811
|
return res.status >= 200 && res.status < 300;
|
|
1802
1812
|
} catch {
|
|
1803
1813
|
return false;
|
|
@@ -1805,7 +1815,7 @@ async function deleteSession(port, id) {
|
|
|
1805
1815
|
}
|
|
1806
1816
|
async function sessionExists(port, id) {
|
|
1807
1817
|
try {
|
|
1808
|
-
const res = await
|
|
1818
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
|
|
1809
1819
|
if (res.status >= 200 && res.status < 300) return true;
|
|
1810
1820
|
if (res.status === 404) return false;
|
|
1811
1821
|
return null;
|
|
@@ -1815,7 +1825,7 @@ async function sessionExists(port, id) {
|
|
|
1815
1825
|
}
|
|
1816
1826
|
async function getSessionStatuses(port) {
|
|
1817
1827
|
try {
|
|
1818
|
-
const res = await
|
|
1828
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/status`);
|
|
1819
1829
|
if (!res.ok) {
|
|
1820
1830
|
console.error(
|
|
1821
1831
|
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
@@ -1848,7 +1858,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1848
1858
|
if (directory && directory.trim()) {
|
|
1849
1859
|
url.searchParams.set("directory", directory.trim());
|
|
1850
1860
|
}
|
|
1851
|
-
const response = await
|
|
1861
|
+
const response = await timedFetch(url, {
|
|
1852
1862
|
method: "POST",
|
|
1853
1863
|
headers: { "Content-Type": "application/json" },
|
|
1854
1864
|
body: JSON.stringify({})
|
|
@@ -1862,7 +1872,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1862
1872
|
}
|
|
1863
1873
|
async function getModelAttachmentCapability(port, model) {
|
|
1864
1874
|
try {
|
|
1865
|
-
const res = await
|
|
1875
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1866
1876
|
if (!res.ok) {
|
|
1867
1877
|
console.error(
|
|
1868
1878
|
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -1995,7 +2005,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1995
2005
|
};
|
|
1996
2006
|
}
|
|
1997
2007
|
}
|
|
1998
|
-
const res = await
|
|
2008
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1999
2009
|
method: "POST",
|
|
2000
2010
|
headers: { "Content-Type": "application/json" },
|
|
2001
2011
|
body: JSON.stringify(body)
|
|
@@ -2244,7 +2254,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
2244
2254
|
}
|
|
2245
2255
|
async function hasAnyConfiguredProvider(port) {
|
|
2246
2256
|
try {
|
|
2247
|
-
const res = await
|
|
2257
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2248
2258
|
if (!res.ok) {
|
|
2249
2259
|
console.error(
|
|
2250
2260
|
`[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -2986,6 +2996,9 @@ function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
|
2986
2996
|
// src/lib/channels/driver.ts
|
|
2987
2997
|
import { homedir as homedir2 } from "os";
|
|
2988
2998
|
|
|
2999
|
+
// src/lib/runner-file-sync.ts
|
|
3000
|
+
import { join as join4 } from "path";
|
|
3001
|
+
|
|
2989
3002
|
// src/lib/file-push.ts
|
|
2990
3003
|
import { randomUUID } from "crypto";
|
|
2991
3004
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
@@ -3179,17 +3192,20 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3179
3192
|
for (const id of options.ackFailures.keys()) {
|
|
3180
3193
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3181
3194
|
}
|
|
3182
|
-
if (pending.length === 0) return 0;
|
|
3195
|
+
if (pending.length === 0) return { applied: 0, claudeCredentialApplied: false };
|
|
3183
3196
|
options.log({
|
|
3184
3197
|
level: "info",
|
|
3185
3198
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3186
3199
|
});
|
|
3187
3200
|
let applied = 0;
|
|
3201
|
+
let claudeCredentialApplied = false;
|
|
3188
3202
|
for (const file of pending) {
|
|
3189
3203
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3190
|
-
|
|
3204
|
+
const outcome = await applyOne(options, file);
|
|
3205
|
+
if (outcome.applied) applied += 1;
|
|
3206
|
+
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
3191
3207
|
}
|
|
3192
|
-
return applied;
|
|
3208
|
+
return { applied, claudeCredentialApplied };
|
|
3193
3209
|
}
|
|
3194
3210
|
async function listPendingFiles(options) {
|
|
3195
3211
|
let res;
|
|
@@ -3250,6 +3266,11 @@ function asPendingFile(entry) {
|
|
|
3250
3266
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3251
3267
|
return { id, path, size };
|
|
3252
3268
|
}
|
|
3269
|
+
var NOT_APPLIED = { applied: false, claudeCredentialApplied: false };
|
|
3270
|
+
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3271
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join4(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3272
|
+
return expanded === join4(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3273
|
+
}
|
|
3253
3274
|
async function applyOne(options, file) {
|
|
3254
3275
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
3255
3276
|
if (options.allowedDirectories.length === 0) {
|
|
@@ -3258,7 +3279,7 @@ async function applyOne(options, file) {
|
|
|
3258
3279
|
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
3259
3280
|
});
|
|
3260
3281
|
await ack(options, file, "rejected", "file_sync_disabled");
|
|
3261
|
-
return
|
|
3282
|
+
return NOT_APPLIED;
|
|
3262
3283
|
}
|
|
3263
3284
|
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
3264
3285
|
options.log({
|
|
@@ -3266,12 +3287,12 @@ async function applyOne(options, file) {
|
|
|
3266
3287
|
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
3267
3288
|
});
|
|
3268
3289
|
await ack(options, file, "rejected", "file_too_large");
|
|
3269
|
-
return
|
|
3290
|
+
return NOT_APPLIED;
|
|
3270
3291
|
}
|
|
3271
3292
|
const download = await downloadContent(options, file, label);
|
|
3272
3293
|
if (!download.ok) {
|
|
3273
3294
|
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
3274
|
-
return
|
|
3295
|
+
return NOT_APPLIED;
|
|
3275
3296
|
}
|
|
3276
3297
|
let outcome;
|
|
3277
3298
|
try {
|
|
@@ -3287,7 +3308,7 @@ async function applyOne(options, file) {
|
|
|
3287
3308
|
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
3288
3309
|
});
|
|
3289
3310
|
await ack(options, file, "rejected", "write_failed");
|
|
3290
|
-
return
|
|
3311
|
+
return NOT_APPLIED;
|
|
3291
3312
|
}
|
|
3292
3313
|
if (!outcome.ok) {
|
|
3293
3314
|
options.log({
|
|
@@ -3295,14 +3316,17 @@ async function applyOne(options, file) {
|
|
|
3295
3316
|
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
3296
3317
|
});
|
|
3297
3318
|
await ack(options, file, "rejected", outcome.code);
|
|
3298
|
-
return
|
|
3319
|
+
return NOT_APPLIED;
|
|
3299
3320
|
}
|
|
3300
3321
|
options.log({
|
|
3301
3322
|
level: "info",
|
|
3302
3323
|
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
3303
3324
|
});
|
|
3304
3325
|
await ack(options, file, "applied");
|
|
3305
|
-
return
|
|
3326
|
+
return {
|
|
3327
|
+
applied: true,
|
|
3328
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir)
|
|
3329
|
+
};
|
|
3306
3330
|
}
|
|
3307
3331
|
function durableDownloadCode(status2) {
|
|
3308
3332
|
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
@@ -3413,8 +3437,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
|
3413
3437
|
var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
|
|
3414
3438
|
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
3415
3439
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
3440
|
+
var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
|
|
3441
|
+
var MAX_WATCHER_STALL_RESTARTS = 3;
|
|
3442
|
+
var MAX_RELEASED_OPENCODE_IDS = 256;
|
|
3416
3443
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3417
3444
|
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
3445
|
+
var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
|
|
3446
|
+
var MAX_WEDGED_CONVERSATIONS = 256;
|
|
3418
3447
|
var ChannelAuthError = class extends Error {
|
|
3419
3448
|
constructor(message) {
|
|
3420
3449
|
super(message);
|
|
@@ -3458,6 +3487,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3458
3487
|
fileSyncDirectories;
|
|
3459
3488
|
homeDir;
|
|
3460
3489
|
maxActiveSessions;
|
|
3490
|
+
watcherStallMs;
|
|
3491
|
+
wedgeWarningIntervalMs;
|
|
3461
3492
|
/** Cache of conversationId → opencode sessionId. */
|
|
3462
3493
|
sessions = /* @__PURE__ */ new Map();
|
|
3463
3494
|
/**
|
|
@@ -3488,6 +3519,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3488
3519
|
* bounded cost.
|
|
3489
3520
|
*/
|
|
3490
3521
|
supersededSessions = /* @__PURE__ */ new Map();
|
|
3522
|
+
/**
|
|
3523
|
+
* Local re-drive fence for a message force-released by the stall watchdog
|
|
3524
|
+
* (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
|
|
3525
|
+
* see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
|
|
3526
|
+
* is `null` for exactly this shape (its `markProcessing` never landed), so
|
|
3527
|
+
* without a local record of the id the driver last knew, the next drain's
|
|
3528
|
+
* `if (message.opencode_message_id)` re-drive-fence check at
|
|
3529
|
+
* `processConversation` would not engage and it would blind-`prompt_async`
|
|
3530
|
+
* a turn that may still be running in opencode — the one duplicate-turn
|
|
3531
|
+
* hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
|
|
3532
|
+
* `processConversation` reads `message.opencode_message_id ?? this
|
|
3533
|
+
* .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
|
|
3534
|
+
* threads it into `resolveRedrive`, which asks opencode itself whether the
|
|
3535
|
+
* turn is still ongoing before ever dispatching.
|
|
3536
|
+
*
|
|
3537
|
+
* Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
|
|
3538
|
+
* `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
|
|
3539
|
+
* non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
|
|
3540
|
+
* dispatch) and at the top-level fresh-dispatch site, so it does not outlive
|
|
3541
|
+
* the row it was recorded for.
|
|
3542
|
+
*/
|
|
3543
|
+
releasedOpencodeIds = /* @__PURE__ */ new Map();
|
|
3544
|
+
/**
|
|
3545
|
+
* Per-conversation throttle state for the #183 recurrence warning (#1618
|
|
3546
|
+
* WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
|
|
3547
|
+
* `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
|
|
3548
|
+
* log line and the `dispatch_wedged` signal to at most once per
|
|
3549
|
+
* `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
|
|
3550
|
+
* so the operator sees magnitude, not repetition. Cleared the moment the
|
|
3551
|
+
* conversation dispatches anything (a fresh wedge, if it recurs, is a new
|
|
3552
|
+
* incident). Bounded FIFO, mirroring `supersededSessions`
|
|
3553
|
+
* (`MAX_WEDGED_CONVERSATIONS`).
|
|
3554
|
+
*/
|
|
3555
|
+
wedgeWarnings = /* @__PURE__ */ new Map();
|
|
3491
3556
|
/**
|
|
3492
3557
|
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
3493
3558
|
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
@@ -3706,6 +3771,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3706
3771
|
* same trick `lastProxiedActivityAt` uses.
|
|
3707
3772
|
*/
|
|
3708
3773
|
appliedFileCount = 0;
|
|
3774
|
+
/**
|
|
3775
|
+
* Generation counter, NOT a tally (#1656): advances by exactly one per sync
|
|
3776
|
+
* batch that applied the Claude CLI credential file, not by how many
|
|
3777
|
+
* credential files were in that batch. `run.ts` only ever tests inequality
|
|
3778
|
+
* against the value it saw last cycle, so magnitude is meaningless — keep it
|
|
3779
|
+
* that way rather than "fixing" it into a count.
|
|
3780
|
+
*/
|
|
3781
|
+
claudeCredentialApplyCount = 0;
|
|
3709
3782
|
/**
|
|
3710
3783
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3711
3784
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -3730,7 +3803,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3730
3803
|
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3731
3804
|
this.log = config.log ?? (() => {
|
|
3732
3805
|
});
|
|
3733
|
-
this.fetchImpl =
|
|
3806
|
+
this.fetchImpl = withRequestTimeout(
|
|
3807
|
+
config.fetchImpl ?? fetch,
|
|
3808
|
+
config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
|
|
3809
|
+
);
|
|
3734
3810
|
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3735
3811
|
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3736
3812
|
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
@@ -3739,6 +3815,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3739
3815
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3740
3816
|
this.homeDir = config.homeDir ?? homedir2();
|
|
3741
3817
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
3818
|
+
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
3819
|
+
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
3742
3820
|
}
|
|
3743
3821
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
3744
3822
|
get opencodeBase() {
|
|
@@ -3752,6 +3830,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3752
3830
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
3753
3831
|
*/
|
|
3754
3832
|
async drainPending() {
|
|
3833
|
+
try {
|
|
3834
|
+
this.reconcileWatchers();
|
|
3835
|
+
} catch (err) {
|
|
3836
|
+
this.log({
|
|
3837
|
+
level: "error",
|
|
3838
|
+
message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
|
|
3839
|
+
});
|
|
3840
|
+
}
|
|
3755
3841
|
if (this.stopped) return 0;
|
|
3756
3842
|
if (this.draining) return 0;
|
|
3757
3843
|
this.draining = true;
|
|
@@ -3785,7 +3871,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3785
3871
|
if (this.syncingFiles) return 0;
|
|
3786
3872
|
this.syncingFiles = true;
|
|
3787
3873
|
try {
|
|
3788
|
-
const
|
|
3874
|
+
const result = await syncPendingRunnerFiles({
|
|
3789
3875
|
agentId: this.agentId,
|
|
3790
3876
|
apiUrl: this.apiUrl,
|
|
3791
3877
|
getAuthHeader: this.getAuthHeader,
|
|
@@ -3795,8 +3881,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3795
3881
|
ackFailures: this.fileAckFailures,
|
|
3796
3882
|
log: this.log
|
|
3797
3883
|
});
|
|
3798
|
-
this.appliedFileCount += applied;
|
|
3799
|
-
|
|
3884
|
+
this.appliedFileCount += result.applied;
|
|
3885
|
+
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
3886
|
+
return result.applied;
|
|
3800
3887
|
} catch (err) {
|
|
3801
3888
|
this.log({
|
|
3802
3889
|
level: "error",
|
|
@@ -3887,12 +3974,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3887
3974
|
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3888
3975
|
* idle checks still shows up as an advance.
|
|
3889
3976
|
*
|
|
3977
|
+
* A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
|
|
3978
|
+
* (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
|
|
3979
|
+
* reporting on it advancing, so an unrelated file sync can never disturb a
|
|
3980
|
+
* healthy reporting cadence (#1627) — it never even reaches that trigger, let
|
|
3981
|
+
* alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
|
|
3982
|
+
* whose consumer is idle-timeout suppression and must key on ANY file, not
|
|
3983
|
+
* just a Claude credential.
|
|
3984
|
+
*
|
|
3890
3985
|
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3891
3986
|
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3892
3987
|
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3893
3988
|
*/
|
|
3894
3989
|
fileSyncActivity() {
|
|
3895
|
-
return {
|
|
3990
|
+
return {
|
|
3991
|
+
appliedFiles: this.appliedFileCount,
|
|
3992
|
+
inFlight: this.syncingFiles,
|
|
3993
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount
|
|
3994
|
+
};
|
|
3896
3995
|
}
|
|
3897
3996
|
/**
|
|
3898
3997
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
@@ -4005,8 +4104,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4005
4104
|
skippedAlreadyDispatched += 1;
|
|
4006
4105
|
continue;
|
|
4007
4106
|
}
|
|
4008
|
-
|
|
4009
|
-
|
|
4107
|
+
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
4108
|
+
if (effectiveOpencodeMessageId) {
|
|
4109
|
+
const outcome = await this.resolveRedrive(
|
|
4110
|
+
conv,
|
|
4111
|
+
sessionId,
|
|
4112
|
+
message,
|
|
4113
|
+
sessionCreated,
|
|
4114
|
+
effectiveOpencodeMessageId
|
|
4115
|
+
);
|
|
4010
4116
|
if (outcome === "abandoned") {
|
|
4011
4117
|
continue;
|
|
4012
4118
|
}
|
|
@@ -4117,21 +4223,102 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4117
4223
|
}
|
|
4118
4224
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4119
4225
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
4226
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
4120
4227
|
this.dispatched.add(message.id);
|
|
4121
4228
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
4122
4229
|
dispatched += 1;
|
|
4123
4230
|
void this.postSignal(conv.id, message.id, "dispatched");
|
|
4124
4231
|
}
|
|
4125
4232
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
4126
|
-
this.
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
conversation_id: conv.id
|
|
4130
|
-
});
|
|
4233
|
+
this.reportWedgedConversation(conv, messages);
|
|
4234
|
+
} else if (dispatched > 0) {
|
|
4235
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4131
4236
|
}
|
|
4132
4237
|
this.ensureWatcherRunning(sessionId);
|
|
4133
4238
|
return dispatched;
|
|
4134
4239
|
}
|
|
4240
|
+
/**
|
|
4241
|
+
* The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
|
|
4242
|
+
* escalated (#1618 WI-4). `messages` is the conversation's full pending list
|
|
4243
|
+
* on THIS tick — the caller has already confirmed every one of them is a
|
|
4244
|
+
* skip-because-already-`dispatched`, the exact signature of a message stuck
|
|
4245
|
+
* acknowledged-but-never-worked.
|
|
4246
|
+
*
|
|
4247
|
+
* Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
|
|
4248
|
+
* (52,843 occurrences observed in one incident) — burning the GLOBAL
|
|
4249
|
+
* 30-events/60s `runner-activity-telemetry.ts` budget that was itself
|
|
4250
|
+
* suppressing the diagnostics needed to debug the wedge. The `warn` log (and
|
|
4251
|
+
* the `dispatch_wedged` signal once the wedge has persisted past the same
|
|
4252
|
+
* interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
|
|
4253
|
+
* naming the consecutive-tick count so the operator sees magnitude rather
|
|
4254
|
+
* than repetition.
|
|
4255
|
+
*
|
|
4256
|
+
* Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
|
|
4257
|
+
* unconditionally on this same tick and is already recovering anything it
|
|
4258
|
+
* can see. This is reporting only — see `countUntrackedIds`'s doc for the
|
|
4259
|
+
* one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
|
|
4260
|
+
*/
|
|
4261
|
+
reportWedgedConversation(conv, messages) {
|
|
4262
|
+
const now = this.now();
|
|
4263
|
+
const existing = this.wedgeWarnings.get(conv.id);
|
|
4264
|
+
const firstWedgedAt = existing?.firstWedgedAt ?? now;
|
|
4265
|
+
const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
|
|
4266
|
+
const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
|
|
4267
|
+
if (!dueForWarn) {
|
|
4268
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4269
|
+
this.wedgeWarnings.set(conv.id, {
|
|
4270
|
+
firstWedgedAt,
|
|
4271
|
+
lastWarnedAt: existing.lastWarnedAt,
|
|
4272
|
+
consecutiveTicks
|
|
4273
|
+
});
|
|
4274
|
+
return;
|
|
4275
|
+
}
|
|
4276
|
+
const stuckForMs = now - firstWedgedAt;
|
|
4277
|
+
const untracked = this.countUntrackedIds(messages);
|
|
4278
|
+
this.log({
|
|
4279
|
+
level: "warn",
|
|
4280
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
4281
|
+
conversation_id: conv.id
|
|
4282
|
+
});
|
|
4283
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4284
|
+
this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
|
|
4285
|
+
while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
|
|
4286
|
+
const oldest = this.wedgeWarnings.keys().next().value;
|
|
4287
|
+
if (oldest === void 0) break;
|
|
4288
|
+
this.wedgeWarnings.delete(oldest);
|
|
4289
|
+
}
|
|
4290
|
+
if (stuckForMs >= this.wedgeWarningIntervalMs) {
|
|
4291
|
+
void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
|
|
4292
|
+
stuck_for_ms: stuckForMs,
|
|
4293
|
+
untracked
|
|
4294
|
+
});
|
|
4295
|
+
}
|
|
4296
|
+
}
|
|
4297
|
+
/**
|
|
4298
|
+
* How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
|
|
4299
|
+
* WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
|
|
4300
|
+
* `dispatched`/`inFlight` pairing invariant holds by construction across
|
|
4301
|
+
* every `dispatched.add` site (see its own doc comment), so `> 0` here means
|
|
4302
|
+
* that invariant has actually been violated for this conversation: there is
|
|
4303
|
+
* no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
|
|
4304
|
+
* `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
|
|
4305
|
+
* recovering. One pass over `this.watchers`, called only when the throttled
|
|
4306
|
+
* warning above is due to fire — not every tick.
|
|
4307
|
+
*/
|
|
4308
|
+
countUntrackedIds(messages) {
|
|
4309
|
+
let untracked = 0;
|
|
4310
|
+
for (const message of messages) {
|
|
4311
|
+
let tracked = false;
|
|
4312
|
+
for (const watcher of this.watchers.values()) {
|
|
4313
|
+
if (watcher.inFlight.has(message.id)) {
|
|
4314
|
+
tracked = true;
|
|
4315
|
+
break;
|
|
4316
|
+
}
|
|
4317
|
+
}
|
|
4318
|
+
if (!tracked) untracked += 1;
|
|
4319
|
+
}
|
|
4320
|
+
return untracked;
|
|
4321
|
+
}
|
|
4135
4322
|
/**
|
|
4136
4323
|
* Poll a session's message list for the re-drive fence (#965), via the
|
|
4137
4324
|
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
@@ -4200,9 +4387,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4200
4387
|
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
4201
4388
|
* other failure resolves to `unresolved` and is retried whole on the next
|
|
4202
4389
|
* ~2s drain tick.
|
|
4390
|
+
*
|
|
4391
|
+
* `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
|
|
4392
|
+
* server `opencode_message_id` when present, else the stall watchdog's local
|
|
4393
|
+
* `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
|
|
4394
|
+
* `message` so every line below — and the signals this method posts —
|
|
4395
|
+
* keeps reporting the REAL server row; a shadow-copied `message` would
|
|
4396
|
+
* silently diverge from it.
|
|
4203
4397
|
*/
|
|
4204
|
-
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
4205
|
-
const ocId =
|
|
4398
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
|
|
4399
|
+
const ocId = effectiveOpencodeMessageId;
|
|
4206
4400
|
if (sessionCreated) {
|
|
4207
4401
|
this.clearRedriveUnresolved(message.id);
|
|
4208
4402
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
@@ -4432,7 +4626,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4432
4626
|
}
|
|
4433
4627
|
return "unresolved";
|
|
4434
4628
|
}
|
|
4435
|
-
/**
|
|
4629
|
+
/**
|
|
4630
|
+
* Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
|
|
4631
|
+
* outcome) — including the stall watchdog's local re-drive fence (#1618): once
|
|
4632
|
+
* `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
|
|
4633
|
+
* a real server-side `opencode_message_id` again or is no longer pending, so
|
|
4634
|
+
* the fence entry is no longer needed.
|
|
4635
|
+
*/
|
|
4436
4636
|
clearRedriveUnresolved(messageId) {
|
|
4437
4637
|
this.redriveUnresolvedSince.delete(messageId);
|
|
4438
4638
|
this.redriveUnresolvedSignalled.delete(messageId);
|
|
@@ -4440,6 +4640,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4440
4640
|
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4441
4641
|
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4442
4642
|
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
4643
|
+
this.releasedOpencodeIds.delete(messageId);
|
|
4443
4644
|
}
|
|
4444
4645
|
/**
|
|
4445
4646
|
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
@@ -4594,6 +4795,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4594
4795
|
this.supersededSessions.delete(oldest);
|
|
4595
4796
|
}
|
|
4596
4797
|
}
|
|
4798
|
+
/**
|
|
4799
|
+
* Record the local re-drive fence for a message force-released without
|
|
4800
|
+
* completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
|
|
4801
|
+
* `removeInFlight`, which is about to drop the `InFlightMessage` this reads
|
|
4802
|
+
* `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
|
|
4803
|
+
*/
|
|
4804
|
+
recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
|
|
4805
|
+
this.releasedOpencodeIds.delete(evidentMessageId);
|
|
4806
|
+
this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
|
|
4807
|
+
while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
|
|
4808
|
+
const oldest = this.releasedOpencodeIds.keys().next().value;
|
|
4809
|
+
if (oldest === void 0) return;
|
|
4810
|
+
this.releasedOpencodeIds.delete(oldest);
|
|
4811
|
+
}
|
|
4812
|
+
}
|
|
4597
4813
|
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
4598
4814
|
isSuperseded(conversationId, sessionId) {
|
|
4599
4815
|
return this.supersededSessions.get(conversationId) === sessionId;
|
|
@@ -4825,15 +5041,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4825
5041
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
4826
5042
|
let watcher = this.watchers.get(sessionId);
|
|
4827
5043
|
if (!watcher) {
|
|
4828
|
-
watcher =
|
|
4829
|
-
conv,
|
|
4830
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4831
|
-
loop: null,
|
|
4832
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4833
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4834
|
-
lastGoodPollAt: this.now(),
|
|
4835
|
-
hadUsablePoll: false
|
|
4836
|
-
};
|
|
5044
|
+
watcher = this.newSessionWatcher(conv);
|
|
4837
5045
|
this.watchers.set(sessionId, watcher);
|
|
4838
5046
|
}
|
|
4839
5047
|
const now = this.now();
|
|
@@ -4864,6 +5072,27 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4864
5072
|
ambiguousResolved: false
|
|
4865
5073
|
});
|
|
4866
5074
|
}
|
|
5075
|
+
/**
|
|
5076
|
+
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
5077
|
+
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
5078
|
+
* misread as stalled by the very first reconciliation that sees it.
|
|
5079
|
+
*/
|
|
5080
|
+
newSessionWatcher(conv) {
|
|
5081
|
+
const now = this.now();
|
|
5082
|
+
return {
|
|
5083
|
+
conv,
|
|
5084
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
5085
|
+
loop: null,
|
|
5086
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
5087
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
5088
|
+
lastGoodPollAt: now,
|
|
5089
|
+
hadUsablePoll: false,
|
|
5090
|
+
generation: 0,
|
|
5091
|
+
lastTickAt: now,
|
|
5092
|
+
lastObservedTickAt: now,
|
|
5093
|
+
consecutiveStallRestarts: 0
|
|
5094
|
+
};
|
|
5095
|
+
}
|
|
4867
5096
|
/**
|
|
4868
5097
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
4869
5098
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
@@ -4893,15 +5122,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4893
5122
|
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
4894
5123
|
let watcher = this.watchers.get(sessionId);
|
|
4895
5124
|
if (!watcher) {
|
|
4896
|
-
watcher =
|
|
4897
|
-
conv,
|
|
4898
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4899
|
-
loop: null,
|
|
4900
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4901
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4902
|
-
lastGoodPollAt: this.now(),
|
|
4903
|
-
hadUsablePoll: false
|
|
4904
|
-
};
|
|
5125
|
+
watcher = this.newSessionWatcher(conv);
|
|
4905
5126
|
this.watchers.set(sessionId, watcher);
|
|
4906
5127
|
}
|
|
4907
5128
|
watcher.inFlight.set(message.id, {
|
|
@@ -4945,12 +5166,110 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4945
5166
|
ambiguousResolved: false
|
|
4946
5167
|
});
|
|
4947
5168
|
}
|
|
5169
|
+
/**
|
|
5170
|
+
* Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
|
|
5171
|
+
* restarts any per-session watcher whose loop has exited or stopped ticking
|
|
5172
|
+
* — escalating to a bounded force-release only once
|
|
5173
|
+
* `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
|
|
5174
|
+
* it. Fully synchronous: it only inspects in-memory state and calls the
|
|
5175
|
+
* synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
|
|
5176
|
+
* run from the very top of `drainPending()` — ahead of the un-timed
|
|
5177
|
+
* `getPendingConversations()` await that would otherwise be able to disable
|
|
5178
|
+
* it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
|
|
5179
|
+
* `drainPending()` from being CALLED again at all, not just from finishing).
|
|
5180
|
+
*
|
|
5181
|
+
* Restarts the loop rather than releasing messages directly: a blind release
|
|
5182
|
+
* would let the next drain re-`prompt_async` a turn that may still be
|
|
5183
|
+
* running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
|
|
5184
|
+
* re-polls with each message's `opencodeMessageId` still in hand and lets
|
|
5185
|
+
* the existing, audited `!activelyRunning` give-up decide, same as it always
|
|
5186
|
+
* has.
|
|
5187
|
+
*
|
|
5188
|
+
* Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
|
|
5189
|
+
* that shape has no in-flight entry and therefore no `opencodeMessageId` to
|
|
5190
|
+
* fence a release with, so releasing it here would blind-re-POST a possibly-
|
|
5191
|
+
* running turn — and there is no conversation id in hand to signal with
|
|
5192
|
+
* either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
|
|
5193
|
+
* instead, where a conversation id already exists. If you find yourself
|
|
5194
|
+
* wanting to add a `dispatched` sweep here, don't — read the drain-wedge
|
|
5195
|
+
* plan's §3/D5 first.
|
|
5196
|
+
*/
|
|
5197
|
+
reconcileWatchers() {
|
|
5198
|
+
const now = this.now();
|
|
5199
|
+
for (const [sessionId, watcher] of [...this.watchers]) {
|
|
5200
|
+
if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
|
|
5201
|
+
watcher.consecutiveStallRestarts = 0;
|
|
5202
|
+
}
|
|
5203
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5204
|
+
if (watcher.inFlight.size === 0 && watcher.loop === null) {
|
|
5205
|
+
this.watchers.delete(sessionId);
|
|
5206
|
+
continue;
|
|
5207
|
+
}
|
|
5208
|
+
if (watcher.loop === null && watcher.inFlight.size > 0) {
|
|
5209
|
+
if (now - watcher.lastTickAt < this.watcherStallMs) continue;
|
|
5210
|
+
this.log({
|
|
5211
|
+
level: "warn",
|
|
5212
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
|
|
5213
|
+
conversation_id: watcher.conv.id
|
|
5214
|
+
});
|
|
5215
|
+
this.ensureWatcherRunning(sessionId);
|
|
5216
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5217
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5218
|
+
recovery: "loop_exited"
|
|
5219
|
+
});
|
|
5220
|
+
}
|
|
5221
|
+
continue;
|
|
5222
|
+
}
|
|
5223
|
+
if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
|
|
5224
|
+
const stalledForMs = now - watcher.lastTickAt;
|
|
5225
|
+
watcher.consecutiveStallRestarts += 1;
|
|
5226
|
+
if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
|
|
5227
|
+
this.log({
|
|
5228
|
+
level: "error",
|
|
5229
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
|
|
5230
|
+
conversation_id: watcher.conv.id
|
|
5231
|
+
});
|
|
5232
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5233
|
+
this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
|
|
5234
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5235
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5236
|
+
recovery: "unrecoverable_released"
|
|
5237
|
+
});
|
|
5238
|
+
}
|
|
5239
|
+
watcher.generation += 1;
|
|
5240
|
+
this.watchers.delete(sessionId);
|
|
5241
|
+
continue;
|
|
5242
|
+
}
|
|
5243
|
+
watcher.generation += 1;
|
|
5244
|
+
watcher.loop = null;
|
|
5245
|
+
watcher.lastGoodPollAt = now;
|
|
5246
|
+
watcher.lastTickAt = now;
|
|
5247
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5248
|
+
this.ensureWatcherRunning(sessionId);
|
|
5249
|
+
this.log({
|
|
5250
|
+
level: "warn",
|
|
5251
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
|
|
5252
|
+
conversation_id: watcher.conv.id
|
|
5253
|
+
});
|
|
5254
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5255
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5256
|
+
recovery: "loop_stalled"
|
|
5257
|
+
});
|
|
5258
|
+
}
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
}
|
|
4948
5262
|
/**
|
|
4949
5263
|
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
4950
5264
|
* work and is not already running. Single-flight per session. The loop is
|
|
4951
5265
|
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
4952
5266
|
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
4953
5267
|
* stays as the safety net.
|
|
5268
|
+
*
|
|
5269
|
+
* The generation started here (#1618) is captured in the `.finally` closure
|
|
5270
|
+
* so a RETIRED loop settling late — after `reconcileWatchers` has already
|
|
5271
|
+
* restarted this watcher under a newer generation — can neither null the new
|
|
5272
|
+
* loop's handle nor delete a watcher that still has live work.
|
|
4954
5273
|
*/
|
|
4955
5274
|
ensureWatcherRunning(sessionId) {
|
|
4956
5275
|
const watcher = this.watchers.get(sessionId);
|
|
@@ -4960,7 +5279,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4960
5279
|
this.watchers.delete(sessionId);
|
|
4961
5280
|
return;
|
|
4962
5281
|
}
|
|
4963
|
-
const
|
|
5282
|
+
const generation = watcher.generation;
|
|
5283
|
+
const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
|
|
5284
|
+
if (watcher.generation !== generation) return;
|
|
4964
5285
|
watcher.loop = null;
|
|
4965
5286
|
if (watcher.inFlight.size === 0) {
|
|
4966
5287
|
this.watchers.delete(sessionId);
|
|
@@ -4980,11 +5301,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4980
5301
|
* `source_message_id`;
|
|
4981
5302
|
* 4. drops messages that completed or timed out from the in-flight set.
|
|
4982
5303
|
* Exits when the in-flight set empties. Never throws.
|
|
5304
|
+
*
|
|
5305
|
+
* `generation` (#1618) is the incarnation this call was started under.
|
|
5306
|
+
* `reconcileWatchers` can restart a stalled loop by bumping
|
|
5307
|
+
* `watcher.generation` and starting a NEW `runWatcherLoop` over the same
|
|
5308
|
+
* `SessionWatcher` object — the stalled promise itself cannot be cancelled,
|
|
5309
|
+
* so this loop instead checks at the top of every iteration, right after
|
|
5310
|
+
* waking from `sleep`, and right before servicing any message, and quietly
|
|
5311
|
+
* retires (returns without touching anything) the moment it is no longer the
|
|
5312
|
+
* watcher's current generation. Retiring mid-tick can still let ONE
|
|
5313
|
+
* `serviceInFlightMessage` pass complete first — acceptable, since that
|
|
5314
|
+
* method contains no non-idempotent action.
|
|
4983
5315
|
*/
|
|
4984
|
-
async runWatcherLoop(sessionId, watcher) {
|
|
5316
|
+
async runWatcherLoop(sessionId, watcher, generation) {
|
|
4985
5317
|
try {
|
|
4986
5318
|
while (watcher.inFlight.size > 0) {
|
|
5319
|
+
if (watcher.generation !== generation) return;
|
|
5320
|
+
watcher.lastTickAt = this.now();
|
|
4987
5321
|
await this.sleep(this.pausedPollIntervalMs);
|
|
5322
|
+
if (watcher.generation !== generation) return;
|
|
4988
5323
|
let messages = null;
|
|
4989
5324
|
try {
|
|
4990
5325
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
@@ -5005,6 +5340,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5005
5340
|
}
|
|
5006
5341
|
}
|
|
5007
5342
|
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
5343
|
+
if (watcher.generation !== generation) return;
|
|
5008
5344
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
5009
5345
|
await this.serviceInFlightMessage(
|
|
5010
5346
|
sessionId,
|
|
@@ -6980,7 +7316,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
6980
7316
|
if (trimmed === "") {
|
|
6981
7317
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6982
7318
|
}
|
|
6983
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7319
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join5(homeDir, trimmed.slice(2)) : trimmed;
|
|
6984
7320
|
if (!isAbsolute2(expanded)) {
|
|
6985
7321
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6986
7322
|
}
|
|
@@ -7187,6 +7523,7 @@ async function driveChannels(state, driver) {
|
|
|
7187
7523
|
let unreachableMs = 0;
|
|
7188
7524
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7189
7525
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
7526
|
+
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
7190
7527
|
while (state.running) {
|
|
7191
7528
|
const cycleStartedAtMs = performance.now();
|
|
7192
7529
|
let idleThisCycle = false;
|
|
@@ -7210,11 +7547,15 @@ async function driveChannels(state, driver) {
|
|
|
7210
7547
|
state.messageCount += processed;
|
|
7211
7548
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
7212
7549
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7213
|
-
const
|
|
7550
|
+
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
7551
|
+
const appliedFiles = fileActivitySnapshot.appliedFiles;
|
|
7214
7552
|
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
7215
7553
|
const fileActivity = carriedOverFileSync || filesApplied;
|
|
7216
7554
|
lastSeenAppliedFiles = appliedFiles;
|
|
7217
|
-
|
|
7555
|
+
const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
|
|
7556
|
+
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
7557
|
+
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
7558
|
+
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
7218
7559
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7219
7560
|
idlePolls = 0;
|
|
7220
7561
|
idleMs = 0;
|
|
@@ -7283,7 +7624,7 @@ async function driveChannels(state, driver) {
|
|
|
7283
7624
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7284
7625
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7285
7626
|
function sessionDbPath() {
|
|
7286
|
-
return
|
|
7627
|
+
return join5(homedir3(), ".local", "share", "opencode", "opencode.db");
|
|
7287
7628
|
}
|
|
7288
7629
|
async function runSweep(state, driver, config) {
|
|
7289
7630
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7418,23 +7759,44 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7418
7759
|
return null;
|
|
7419
7760
|
}
|
|
7420
7761
|
let consecutiveFailures = 0;
|
|
7421
|
-
let
|
|
7762
|
+
let phase = "dormant";
|
|
7422
7763
|
let rearmRequested = false;
|
|
7764
|
+
const armProbe = () => {
|
|
7765
|
+
phase = "probe-pending";
|
|
7766
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7767
|
+
};
|
|
7423
7768
|
const scheduleNextTick = () => {
|
|
7424
|
-
|
|
7425
|
-
|
|
7769
|
+
if (rearmRequested) {
|
|
7770
|
+
rearmRequested = false;
|
|
7771
|
+
armProbe();
|
|
7772
|
+
return;
|
|
7773
|
+
}
|
|
7774
|
+
phase = "steady-pending";
|
|
7426
7775
|
state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
|
|
7427
7776
|
};
|
|
7428
7777
|
const rearm = () => {
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7778
|
+
switch (phase) {
|
|
7779
|
+
case "tick-in-flight":
|
|
7780
|
+
rearmRequested = true;
|
|
7781
|
+
return;
|
|
7782
|
+
case "probe-pending":
|
|
7783
|
+
return;
|
|
7784
|
+
case "steady-pending":
|
|
7785
|
+
if (state.claudeUsageTimer) {
|
|
7786
|
+
clearTimeout(state.claudeUsageTimer);
|
|
7787
|
+
state.claudeUsageTimer = null;
|
|
7788
|
+
}
|
|
7789
|
+
rearmRequested = false;
|
|
7790
|
+
armProbe();
|
|
7791
|
+
return;
|
|
7792
|
+
case "dormant":
|
|
7793
|
+
rearmRequested = false;
|
|
7794
|
+
armProbe();
|
|
7795
|
+
return;
|
|
7432
7796
|
}
|
|
7433
|
-
rearmRequested = false;
|
|
7434
|
-
armed = true;
|
|
7435
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7436
7797
|
};
|
|
7437
7798
|
const tick = async (isProbe) => {
|
|
7799
|
+
phase = "tick-in-flight";
|
|
7438
7800
|
try {
|
|
7439
7801
|
const usage = await getClaudeUsage();
|
|
7440
7802
|
const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
|
|
@@ -7476,7 +7838,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7476
7838
|
level: "debug",
|
|
7477
7839
|
message: `Claude usage reporting: ${error2.message}`
|
|
7478
7840
|
});
|
|
7479
|
-
|
|
7841
|
+
phase = "dormant";
|
|
7480
7842
|
if (rearmRequested) rearm();
|
|
7481
7843
|
} else {
|
|
7482
7844
|
logActivity(state, {
|
|
@@ -7498,8 +7860,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7498
7860
|
}
|
|
7499
7861
|
}
|
|
7500
7862
|
};
|
|
7501
|
-
|
|
7502
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7863
|
+
armProbe();
|
|
7503
7864
|
return rearm;
|
|
7504
7865
|
}
|
|
7505
7866
|
async function notifyOffline(state) {
|