@evident-ai/cli 3.3.1-dev.a18f9b8 → 3.3.1-dev.b5645b1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/index.js +638 -81
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -746,6 +746,32 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
746
746
|
return { ok: false, error: describeBestEffortError(error2) };
|
|
747
747
|
}
|
|
748
748
|
}
|
|
749
|
+
async function reportResourceUsage(agentId, authHeader, usage) {
|
|
750
|
+
try {
|
|
751
|
+
const apiUrl = getApiUrlConfig();
|
|
752
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/resource-usage`, {
|
|
753
|
+
method: "POST",
|
|
754
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
755
|
+
body: JSON.stringify({
|
|
756
|
+
cpu_percent: usage.cpuPercent,
|
|
757
|
+
cpu_count: usage.cpuCount,
|
|
758
|
+
memory_total_bytes: usage.memoryTotalBytes,
|
|
759
|
+
memory_available_bytes: usage.memoryAvailableBytes
|
|
760
|
+
}),
|
|
761
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
762
|
+
});
|
|
763
|
+
if (!response.ok) {
|
|
764
|
+
const serverMessage = await readErrorMessage(response);
|
|
765
|
+
return {
|
|
766
|
+
ok: false,
|
|
767
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
return { ok: true };
|
|
771
|
+
} catch (error2) {
|
|
772
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
773
|
+
}
|
|
774
|
+
}
|
|
749
775
|
async function getAgentInfo(agentId, authHeader) {
|
|
750
776
|
const apiUrl = getApiUrlConfig();
|
|
751
777
|
try {
|
|
@@ -934,6 +960,7 @@ import { homedir } from "os";
|
|
|
934
960
|
import { join } from "path";
|
|
935
961
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
936
962
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
963
|
+
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
937
964
|
function parseClaudeCliCredentials(raw) {
|
|
938
965
|
let parsed;
|
|
939
966
|
try {
|
|
@@ -967,7 +994,7 @@ function readClaudeCliCredentials() {
|
|
|
967
994
|
}
|
|
968
995
|
}
|
|
969
996
|
try {
|
|
970
|
-
const raw = readFileSync(join(homedir(),
|
|
997
|
+
const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
|
|
971
998
|
return parseClaudeCliCredentials(raw);
|
|
972
999
|
} catch (err) {
|
|
973
1000
|
const code = err.code;
|
|
@@ -1063,7 +1090,7 @@ async function claudeUsage() {
|
|
|
1063
1090
|
|
|
1064
1091
|
// src/commands/run.ts
|
|
1065
1092
|
import { homedir as homedir3 } from "os";
|
|
1066
|
-
import { isAbsolute as isAbsolute2, join as
|
|
1093
|
+
import { isAbsolute as isAbsolute2, join as join5, parse, resolve as resolvePath } from "path";
|
|
1067
1094
|
import chalk6 from "chalk";
|
|
1068
1095
|
|
|
1069
1096
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1700,13 +1727,22 @@ function buildNoProviderWarning(hasProvider) {
|
|
|
1700
1727
|
return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
|
|
1701
1728
|
}
|
|
1702
1729
|
|
|
1730
|
+
// src/lib/http-timeout.ts
|
|
1731
|
+
var REQUEST_TIMEOUT_MS = 6e4;
|
|
1732
|
+
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1733
|
+
return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1703
1736
|
// src/lib/opencode/session.ts
|
|
1737
|
+
function timedFetch(input, init) {
|
|
1738
|
+
return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
|
|
1739
|
+
}
|
|
1704
1740
|
function opencodeBase(port) {
|
|
1705
1741
|
return `http://127.0.0.1:${port}`;
|
|
1706
1742
|
}
|
|
1707
1743
|
async function getOpenCodeDirectory(port) {
|
|
1708
1744
|
try {
|
|
1709
|
-
const res = await
|
|
1745
|
+
const res = await timedFetch(`${opencodeBase(port)}/path`);
|
|
1710
1746
|
if (!res.ok) return null;
|
|
1711
1747
|
const body = await res.json();
|
|
1712
1748
|
const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
|
|
@@ -1757,7 +1793,7 @@ function isAssistantInFlight(m) {
|
|
|
1757
1793
|
}
|
|
1758
1794
|
async function getSessionMessages(port, sessionId) {
|
|
1759
1795
|
try {
|
|
1760
|
-
const res = await
|
|
1796
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1761
1797
|
if (!res.ok) return null;
|
|
1762
1798
|
const body = await res.json();
|
|
1763
1799
|
return Array.isArray(body) ? body : null;
|
|
@@ -1787,7 +1823,7 @@ function sessionLastActivityMs(session) {
|
|
|
1787
1823
|
}
|
|
1788
1824
|
async function listSessions(port) {
|
|
1789
1825
|
try {
|
|
1790
|
-
const res = await
|
|
1826
|
+
const res = await timedFetch(`${opencodeBase(port)}/session`);
|
|
1791
1827
|
if (!res.ok) return null;
|
|
1792
1828
|
const body = await res.json();
|
|
1793
1829
|
return Array.isArray(body) ? body : null;
|
|
@@ -1797,7 +1833,7 @@ async function listSessions(port) {
|
|
|
1797
1833
|
}
|
|
1798
1834
|
async function deleteSession(port, id) {
|
|
1799
1835
|
try {
|
|
1800
|
-
const res = await
|
|
1836
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1801
1837
|
return res.status >= 200 && res.status < 300;
|
|
1802
1838
|
} catch {
|
|
1803
1839
|
return false;
|
|
@@ -1805,7 +1841,7 @@ async function deleteSession(port, id) {
|
|
|
1805
1841
|
}
|
|
1806
1842
|
async function sessionExists(port, id) {
|
|
1807
1843
|
try {
|
|
1808
|
-
const res = await
|
|
1844
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
|
|
1809
1845
|
if (res.status >= 200 && res.status < 300) return true;
|
|
1810
1846
|
if (res.status === 404) return false;
|
|
1811
1847
|
return null;
|
|
@@ -1815,7 +1851,7 @@ async function sessionExists(port, id) {
|
|
|
1815
1851
|
}
|
|
1816
1852
|
async function getSessionStatuses(port) {
|
|
1817
1853
|
try {
|
|
1818
|
-
const res = await
|
|
1854
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/status`);
|
|
1819
1855
|
if (!res.ok) {
|
|
1820
1856
|
console.error(
|
|
1821
1857
|
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
@@ -1848,7 +1884,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1848
1884
|
if (directory && directory.trim()) {
|
|
1849
1885
|
url.searchParams.set("directory", directory.trim());
|
|
1850
1886
|
}
|
|
1851
|
-
const response = await
|
|
1887
|
+
const response = await timedFetch(url, {
|
|
1852
1888
|
method: "POST",
|
|
1853
1889
|
headers: { "Content-Type": "application/json" },
|
|
1854
1890
|
body: JSON.stringify({})
|
|
@@ -1862,7 +1898,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1862
1898
|
}
|
|
1863
1899
|
async function getModelAttachmentCapability(port, model) {
|
|
1864
1900
|
try {
|
|
1865
|
-
const res = await
|
|
1901
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1866
1902
|
if (!res.ok) {
|
|
1867
1903
|
console.error(
|
|
1868
1904
|
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -1995,7 +2031,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1995
2031
|
};
|
|
1996
2032
|
}
|
|
1997
2033
|
}
|
|
1998
|
-
const res = await
|
|
2034
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1999
2035
|
method: "POST",
|
|
2000
2036
|
headers: { "Content-Type": "application/json" },
|
|
2001
2037
|
body: JSON.stringify(body)
|
|
@@ -2244,7 +2280,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
2244
2280
|
}
|
|
2245
2281
|
async function hasAnyConfiguredProvider(port) {
|
|
2246
2282
|
try {
|
|
2247
|
-
const res = await
|
|
2283
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2248
2284
|
if (!res.ok) {
|
|
2249
2285
|
console.error(
|
|
2250
2286
|
`[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -2952,6 +2988,21 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
2952
2988
|
}
|
|
2953
2989
|
}
|
|
2954
2990
|
|
|
2991
|
+
// src/lib/reporting-schedule.ts
|
|
2992
|
+
function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
2993
|
+
const jitterRangeMs = baseMs * jitterFraction;
|
|
2994
|
+
return baseMs - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
2995
|
+
}
|
|
2996
|
+
function firstReportDelayMs(random = Math.random) {
|
|
2997
|
+
return 5e3 + random() * 1e4;
|
|
2998
|
+
}
|
|
2999
|
+
function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
|
|
3000
|
+
return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
|
|
3001
|
+
}
|
|
3002
|
+
function failureStreakSuffix(consecutiveFailures) {
|
|
3003
|
+
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
3004
|
+
}
|
|
3005
|
+
|
|
2955
3006
|
// src/lib/claude-usage-reporting.ts
|
|
2956
3007
|
var VALID_MODES = ["auto", "on", "off"];
|
|
2957
3008
|
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
@@ -2974,18 +3025,79 @@ function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
|
2974
3025
|
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
2975
3026
|
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
2976
3027
|
function nextReportDelayMs(random = Math.random) {
|
|
2977
|
-
|
|
2978
|
-
return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
3028
|
+
return jitteredDelayMs(BASE_REPORT_DELAY_MS, REPORT_DELAY_JITTER_FRACTION, random);
|
|
2979
3029
|
}
|
|
2980
|
-
var FIRST_REPORT_DELAY_MS =
|
|
3030
|
+
var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
|
|
2981
3031
|
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
2982
3032
|
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
2983
|
-
return consecutiveFailures
|
|
3033
|
+
return reportFailureLogLevel(consecutiveFailures, CLAUDE_USAGE_FAILURE_REESCALATION_TICKS);
|
|
3034
|
+
}
|
|
3035
|
+
|
|
3036
|
+
// src/lib/resource-usage-reporting.ts
|
|
3037
|
+
var ENABLED_VALUES = /* @__PURE__ */ new Set(["on", "true", "1"]);
|
|
3038
|
+
var DISABLED_VALUES = /* @__PURE__ */ new Set(["off", "false", "0"]);
|
|
3039
|
+
function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
3040
|
+
if (flagValue === false) {
|
|
3041
|
+
return { enabled: false, warnings: [] };
|
|
3042
|
+
}
|
|
3043
|
+
const raw = env.EVIDENT_RESOURCE_USAGE_REPORTING;
|
|
3044
|
+
if (raw === void 0 || raw === "") {
|
|
3045
|
+
return { enabled: true, warnings: [] };
|
|
3046
|
+
}
|
|
3047
|
+
const normalized = raw.trim().toLowerCase();
|
|
3048
|
+
if (DISABLED_VALUES.has(normalized)) {
|
|
3049
|
+
return { enabled: false, warnings: [] };
|
|
3050
|
+
}
|
|
3051
|
+
if (ENABLED_VALUES.has(normalized)) {
|
|
3052
|
+
return { enabled: true, warnings: [] };
|
|
3053
|
+
}
|
|
3054
|
+
return {
|
|
3055
|
+
enabled: true,
|
|
3056
|
+
warnings: [
|
|
3057
|
+
`Ignoring invalid EVIDENT_RESOURCE_USAGE_REPORTING "${raw}": expected on or off; leaving reporting on`
|
|
3058
|
+
]
|
|
3059
|
+
};
|
|
3060
|
+
}
|
|
3061
|
+
|
|
3062
|
+
// src/lib/resource-usage.ts
|
|
3063
|
+
import { cpus, totalmem, freemem } from "os";
|
|
3064
|
+
function readCpuSample() {
|
|
3065
|
+
let busyMs = 0;
|
|
3066
|
+
let idleMs = 0;
|
|
3067
|
+
for (const cpu of cpus()) {
|
|
3068
|
+
busyMs += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.irq;
|
|
3069
|
+
idleMs += cpu.times.idle;
|
|
3070
|
+
}
|
|
3071
|
+
return { busyMs, idleMs };
|
|
3072
|
+
}
|
|
3073
|
+
function cpuPercentBetween(previous, current) {
|
|
3074
|
+
const deltaBusy = current.busyMs - previous.busyMs;
|
|
3075
|
+
const deltaIdle = current.idleMs - previous.idleMs;
|
|
3076
|
+
const total = deltaBusy + deltaIdle;
|
|
3077
|
+
if (total === 0) return null;
|
|
3078
|
+
return Math.round((deltaBusy / total * 100 + Number.EPSILON) * 100) / 100;
|
|
3079
|
+
}
|
|
3080
|
+
function createResourceUsageCollector() {
|
|
3081
|
+
let previous = readCpuSample();
|
|
3082
|
+
return () => {
|
|
3083
|
+
const current = readCpuSample();
|
|
3084
|
+
const cpuPercent = cpuPercentBetween(previous, current);
|
|
3085
|
+
previous = current;
|
|
3086
|
+
return {
|
|
3087
|
+
cpuPercent,
|
|
3088
|
+
cpuCount: cpus().length,
|
|
3089
|
+
memoryTotalBytes: totalmem(),
|
|
3090
|
+
memoryAvailableBytes: freemem()
|
|
3091
|
+
};
|
|
3092
|
+
};
|
|
2984
3093
|
}
|
|
2985
3094
|
|
|
2986
3095
|
// src/lib/channels/driver.ts
|
|
2987
3096
|
import { homedir as homedir2 } from "os";
|
|
2988
3097
|
|
|
3098
|
+
// src/lib/runner-file-sync.ts
|
|
3099
|
+
import { join as join4 } from "path";
|
|
3100
|
+
|
|
2989
3101
|
// src/lib/file-push.ts
|
|
2990
3102
|
import { randomUUID } from "crypto";
|
|
2991
3103
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
@@ -3179,17 +3291,20 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3179
3291
|
for (const id of options.ackFailures.keys()) {
|
|
3180
3292
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3181
3293
|
}
|
|
3182
|
-
if (pending.length === 0) return 0;
|
|
3294
|
+
if (pending.length === 0) return { applied: 0, claudeCredentialApplied: false };
|
|
3183
3295
|
options.log({
|
|
3184
3296
|
level: "info",
|
|
3185
3297
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3186
3298
|
});
|
|
3187
3299
|
let applied = 0;
|
|
3300
|
+
let claudeCredentialApplied = false;
|
|
3188
3301
|
for (const file of pending) {
|
|
3189
3302
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3190
|
-
|
|
3303
|
+
const outcome = await applyOne(options, file);
|
|
3304
|
+
if (outcome.applied) applied += 1;
|
|
3305
|
+
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
3191
3306
|
}
|
|
3192
|
-
return applied;
|
|
3307
|
+
return { applied, claudeCredentialApplied };
|
|
3193
3308
|
}
|
|
3194
3309
|
async function listPendingFiles(options) {
|
|
3195
3310
|
let res;
|
|
@@ -3250,6 +3365,11 @@ function asPendingFile(entry) {
|
|
|
3250
3365
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3251
3366
|
return { id, path, size };
|
|
3252
3367
|
}
|
|
3368
|
+
var NOT_APPLIED = { applied: false, claudeCredentialApplied: false };
|
|
3369
|
+
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3370
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join4(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3371
|
+
return expanded === join4(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3372
|
+
}
|
|
3253
3373
|
async function applyOne(options, file) {
|
|
3254
3374
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
3255
3375
|
if (options.allowedDirectories.length === 0) {
|
|
@@ -3258,7 +3378,7 @@ async function applyOne(options, file) {
|
|
|
3258
3378
|
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
3259
3379
|
});
|
|
3260
3380
|
await ack(options, file, "rejected", "file_sync_disabled");
|
|
3261
|
-
return
|
|
3381
|
+
return NOT_APPLIED;
|
|
3262
3382
|
}
|
|
3263
3383
|
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
3264
3384
|
options.log({
|
|
@@ -3266,12 +3386,12 @@ async function applyOne(options, file) {
|
|
|
3266
3386
|
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
3267
3387
|
});
|
|
3268
3388
|
await ack(options, file, "rejected", "file_too_large");
|
|
3269
|
-
return
|
|
3389
|
+
return NOT_APPLIED;
|
|
3270
3390
|
}
|
|
3271
3391
|
const download = await downloadContent(options, file, label);
|
|
3272
3392
|
if (!download.ok) {
|
|
3273
3393
|
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
3274
|
-
return
|
|
3394
|
+
return NOT_APPLIED;
|
|
3275
3395
|
}
|
|
3276
3396
|
let outcome;
|
|
3277
3397
|
try {
|
|
@@ -3287,7 +3407,7 @@ async function applyOne(options, file) {
|
|
|
3287
3407
|
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
3288
3408
|
});
|
|
3289
3409
|
await ack(options, file, "rejected", "write_failed");
|
|
3290
|
-
return
|
|
3410
|
+
return NOT_APPLIED;
|
|
3291
3411
|
}
|
|
3292
3412
|
if (!outcome.ok) {
|
|
3293
3413
|
options.log({
|
|
@@ -3295,14 +3415,17 @@ async function applyOne(options, file) {
|
|
|
3295
3415
|
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
3296
3416
|
});
|
|
3297
3417
|
await ack(options, file, "rejected", outcome.code);
|
|
3298
|
-
return
|
|
3418
|
+
return NOT_APPLIED;
|
|
3299
3419
|
}
|
|
3300
3420
|
options.log({
|
|
3301
3421
|
level: "info",
|
|
3302
3422
|
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
3303
3423
|
});
|
|
3304
3424
|
await ack(options, file, "applied");
|
|
3305
|
-
return
|
|
3425
|
+
return {
|
|
3426
|
+
applied: true,
|
|
3427
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir)
|
|
3428
|
+
};
|
|
3306
3429
|
}
|
|
3307
3430
|
function durableDownloadCode(status2) {
|
|
3308
3431
|
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
@@ -3413,8 +3536,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
|
3413
3536
|
var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
|
|
3414
3537
|
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
3415
3538
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
3539
|
+
var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
|
|
3540
|
+
var MAX_WATCHER_STALL_RESTARTS = 3;
|
|
3541
|
+
var MAX_RELEASED_OPENCODE_IDS = 256;
|
|
3416
3542
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3417
3543
|
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
3544
|
+
var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
|
|
3545
|
+
var MAX_WEDGED_CONVERSATIONS = 256;
|
|
3418
3546
|
var ChannelAuthError = class extends Error {
|
|
3419
3547
|
constructor(message) {
|
|
3420
3548
|
super(message);
|
|
@@ -3458,6 +3586,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3458
3586
|
fileSyncDirectories;
|
|
3459
3587
|
homeDir;
|
|
3460
3588
|
maxActiveSessions;
|
|
3589
|
+
watcherStallMs;
|
|
3590
|
+
wedgeWarningIntervalMs;
|
|
3461
3591
|
/** Cache of conversationId → opencode sessionId. */
|
|
3462
3592
|
sessions = /* @__PURE__ */ new Map();
|
|
3463
3593
|
/**
|
|
@@ -3488,6 +3618,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3488
3618
|
* bounded cost.
|
|
3489
3619
|
*/
|
|
3490
3620
|
supersededSessions = /* @__PURE__ */ new Map();
|
|
3621
|
+
/**
|
|
3622
|
+
* Local re-drive fence for a message force-released by the stall watchdog
|
|
3623
|
+
* (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
|
|
3624
|
+
* see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
|
|
3625
|
+
* is `null` for exactly this shape (its `markProcessing` never landed), so
|
|
3626
|
+
* without a local record of the id the driver last knew, the next drain's
|
|
3627
|
+
* `if (message.opencode_message_id)` re-drive-fence check at
|
|
3628
|
+
* `processConversation` would not engage and it would blind-`prompt_async`
|
|
3629
|
+
* a turn that may still be running in opencode — the one duplicate-turn
|
|
3630
|
+
* hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
|
|
3631
|
+
* `processConversation` reads `message.opencode_message_id ?? this
|
|
3632
|
+
* .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
|
|
3633
|
+
* threads it into `resolveRedrive`, which asks opencode itself whether the
|
|
3634
|
+
* turn is still ongoing before ever dispatching.
|
|
3635
|
+
*
|
|
3636
|
+
* Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
|
|
3637
|
+
* `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
|
|
3638
|
+
* non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
|
|
3639
|
+
* dispatch) and at the top-level fresh-dispatch site, so it does not outlive
|
|
3640
|
+
* the row it was recorded for.
|
|
3641
|
+
*/
|
|
3642
|
+
releasedOpencodeIds = /* @__PURE__ */ new Map();
|
|
3643
|
+
/**
|
|
3644
|
+
* Per-conversation throttle state for the #183 recurrence warning (#1618
|
|
3645
|
+
* WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
|
|
3646
|
+
* `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
|
|
3647
|
+
* log line and the `dispatch_wedged` signal to at most once per
|
|
3648
|
+
* `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
|
|
3649
|
+
* so the operator sees magnitude, not repetition. Cleared the moment the
|
|
3650
|
+
* conversation dispatches anything (a fresh wedge, if it recurs, is a new
|
|
3651
|
+
* incident). Bounded FIFO, mirroring `supersededSessions`
|
|
3652
|
+
* (`MAX_WEDGED_CONVERSATIONS`).
|
|
3653
|
+
*/
|
|
3654
|
+
wedgeWarnings = /* @__PURE__ */ new Map();
|
|
3491
3655
|
/**
|
|
3492
3656
|
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
3493
3657
|
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
@@ -3706,6 +3870,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3706
3870
|
* same trick `lastProxiedActivityAt` uses.
|
|
3707
3871
|
*/
|
|
3708
3872
|
appliedFileCount = 0;
|
|
3873
|
+
/**
|
|
3874
|
+
* Generation counter, NOT a tally (#1656): advances by exactly one per sync
|
|
3875
|
+
* batch that applied the Claude CLI credential file, not by how many
|
|
3876
|
+
* credential files were in that batch. `run.ts` only ever tests inequality
|
|
3877
|
+
* against the value it saw last cycle, so magnitude is meaningless — keep it
|
|
3878
|
+
* that way rather than "fixing" it into a count.
|
|
3879
|
+
*/
|
|
3880
|
+
claudeCredentialApplyCount = 0;
|
|
3709
3881
|
/**
|
|
3710
3882
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3711
3883
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -3730,7 +3902,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3730
3902
|
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3731
3903
|
this.log = config.log ?? (() => {
|
|
3732
3904
|
});
|
|
3733
|
-
this.fetchImpl =
|
|
3905
|
+
this.fetchImpl = withRequestTimeout(
|
|
3906
|
+
config.fetchImpl ?? fetch,
|
|
3907
|
+
config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
|
|
3908
|
+
);
|
|
3734
3909
|
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3735
3910
|
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3736
3911
|
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
@@ -3739,6 +3914,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3739
3914
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3740
3915
|
this.homeDir = config.homeDir ?? homedir2();
|
|
3741
3916
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
3917
|
+
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
3918
|
+
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
3742
3919
|
}
|
|
3743
3920
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
3744
3921
|
get opencodeBase() {
|
|
@@ -3752,6 +3929,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3752
3929
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
3753
3930
|
*/
|
|
3754
3931
|
async drainPending() {
|
|
3932
|
+
try {
|
|
3933
|
+
this.reconcileWatchers();
|
|
3934
|
+
} catch (err) {
|
|
3935
|
+
this.log({
|
|
3936
|
+
level: "error",
|
|
3937
|
+
message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
|
|
3938
|
+
});
|
|
3939
|
+
}
|
|
3755
3940
|
if (this.stopped) return 0;
|
|
3756
3941
|
if (this.draining) return 0;
|
|
3757
3942
|
this.draining = true;
|
|
@@ -3785,7 +3970,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3785
3970
|
if (this.syncingFiles) return 0;
|
|
3786
3971
|
this.syncingFiles = true;
|
|
3787
3972
|
try {
|
|
3788
|
-
const
|
|
3973
|
+
const result = await syncPendingRunnerFiles({
|
|
3789
3974
|
agentId: this.agentId,
|
|
3790
3975
|
apiUrl: this.apiUrl,
|
|
3791
3976
|
getAuthHeader: this.getAuthHeader,
|
|
@@ -3795,8 +3980,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3795
3980
|
ackFailures: this.fileAckFailures,
|
|
3796
3981
|
log: this.log
|
|
3797
3982
|
});
|
|
3798
|
-
this.appliedFileCount += applied;
|
|
3799
|
-
|
|
3983
|
+
this.appliedFileCount += result.applied;
|
|
3984
|
+
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
3985
|
+
return result.applied;
|
|
3800
3986
|
} catch (err) {
|
|
3801
3987
|
this.log({
|
|
3802
3988
|
level: "error",
|
|
@@ -3887,12 +4073,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3887
4073
|
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3888
4074
|
* idle checks still shows up as an advance.
|
|
3889
4075
|
*
|
|
4076
|
+
* A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
|
|
4077
|
+
* (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
|
|
4078
|
+
* reporting on it advancing, so an unrelated file sync can never disturb a
|
|
4079
|
+
* healthy reporting cadence (#1627) — it never even reaches that trigger, let
|
|
4080
|
+
* alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
|
|
4081
|
+
* whose consumer is idle-timeout suppression and must key on ANY file, not
|
|
4082
|
+
* just a Claude credential.
|
|
4083
|
+
*
|
|
3890
4084
|
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3891
4085
|
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3892
4086
|
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3893
4087
|
*/
|
|
3894
4088
|
fileSyncActivity() {
|
|
3895
|
-
return {
|
|
4089
|
+
return {
|
|
4090
|
+
appliedFiles: this.appliedFileCount,
|
|
4091
|
+
inFlight: this.syncingFiles,
|
|
4092
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount
|
|
4093
|
+
};
|
|
3896
4094
|
}
|
|
3897
4095
|
/**
|
|
3898
4096
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
@@ -4005,8 +4203,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4005
4203
|
skippedAlreadyDispatched += 1;
|
|
4006
4204
|
continue;
|
|
4007
4205
|
}
|
|
4008
|
-
|
|
4009
|
-
|
|
4206
|
+
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
4207
|
+
if (effectiveOpencodeMessageId) {
|
|
4208
|
+
const outcome = await this.resolveRedrive(
|
|
4209
|
+
conv,
|
|
4210
|
+
sessionId,
|
|
4211
|
+
message,
|
|
4212
|
+
sessionCreated,
|
|
4213
|
+
effectiveOpencodeMessageId
|
|
4214
|
+
);
|
|
4010
4215
|
if (outcome === "abandoned") {
|
|
4011
4216
|
continue;
|
|
4012
4217
|
}
|
|
@@ -4117,21 +4322,102 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4117
4322
|
}
|
|
4118
4323
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4119
4324
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
4325
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
4120
4326
|
this.dispatched.add(message.id);
|
|
4121
4327
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
4122
4328
|
dispatched += 1;
|
|
4123
4329
|
void this.postSignal(conv.id, message.id, "dispatched");
|
|
4124
4330
|
}
|
|
4125
4331
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
4126
|
-
this.
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
conversation_id: conv.id
|
|
4130
|
-
});
|
|
4332
|
+
this.reportWedgedConversation(conv, messages);
|
|
4333
|
+
} else if (dispatched > 0) {
|
|
4334
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4131
4335
|
}
|
|
4132
4336
|
this.ensureWatcherRunning(sessionId);
|
|
4133
4337
|
return dispatched;
|
|
4134
4338
|
}
|
|
4339
|
+
/**
|
|
4340
|
+
* The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
|
|
4341
|
+
* escalated (#1618 WI-4). `messages` is the conversation's full pending list
|
|
4342
|
+
* on THIS tick — the caller has already confirmed every one of them is a
|
|
4343
|
+
* skip-because-already-`dispatched`, the exact signature of a message stuck
|
|
4344
|
+
* acknowledged-but-never-worked.
|
|
4345
|
+
*
|
|
4346
|
+
* Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
|
|
4347
|
+
* (52,843 occurrences observed in one incident) — burning the GLOBAL
|
|
4348
|
+
* 30-events/60s `runner-activity-telemetry.ts` budget that was itself
|
|
4349
|
+
* suppressing the diagnostics needed to debug the wedge. The `warn` log (and
|
|
4350
|
+
* the `dispatch_wedged` signal once the wedge has persisted past the same
|
|
4351
|
+
* interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
|
|
4352
|
+
* naming the consecutive-tick count so the operator sees magnitude rather
|
|
4353
|
+
* than repetition.
|
|
4354
|
+
*
|
|
4355
|
+
* Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
|
|
4356
|
+
* unconditionally on this same tick and is already recovering anything it
|
|
4357
|
+
* can see. This is reporting only — see `countUntrackedIds`'s doc for the
|
|
4358
|
+
* one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
|
|
4359
|
+
*/
|
|
4360
|
+
reportWedgedConversation(conv, messages) {
|
|
4361
|
+
const now = this.now();
|
|
4362
|
+
const existing = this.wedgeWarnings.get(conv.id);
|
|
4363
|
+
const firstWedgedAt = existing?.firstWedgedAt ?? now;
|
|
4364
|
+
const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
|
|
4365
|
+
const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
|
|
4366
|
+
if (!dueForWarn) {
|
|
4367
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4368
|
+
this.wedgeWarnings.set(conv.id, {
|
|
4369
|
+
firstWedgedAt,
|
|
4370
|
+
lastWarnedAt: existing.lastWarnedAt,
|
|
4371
|
+
consecutiveTicks
|
|
4372
|
+
});
|
|
4373
|
+
return;
|
|
4374
|
+
}
|
|
4375
|
+
const stuckForMs = now - firstWedgedAt;
|
|
4376
|
+
const untracked = this.countUntrackedIds(messages);
|
|
4377
|
+
this.log({
|
|
4378
|
+
level: "warn",
|
|
4379
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
4380
|
+
conversation_id: conv.id
|
|
4381
|
+
});
|
|
4382
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4383
|
+
this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
|
|
4384
|
+
while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
|
|
4385
|
+
const oldest = this.wedgeWarnings.keys().next().value;
|
|
4386
|
+
if (oldest === void 0) break;
|
|
4387
|
+
this.wedgeWarnings.delete(oldest);
|
|
4388
|
+
}
|
|
4389
|
+
if (stuckForMs >= this.wedgeWarningIntervalMs) {
|
|
4390
|
+
void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
|
|
4391
|
+
stuck_for_ms: stuckForMs,
|
|
4392
|
+
untracked
|
|
4393
|
+
});
|
|
4394
|
+
}
|
|
4395
|
+
}
|
|
4396
|
+
/**
|
|
4397
|
+
* How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
|
|
4398
|
+
* WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
|
|
4399
|
+
* `dispatched`/`inFlight` pairing invariant holds by construction across
|
|
4400
|
+
* every `dispatched.add` site (see its own doc comment), so `> 0` here means
|
|
4401
|
+
* that invariant has actually been violated for this conversation: there is
|
|
4402
|
+
* no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
|
|
4403
|
+
* `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
|
|
4404
|
+
* recovering. One pass over `this.watchers`, called only when the throttled
|
|
4405
|
+
* warning above is due to fire — not every tick.
|
|
4406
|
+
*/
|
|
4407
|
+
countUntrackedIds(messages) {
|
|
4408
|
+
let untracked = 0;
|
|
4409
|
+
for (const message of messages) {
|
|
4410
|
+
let tracked = false;
|
|
4411
|
+
for (const watcher of this.watchers.values()) {
|
|
4412
|
+
if (watcher.inFlight.has(message.id)) {
|
|
4413
|
+
tracked = true;
|
|
4414
|
+
break;
|
|
4415
|
+
}
|
|
4416
|
+
}
|
|
4417
|
+
if (!tracked) untracked += 1;
|
|
4418
|
+
}
|
|
4419
|
+
return untracked;
|
|
4420
|
+
}
|
|
4135
4421
|
/**
|
|
4136
4422
|
* Poll a session's message list for the re-drive fence (#965), via the
|
|
4137
4423
|
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
@@ -4200,9 +4486,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4200
4486
|
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
4201
4487
|
* other failure resolves to `unresolved` and is retried whole on the next
|
|
4202
4488
|
* ~2s drain tick.
|
|
4489
|
+
*
|
|
4490
|
+
* `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
|
|
4491
|
+
* server `opencode_message_id` when present, else the stall watchdog's local
|
|
4492
|
+
* `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
|
|
4493
|
+
* `message` so every line below — and the signals this method posts —
|
|
4494
|
+
* keeps reporting the REAL server row; a shadow-copied `message` would
|
|
4495
|
+
* silently diverge from it.
|
|
4203
4496
|
*/
|
|
4204
|
-
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
4205
|
-
const ocId =
|
|
4497
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
|
|
4498
|
+
const ocId = effectiveOpencodeMessageId;
|
|
4206
4499
|
if (sessionCreated) {
|
|
4207
4500
|
this.clearRedriveUnresolved(message.id);
|
|
4208
4501
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
@@ -4432,7 +4725,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4432
4725
|
}
|
|
4433
4726
|
return "unresolved";
|
|
4434
4727
|
}
|
|
4435
|
-
/**
|
|
4728
|
+
/**
|
|
4729
|
+
* Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
|
|
4730
|
+
* outcome) — including the stall watchdog's local re-drive fence (#1618): once
|
|
4731
|
+
* `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
|
|
4732
|
+
* a real server-side `opencode_message_id` again or is no longer pending, so
|
|
4733
|
+
* the fence entry is no longer needed.
|
|
4734
|
+
*/
|
|
4436
4735
|
clearRedriveUnresolved(messageId) {
|
|
4437
4736
|
this.redriveUnresolvedSince.delete(messageId);
|
|
4438
4737
|
this.redriveUnresolvedSignalled.delete(messageId);
|
|
@@ -4440,6 +4739,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4440
4739
|
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4441
4740
|
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4442
4741
|
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
4742
|
+
this.releasedOpencodeIds.delete(messageId);
|
|
4443
4743
|
}
|
|
4444
4744
|
/**
|
|
4445
4745
|
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
@@ -4594,6 +4894,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4594
4894
|
this.supersededSessions.delete(oldest);
|
|
4595
4895
|
}
|
|
4596
4896
|
}
|
|
4897
|
+
/**
|
|
4898
|
+
* Record the local re-drive fence for a message force-released without
|
|
4899
|
+
* completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
|
|
4900
|
+
* `removeInFlight`, which is about to drop the `InFlightMessage` this reads
|
|
4901
|
+
* `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
|
|
4902
|
+
*/
|
|
4903
|
+
recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
|
|
4904
|
+
this.releasedOpencodeIds.delete(evidentMessageId);
|
|
4905
|
+
this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
|
|
4906
|
+
while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
|
|
4907
|
+
const oldest = this.releasedOpencodeIds.keys().next().value;
|
|
4908
|
+
if (oldest === void 0) return;
|
|
4909
|
+
this.releasedOpencodeIds.delete(oldest);
|
|
4910
|
+
}
|
|
4911
|
+
}
|
|
4597
4912
|
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
4598
4913
|
isSuperseded(conversationId, sessionId) {
|
|
4599
4914
|
return this.supersededSessions.get(conversationId) === sessionId;
|
|
@@ -4635,6 +4950,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4635
4950
|
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
4636
4951
|
conversation_id: conv.id
|
|
4637
4952
|
});
|
|
4953
|
+
const watcher = this.watchers.get(bound);
|
|
4954
|
+
if (watcher) {
|
|
4955
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
4956
|
+
this.recordReleasedOpencodeId(evidentMessageId, bound, inFlight.opencodeMessageId);
|
|
4957
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
4958
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
4959
|
+
recovery: "session_gone_released"
|
|
4960
|
+
});
|
|
4961
|
+
}
|
|
4962
|
+
this.watchers.delete(bound);
|
|
4963
|
+
}
|
|
4638
4964
|
this.sessions.delete(conv.id);
|
|
4639
4965
|
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
4640
4966
|
}
|
|
@@ -4825,15 +5151,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4825
5151
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
4826
5152
|
let watcher = this.watchers.get(sessionId);
|
|
4827
5153
|
if (!watcher) {
|
|
4828
|
-
watcher =
|
|
4829
|
-
conv,
|
|
4830
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4831
|
-
loop: null,
|
|
4832
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4833
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4834
|
-
lastGoodPollAt: this.now(),
|
|
4835
|
-
hadUsablePoll: false
|
|
4836
|
-
};
|
|
5154
|
+
watcher = this.newSessionWatcher(conv);
|
|
4837
5155
|
this.watchers.set(sessionId, watcher);
|
|
4838
5156
|
}
|
|
4839
5157
|
const now = this.now();
|
|
@@ -4864,6 +5182,27 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4864
5182
|
ambiguousResolved: false
|
|
4865
5183
|
});
|
|
4866
5184
|
}
|
|
5185
|
+
/**
|
|
5186
|
+
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
5187
|
+
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
5188
|
+
* misread as stalled by the very first reconciliation that sees it.
|
|
5189
|
+
*/
|
|
5190
|
+
newSessionWatcher(conv) {
|
|
5191
|
+
const now = this.now();
|
|
5192
|
+
return {
|
|
5193
|
+
conv,
|
|
5194
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
5195
|
+
loop: null,
|
|
5196
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
5197
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
5198
|
+
lastGoodPollAt: now,
|
|
5199
|
+
hadUsablePoll: false,
|
|
5200
|
+
generation: 0,
|
|
5201
|
+
lastTickAt: now,
|
|
5202
|
+
lastObservedTickAt: now,
|
|
5203
|
+
consecutiveStallRestarts: 0
|
|
5204
|
+
};
|
|
5205
|
+
}
|
|
4867
5206
|
/**
|
|
4868
5207
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
4869
5208
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
@@ -4893,15 +5232,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4893
5232
|
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
4894
5233
|
let watcher = this.watchers.get(sessionId);
|
|
4895
5234
|
if (!watcher) {
|
|
4896
|
-
watcher =
|
|
4897
|
-
conv,
|
|
4898
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4899
|
-
loop: null,
|
|
4900
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4901
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4902
|
-
lastGoodPollAt: this.now(),
|
|
4903
|
-
hadUsablePoll: false
|
|
4904
|
-
};
|
|
5235
|
+
watcher = this.newSessionWatcher(conv);
|
|
4905
5236
|
this.watchers.set(sessionId, watcher);
|
|
4906
5237
|
}
|
|
4907
5238
|
watcher.inFlight.set(message.id, {
|
|
@@ -4945,12 +5276,110 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4945
5276
|
ambiguousResolved: false
|
|
4946
5277
|
});
|
|
4947
5278
|
}
|
|
5279
|
+
/**
|
|
5280
|
+
* Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
|
|
5281
|
+
* restarts any per-session watcher whose loop has exited or stopped ticking
|
|
5282
|
+
* — escalating to a bounded force-release only once
|
|
5283
|
+
* `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
|
|
5284
|
+
* it. Fully synchronous: it only inspects in-memory state and calls the
|
|
5285
|
+
* synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
|
|
5286
|
+
* run from the very top of `drainPending()` — ahead of the un-timed
|
|
5287
|
+
* `getPendingConversations()` await that would otherwise be able to disable
|
|
5288
|
+
* it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
|
|
5289
|
+
* `drainPending()` from being CALLED again at all, not just from finishing).
|
|
5290
|
+
*
|
|
5291
|
+
* Restarts the loop rather than releasing messages directly: a blind release
|
|
5292
|
+
* would let the next drain re-`prompt_async` a turn that may still be
|
|
5293
|
+
* running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
|
|
5294
|
+
* re-polls with each message's `opencodeMessageId` still in hand and lets
|
|
5295
|
+
* the existing, audited `!activelyRunning` give-up decide, same as it always
|
|
5296
|
+
* has.
|
|
5297
|
+
*
|
|
5298
|
+
* Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
|
|
5299
|
+
* that shape has no in-flight entry and therefore no `opencodeMessageId` to
|
|
5300
|
+
* fence a release with, so releasing it here would blind-re-POST a possibly-
|
|
5301
|
+
* running turn — and there is no conversation id in hand to signal with
|
|
5302
|
+
* either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
|
|
5303
|
+
* instead, where a conversation id already exists. If you find yourself
|
|
5304
|
+
* wanting to add a `dispatched` sweep here, don't — read the drain-wedge
|
|
5305
|
+
* plan's §3/D5 first.
|
|
5306
|
+
*/
|
|
5307
|
+
reconcileWatchers() {
|
|
5308
|
+
const now = this.now();
|
|
5309
|
+
for (const [sessionId, watcher] of [...this.watchers]) {
|
|
5310
|
+
if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
|
|
5311
|
+
watcher.consecutiveStallRestarts = 0;
|
|
5312
|
+
}
|
|
5313
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5314
|
+
if (watcher.inFlight.size === 0 && watcher.loop === null) {
|
|
5315
|
+
this.watchers.delete(sessionId);
|
|
5316
|
+
continue;
|
|
5317
|
+
}
|
|
5318
|
+
if (watcher.loop === null && watcher.inFlight.size > 0) {
|
|
5319
|
+
if (now - watcher.lastTickAt < this.watcherStallMs) continue;
|
|
5320
|
+
this.log({
|
|
5321
|
+
level: "warn",
|
|
5322
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
|
|
5323
|
+
conversation_id: watcher.conv.id
|
|
5324
|
+
});
|
|
5325
|
+
this.ensureWatcherRunning(sessionId);
|
|
5326
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5327
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5328
|
+
recovery: "loop_exited"
|
|
5329
|
+
});
|
|
5330
|
+
}
|
|
5331
|
+
continue;
|
|
5332
|
+
}
|
|
5333
|
+
if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
|
|
5334
|
+
const stalledForMs = now - watcher.lastTickAt;
|
|
5335
|
+
watcher.consecutiveStallRestarts += 1;
|
|
5336
|
+
if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
|
|
5337
|
+
this.log({
|
|
5338
|
+
level: "error",
|
|
5339
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
|
|
5340
|
+
conversation_id: watcher.conv.id
|
|
5341
|
+
});
|
|
5342
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5343
|
+
this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
|
|
5344
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5345
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5346
|
+
recovery: "unrecoverable_released"
|
|
5347
|
+
});
|
|
5348
|
+
}
|
|
5349
|
+
watcher.generation += 1;
|
|
5350
|
+
this.watchers.delete(sessionId);
|
|
5351
|
+
continue;
|
|
5352
|
+
}
|
|
5353
|
+
watcher.generation += 1;
|
|
5354
|
+
watcher.loop = null;
|
|
5355
|
+
watcher.lastGoodPollAt = now;
|
|
5356
|
+
watcher.lastTickAt = now;
|
|
5357
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5358
|
+
this.ensureWatcherRunning(sessionId);
|
|
5359
|
+
this.log({
|
|
5360
|
+
level: "warn",
|
|
5361
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
|
|
5362
|
+
conversation_id: watcher.conv.id
|
|
5363
|
+
});
|
|
5364
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5365
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5366
|
+
recovery: "loop_stalled"
|
|
5367
|
+
});
|
|
5368
|
+
}
|
|
5369
|
+
}
|
|
5370
|
+
}
|
|
5371
|
+
}
|
|
4948
5372
|
/**
|
|
4949
5373
|
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
4950
5374
|
* work and is not already running. Single-flight per session. The loop is
|
|
4951
5375
|
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
4952
5376
|
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
4953
5377
|
* stays as the safety net.
|
|
5378
|
+
*
|
|
5379
|
+
* The generation started here (#1618) is captured in the `.finally` closure
|
|
5380
|
+
* so a RETIRED loop settling late — after `reconcileWatchers` has already
|
|
5381
|
+
* restarted this watcher under a newer generation — can neither null the new
|
|
5382
|
+
* loop's handle nor delete a watcher that still has live work.
|
|
4954
5383
|
*/
|
|
4955
5384
|
ensureWatcherRunning(sessionId) {
|
|
4956
5385
|
const watcher = this.watchers.get(sessionId);
|
|
@@ -4960,7 +5389,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4960
5389
|
this.watchers.delete(sessionId);
|
|
4961
5390
|
return;
|
|
4962
5391
|
}
|
|
4963
|
-
const
|
|
5392
|
+
const generation = watcher.generation;
|
|
5393
|
+
const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
|
|
5394
|
+
if (watcher.generation !== generation) return;
|
|
4964
5395
|
watcher.loop = null;
|
|
4965
5396
|
if (watcher.inFlight.size === 0) {
|
|
4966
5397
|
this.watchers.delete(sessionId);
|
|
@@ -4980,11 +5411,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4980
5411
|
* `source_message_id`;
|
|
4981
5412
|
* 4. drops messages that completed or timed out from the in-flight set.
|
|
4982
5413
|
* Exits when the in-flight set empties. Never throws.
|
|
5414
|
+
*
|
|
5415
|
+
* `generation` (#1618) is the incarnation this call was started under.
|
|
5416
|
+
* `reconcileWatchers` can restart a stalled loop by bumping
|
|
5417
|
+
* `watcher.generation` and starting a NEW `runWatcherLoop` over the same
|
|
5418
|
+
* `SessionWatcher` object — the stalled promise itself cannot be cancelled,
|
|
5419
|
+
* so this loop instead checks at the top of every iteration, right after
|
|
5420
|
+
* waking from `sleep`, and right before servicing any message, and quietly
|
|
5421
|
+
* retires (returns without touching anything) the moment it is no longer the
|
|
5422
|
+
* watcher's current generation. Retiring mid-tick can still let ONE
|
|
5423
|
+
* `serviceInFlightMessage` pass complete first — acceptable, since that
|
|
5424
|
+
* method contains no non-idempotent action.
|
|
4983
5425
|
*/
|
|
4984
|
-
async runWatcherLoop(sessionId, watcher) {
|
|
5426
|
+
async runWatcherLoop(sessionId, watcher, generation) {
|
|
4985
5427
|
try {
|
|
4986
5428
|
while (watcher.inFlight.size > 0) {
|
|
5429
|
+
if (watcher.generation !== generation) return;
|
|
5430
|
+
watcher.lastTickAt = this.now();
|
|
4987
5431
|
await this.sleep(this.pausedPollIntervalMs);
|
|
5432
|
+
if (watcher.generation !== generation) return;
|
|
4988
5433
|
let messages = null;
|
|
4989
5434
|
try {
|
|
4990
5435
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
@@ -5005,6 +5450,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5005
5450
|
}
|
|
5006
5451
|
}
|
|
5007
5452
|
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
5453
|
+
if (watcher.generation !== generation) return;
|
|
5008
5454
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
5009
5455
|
await this.serviceInFlightMessage(
|
|
5010
5456
|
sessionId,
|
|
@@ -6980,7 +7426,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
6980
7426
|
if (trimmed === "") {
|
|
6981
7427
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6982
7428
|
}
|
|
6983
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7429
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join5(homeDir, trimmed.slice(2)) : trimmed;
|
|
6984
7430
|
if (!isAbsolute2(expanded)) {
|
|
6985
7431
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6986
7432
|
}
|
|
@@ -7187,6 +7633,7 @@ async function driveChannels(state, driver) {
|
|
|
7187
7633
|
let unreachableMs = 0;
|
|
7188
7634
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7189
7635
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
7636
|
+
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
7190
7637
|
while (state.running) {
|
|
7191
7638
|
const cycleStartedAtMs = performance.now();
|
|
7192
7639
|
let idleThisCycle = false;
|
|
@@ -7210,11 +7657,15 @@ async function driveChannels(state, driver) {
|
|
|
7210
7657
|
state.messageCount += processed;
|
|
7211
7658
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
7212
7659
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7213
|
-
const
|
|
7660
|
+
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
7661
|
+
const appliedFiles = fileActivitySnapshot.appliedFiles;
|
|
7214
7662
|
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
7215
7663
|
const fileActivity = carriedOverFileSync || filesApplied;
|
|
7216
7664
|
lastSeenAppliedFiles = appliedFiles;
|
|
7217
|
-
|
|
7665
|
+
const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
|
|
7666
|
+
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
7667
|
+
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
7668
|
+
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
7218
7669
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7219
7670
|
idlePolls = 0;
|
|
7220
7671
|
idleMs = 0;
|
|
@@ -7283,7 +7734,7 @@ async function driveChannels(state, driver) {
|
|
|
7283
7734
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7284
7735
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7285
7736
|
function sessionDbPath() {
|
|
7286
|
-
return
|
|
7737
|
+
return join5(homedir3(), ".local", "share", "opencode", "opencode.db");
|
|
7287
7738
|
}
|
|
7288
7739
|
async function runSweep(state, driver, config) {
|
|
7289
7740
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7394,9 +7845,6 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7394
7845
|
);
|
|
7395
7846
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
7396
7847
|
}
|
|
7397
|
-
function claudeUsageFailureStreakSuffix(consecutiveFailures) {
|
|
7398
|
-
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
7399
|
-
}
|
|
7400
7848
|
function scheduleClaudeUsageReporting(state, options) {
|
|
7401
7849
|
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
7402
7850
|
options.claudeUsageReporting,
|
|
@@ -7418,23 +7866,44 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7418
7866
|
return null;
|
|
7419
7867
|
}
|
|
7420
7868
|
let consecutiveFailures = 0;
|
|
7421
|
-
let
|
|
7869
|
+
let phase = "dormant";
|
|
7422
7870
|
let rearmRequested = false;
|
|
7871
|
+
const armProbe = () => {
|
|
7872
|
+
phase = "probe-pending";
|
|
7873
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7874
|
+
};
|
|
7423
7875
|
const scheduleNextTick = () => {
|
|
7424
|
-
|
|
7425
|
-
|
|
7876
|
+
if (rearmRequested) {
|
|
7877
|
+
rearmRequested = false;
|
|
7878
|
+
armProbe();
|
|
7879
|
+
return;
|
|
7880
|
+
}
|
|
7881
|
+
phase = "steady-pending";
|
|
7426
7882
|
state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
|
|
7427
7883
|
};
|
|
7428
7884
|
const rearm = () => {
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7885
|
+
switch (phase) {
|
|
7886
|
+
case "tick-in-flight":
|
|
7887
|
+
rearmRequested = true;
|
|
7888
|
+
return;
|
|
7889
|
+
case "probe-pending":
|
|
7890
|
+
return;
|
|
7891
|
+
case "steady-pending":
|
|
7892
|
+
if (state.claudeUsageTimer) {
|
|
7893
|
+
clearTimeout(state.claudeUsageTimer);
|
|
7894
|
+
state.claudeUsageTimer = null;
|
|
7895
|
+
}
|
|
7896
|
+
rearmRequested = false;
|
|
7897
|
+
armProbe();
|
|
7898
|
+
return;
|
|
7899
|
+
case "dormant":
|
|
7900
|
+
rearmRequested = false;
|
|
7901
|
+
armProbe();
|
|
7902
|
+
return;
|
|
7432
7903
|
}
|
|
7433
|
-
rearmRequested = false;
|
|
7434
|
-
armed = true;
|
|
7435
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7436
7904
|
};
|
|
7437
7905
|
const tick = async (isProbe) => {
|
|
7906
|
+
phase = "tick-in-flight";
|
|
7438
7907
|
try {
|
|
7439
7908
|
const usage = await getClaudeUsage();
|
|
7440
7909
|
const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
|
|
@@ -7457,7 +7926,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7457
7926
|
logActivity(state, {
|
|
7458
7927
|
type: "info",
|
|
7459
7928
|
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7460
|
-
message: `Failed to report Claude usage: ${result.error}${
|
|
7929
|
+
message: `Failed to report Claude usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
7461
7930
|
});
|
|
7462
7931
|
}
|
|
7463
7932
|
scheduleNextTick();
|
|
@@ -7476,7 +7945,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7476
7945
|
level: "debug",
|
|
7477
7946
|
message: `Claude usage reporting: ${error2.message}`
|
|
7478
7947
|
});
|
|
7479
|
-
|
|
7948
|
+
phase = "dormant";
|
|
7480
7949
|
if (rearmRequested) rearm();
|
|
7481
7950
|
} else {
|
|
7482
7951
|
logActivity(state, {
|
|
@@ -7492,16 +7961,92 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7492
7961
|
logActivity(state, {
|
|
7493
7962
|
type: "info",
|
|
7494
7963
|
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7495
|
-
message: `Claude usage reporting failed: ${message}${
|
|
7964
|
+
message: `Claude usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
7496
7965
|
});
|
|
7497
7966
|
scheduleNextTick();
|
|
7498
7967
|
}
|
|
7499
7968
|
}
|
|
7500
7969
|
};
|
|
7501
|
-
|
|
7502
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7970
|
+
armProbe();
|
|
7503
7971
|
return rearm;
|
|
7504
7972
|
}
|
|
7973
|
+
var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
7974
|
+
var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
7975
|
+
var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
7976
|
+
function scheduleResourceUsageReporting(state, options) {
|
|
7977
|
+
const { enabled, warnings } = resolveResourceUsageReportingEnabled(
|
|
7978
|
+
options.resourceUsageReporting,
|
|
7979
|
+
process.env
|
|
7980
|
+
);
|
|
7981
|
+
for (const warning2 of warnings) {
|
|
7982
|
+
logActivity(state, {
|
|
7983
|
+
type: "info",
|
|
7984
|
+
level: "warn",
|
|
7985
|
+
message: `Resource usage reporting: ${warning2}`
|
|
7986
|
+
});
|
|
7987
|
+
}
|
|
7988
|
+
if (!enabled) {
|
|
7989
|
+
logActivity(state, {
|
|
7990
|
+
type: "info",
|
|
7991
|
+
level: "debug",
|
|
7992
|
+
message: "Resource usage reporting is off (--no-resource-usage-reporting)"
|
|
7993
|
+
});
|
|
7994
|
+
return;
|
|
7995
|
+
}
|
|
7996
|
+
const collect = createResourceUsageCollector();
|
|
7997
|
+
let consecutiveFailures = 0;
|
|
7998
|
+
const tick = async () => {
|
|
7999
|
+
try {
|
|
8000
|
+
const usage = collect();
|
|
8001
|
+
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
8002
|
+
if (result.ok) {
|
|
8003
|
+
if (consecutiveFailures > 0) {
|
|
8004
|
+
logActivity(state, {
|
|
8005
|
+
type: "info",
|
|
8006
|
+
level: "info",
|
|
8007
|
+
message: "Resource usage reporting recovered"
|
|
8008
|
+
});
|
|
8009
|
+
}
|
|
8010
|
+
consecutiveFailures = 0;
|
|
8011
|
+
logActivity(state, {
|
|
8012
|
+
type: "info",
|
|
8013
|
+
level: "debug",
|
|
8014
|
+
message: "Reported resource usage to Evident"
|
|
8015
|
+
});
|
|
8016
|
+
} else {
|
|
8017
|
+
consecutiveFailures++;
|
|
8018
|
+
logActivity(state, {
|
|
8019
|
+
type: "info",
|
|
8020
|
+
level: reportFailureLogLevel(
|
|
8021
|
+
consecutiveFailures,
|
|
8022
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8023
|
+
),
|
|
8024
|
+
message: `Failed to report resource usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
8025
|
+
});
|
|
8026
|
+
}
|
|
8027
|
+
} catch (error2) {
|
|
8028
|
+
consecutiveFailures++;
|
|
8029
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8030
|
+
logActivity(state, {
|
|
8031
|
+
type: "info",
|
|
8032
|
+
level: reportFailureLogLevel(
|
|
8033
|
+
consecutiveFailures,
|
|
8034
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8035
|
+
),
|
|
8036
|
+
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
8037
|
+
});
|
|
8038
|
+
} finally {
|
|
8039
|
+
state.resourceUsageTimer = setTimeout(
|
|
8040
|
+
() => void tick(),
|
|
8041
|
+
jitteredDelayMs(
|
|
8042
|
+
RESOURCE_USAGE_BASE_REPORT_DELAY_MS,
|
|
8043
|
+
RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION
|
|
8044
|
+
)
|
|
8045
|
+
);
|
|
8046
|
+
}
|
|
8047
|
+
};
|
|
8048
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
8049
|
+
}
|
|
7505
8050
|
async function notifyOffline(state) {
|
|
7506
8051
|
if (!state.agentId || !state.authHeader) return;
|
|
7507
8052
|
if (!state.connected) {
|
|
@@ -7542,6 +8087,10 @@ async function cleanup(state, opts = {}) {
|
|
|
7542
8087
|
state.claudeUsageTimer = null;
|
|
7543
8088
|
}
|
|
7544
8089
|
state.claudeUsageRearm = null;
|
|
8090
|
+
if (state.resourceUsageTimer) {
|
|
8091
|
+
clearTimeout(state.resourceUsageTimer);
|
|
8092
|
+
state.resourceUsageTimer = null;
|
|
8093
|
+
}
|
|
7545
8094
|
if (opts.graceful && state.channelDriver) {
|
|
7546
8095
|
state.channelDriver.stop();
|
|
7547
8096
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -7624,6 +8173,7 @@ async function run(options) {
|
|
|
7624
8173
|
sessionCleanupTimers: [],
|
|
7625
8174
|
claudeUsageTimer: null,
|
|
7626
8175
|
claudeUsageRearm: null,
|
|
8176
|
+
resourceUsageTimer: null,
|
|
7627
8177
|
authHeader: ""
|
|
7628
8178
|
};
|
|
7629
8179
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
@@ -8023,6 +8573,7 @@ async function run(options) {
|
|
|
8023
8573
|
}
|
|
8024
8574
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
8025
8575
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
8576
|
+
scheduleResourceUsageReporting(state, options);
|
|
8026
8577
|
if (!interactive || state.json) {
|
|
8027
8578
|
log2(state, "Driving channel messages...");
|
|
8028
8579
|
}
|
|
@@ -8103,6 +8654,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8103
8654
|
).option(
|
|
8104
8655
|
"--claude-usage-reporting <mode>",
|
|
8105
8656
|
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
8657
|
+
).option(
|
|
8658
|
+
"--no-resource-usage-reporting",
|
|
8659
|
+
"Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
|
|
8106
8660
|
).option(
|
|
8107
8661
|
"--enable-file-sync-to <dir>",
|
|
8108
8662
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
@@ -8135,6 +8689,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8135
8689
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
8136
8690
|
// (resolveClaudeUsageReportingMode).
|
|
8137
8691
|
claudeUsageReporting: options.claudeUsageReporting,
|
|
8692
|
+
// Raw value — resolution is single-sourced in run.ts's
|
|
8693
|
+
// resolveResourceUsageReportingEnabled.
|
|
8694
|
+
resourceUsageReporting: options.resourceUsageReporting,
|
|
8138
8695
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
8139
8696
|
// resolveFileSyncDirectories.
|
|
8140
8697
|
enableFileSyncTo: options.enableFileSyncTo,
|