@evident-ai/cli 3.2.1-dev.fa69368 → 3.3.1-dev.28c2528
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/index.js +753 -115
- package/dist/index.js.map +1 -1
- package/package.json +3 -5
package/dist/index.js
CHANGED
|
@@ -212,51 +212,80 @@ var api = {
|
|
|
212
212
|
|
|
213
213
|
// src/lib/keychain.ts
|
|
214
214
|
var SERVICE_NAME = "evident-cli";
|
|
215
|
-
var
|
|
216
|
-
|
|
215
|
+
var PROBE_SERVICE_NAME = "evident-cli-probe";
|
|
216
|
+
var keychainWarned = false;
|
|
217
|
+
function warnUnavailable(err) {
|
|
218
|
+
if (!keychainWarned) {
|
|
219
|
+
keychainWarned = true;
|
|
220
|
+
console.warn(
|
|
221
|
+
`System keychain unavailable, falling back to file-based credential storage: ${err instanceof Error ? err.message : String(err)}`
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
var keychain;
|
|
226
|
+
async function probeKeychain() {
|
|
217
227
|
try {
|
|
218
|
-
const keytar = await import("keytar");
|
|
228
|
+
const keytar = await import("@napi-rs/keyring/keytar.js");
|
|
219
229
|
if (typeof keytar.setPassword !== "function") {
|
|
220
230
|
return null;
|
|
221
231
|
}
|
|
232
|
+
await keytar.findCredentials(PROBE_SERVICE_NAME);
|
|
222
233
|
return keytar;
|
|
223
234
|
} catch (err) {
|
|
224
|
-
|
|
225
|
-
keytarWarned = true;
|
|
226
|
-
console.warn(
|
|
227
|
-
`System keychain unavailable, falling back to file-based credential storage: ${err instanceof Error ? err.message : String(err)}`
|
|
228
|
-
);
|
|
229
|
-
}
|
|
235
|
+
warnUnavailable(err);
|
|
230
236
|
return null;
|
|
231
237
|
}
|
|
232
238
|
}
|
|
239
|
+
function resolveKeychain() {
|
|
240
|
+
if (!keychain) {
|
|
241
|
+
keychain = probeKeychain();
|
|
242
|
+
}
|
|
243
|
+
return keychain;
|
|
244
|
+
}
|
|
233
245
|
function keychainAccount() {
|
|
234
246
|
return getApiUrlConfig();
|
|
235
247
|
}
|
|
248
|
+
function storeInFileFallback(credentials2) {
|
|
249
|
+
setCredentials({
|
|
250
|
+
token: credentials2.token,
|
|
251
|
+
user: credentials2.user,
|
|
252
|
+
expiresAt: credentials2.expiresAt
|
|
253
|
+
});
|
|
254
|
+
}
|
|
236
255
|
async function storeToken(credentials2) {
|
|
237
|
-
const keytar = await
|
|
256
|
+
const keytar = await resolveKeychain();
|
|
238
257
|
if (keytar) {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
});
|
|
258
|
+
try {
|
|
259
|
+
await keytar.setPassword(SERVICE_NAME, keychainAccount(), JSON.stringify(credentials2));
|
|
260
|
+
return;
|
|
261
|
+
} catch (err) {
|
|
262
|
+
warnUnavailable(err);
|
|
263
|
+
}
|
|
246
264
|
}
|
|
265
|
+
storeInFileFallback(credentials2);
|
|
247
266
|
}
|
|
248
267
|
async function getToken() {
|
|
249
|
-
const keytar = await
|
|
268
|
+
const keytar = await resolveKeychain();
|
|
250
269
|
if (keytar) {
|
|
251
270
|
const account = keychainAccount();
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
271
|
+
try {
|
|
272
|
+
const stored = await keytar.getPassword(SERVICE_NAME, account);
|
|
273
|
+
if (stored) {
|
|
274
|
+
try {
|
|
275
|
+
return JSON.parse(stored);
|
|
276
|
+
} catch {
|
|
277
|
+
try {
|
|
278
|
+
await keytar.deletePassword(SERVICE_NAME, account);
|
|
279
|
+
} catch (err) {
|
|
280
|
+
console.warn(
|
|
281
|
+
`Failed to clear invalid keychain entry for ${account}: ${err instanceof Error ? err.message : String(err)}`
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
return null;
|
|
285
|
+
}
|
|
259
286
|
}
|
|
287
|
+
} catch (err) {
|
|
288
|
+
warnUnavailable(err);
|
|
260
289
|
}
|
|
261
290
|
}
|
|
262
291
|
const creds = getCredentials();
|
|
@@ -273,7 +302,7 @@ function toError(err) {
|
|
|
273
302
|
return err instanceof Error ? err : new Error(String(err));
|
|
274
303
|
}
|
|
275
304
|
async function deleteToken(options = {}) {
|
|
276
|
-
const keytar = await
|
|
305
|
+
const keytar = await resolveKeychain();
|
|
277
306
|
const failures = [];
|
|
278
307
|
if (keytar) {
|
|
279
308
|
if (options.all) {
|
|
@@ -286,14 +315,25 @@ async function deleteToken(options = {}) {
|
|
|
286
315
|
await Promise.all(
|
|
287
316
|
accounts.map(async (entry) => {
|
|
288
317
|
try {
|
|
289
|
-
await keytar.deletePassword(SERVICE_NAME, entry.account);
|
|
318
|
+
const deleted = await keytar.deletePassword(SERVICE_NAME, entry.account);
|
|
319
|
+
if (!deleted) {
|
|
320
|
+
failures.push({
|
|
321
|
+
type: "delete",
|
|
322
|
+
account: entry.account,
|
|
323
|
+
error: new Error("deletePassword resolved false")
|
|
324
|
+
});
|
|
325
|
+
}
|
|
290
326
|
} catch (err) {
|
|
291
327
|
failures.push({ type: "delete", account: entry.account, error: toError(err) });
|
|
292
328
|
}
|
|
293
329
|
})
|
|
294
330
|
);
|
|
295
331
|
} else {
|
|
296
|
-
|
|
332
|
+
try {
|
|
333
|
+
await keytar.deletePassword(SERVICE_NAME, keychainAccount());
|
|
334
|
+
} catch (err) {
|
|
335
|
+
warnUnavailable(err);
|
|
336
|
+
}
|
|
297
337
|
}
|
|
298
338
|
}
|
|
299
339
|
if (options.all) {
|
|
@@ -706,6 +746,32 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
706
746
|
return { ok: false, error: describeBestEffortError(error2) };
|
|
707
747
|
}
|
|
708
748
|
}
|
|
749
|
+
async function reportResourceUsage(agentId, authHeader, usage) {
|
|
750
|
+
try {
|
|
751
|
+
const apiUrl = getApiUrlConfig();
|
|
752
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/resource-usage`, {
|
|
753
|
+
method: "POST",
|
|
754
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
755
|
+
body: JSON.stringify({
|
|
756
|
+
cpu_percent: usage.cpuPercent,
|
|
757
|
+
cpu_count: usage.cpuCount,
|
|
758
|
+
memory_total_bytes: usage.memoryTotalBytes,
|
|
759
|
+
memory_available_bytes: usage.memoryAvailableBytes
|
|
760
|
+
}),
|
|
761
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
762
|
+
});
|
|
763
|
+
if (!response.ok) {
|
|
764
|
+
const serverMessage = await readErrorMessage(response);
|
|
765
|
+
return {
|
|
766
|
+
ok: false,
|
|
767
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
return { ok: true };
|
|
771
|
+
} catch (error2) {
|
|
772
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
773
|
+
}
|
|
774
|
+
}
|
|
709
775
|
async function getAgentInfo(agentId, authHeader) {
|
|
710
776
|
const apiUrl = getApiUrlConfig();
|
|
711
777
|
try {
|
|
@@ -894,6 +960,7 @@ import { homedir } from "os";
|
|
|
894
960
|
import { join } from "path";
|
|
895
961
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
896
962
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
963
|
+
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
897
964
|
function parseClaudeCliCredentials(raw) {
|
|
898
965
|
let parsed;
|
|
899
966
|
try {
|
|
@@ -927,7 +994,7 @@ function readClaudeCliCredentials() {
|
|
|
927
994
|
}
|
|
928
995
|
}
|
|
929
996
|
try {
|
|
930
|
-
const raw = readFileSync(join(homedir(),
|
|
997
|
+
const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
|
|
931
998
|
return parseClaudeCliCredentials(raw);
|
|
932
999
|
} catch (err) {
|
|
933
1000
|
const code = err.code;
|
|
@@ -1023,7 +1090,7 @@ async function claudeUsage() {
|
|
|
1023
1090
|
|
|
1024
1091
|
// src/commands/run.ts
|
|
1025
1092
|
import { homedir as homedir3 } from "os";
|
|
1026
|
-
import { isAbsolute as isAbsolute2, join as
|
|
1093
|
+
import { isAbsolute as isAbsolute2, join as join5, parse, resolve as resolvePath } from "path";
|
|
1027
1094
|
import chalk6 from "chalk";
|
|
1028
1095
|
|
|
1029
1096
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1052,6 +1119,7 @@ var MAX_FILE_SYNC_DIRECTORIES = 16;
|
|
|
1052
1119
|
|
|
1053
1120
|
// ../../packages/types/src/logging/index.ts
|
|
1054
1121
|
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
1122
|
+
var FORWARD_FAILURE_REASON_HEADER = "X-Evident-Failure-Reason";
|
|
1055
1123
|
function log(level, event, fields) {
|
|
1056
1124
|
const method = level === "debug" ? "log" : level;
|
|
1057
1125
|
try {
|
|
@@ -1659,13 +1727,22 @@ function buildNoProviderWarning(hasProvider) {
|
|
|
1659
1727
|
return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
|
|
1660
1728
|
}
|
|
1661
1729
|
|
|
1730
|
+
// src/lib/http-timeout.ts
|
|
1731
|
+
var REQUEST_TIMEOUT_MS = 6e4;
|
|
1732
|
+
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1733
|
+
return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1662
1736
|
// src/lib/opencode/session.ts
|
|
1737
|
+
function timedFetch(input, init) {
|
|
1738
|
+
return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
|
|
1739
|
+
}
|
|
1663
1740
|
function opencodeBase(port) {
|
|
1664
1741
|
return `http://127.0.0.1:${port}`;
|
|
1665
1742
|
}
|
|
1666
1743
|
async function getOpenCodeDirectory(port) {
|
|
1667
1744
|
try {
|
|
1668
|
-
const res = await
|
|
1745
|
+
const res = await timedFetch(`${opencodeBase(port)}/path`);
|
|
1669
1746
|
if (!res.ok) return null;
|
|
1670
1747
|
const body = await res.json();
|
|
1671
1748
|
const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
|
|
@@ -1716,7 +1793,7 @@ function isAssistantInFlight(m) {
|
|
|
1716
1793
|
}
|
|
1717
1794
|
async function getSessionMessages(port, sessionId) {
|
|
1718
1795
|
try {
|
|
1719
|
-
const res = await
|
|
1796
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1720
1797
|
if (!res.ok) return null;
|
|
1721
1798
|
const body = await res.json();
|
|
1722
1799
|
return Array.isArray(body) ? body : null;
|
|
@@ -1746,7 +1823,7 @@ function sessionLastActivityMs(session) {
|
|
|
1746
1823
|
}
|
|
1747
1824
|
async function listSessions(port) {
|
|
1748
1825
|
try {
|
|
1749
|
-
const res = await
|
|
1826
|
+
const res = await timedFetch(`${opencodeBase(port)}/session`);
|
|
1750
1827
|
if (!res.ok) return null;
|
|
1751
1828
|
const body = await res.json();
|
|
1752
1829
|
return Array.isArray(body) ? body : null;
|
|
@@ -1756,7 +1833,7 @@ async function listSessions(port) {
|
|
|
1756
1833
|
}
|
|
1757
1834
|
async function deleteSession(port, id) {
|
|
1758
1835
|
try {
|
|
1759
|
-
const res = await
|
|
1836
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1760
1837
|
return res.status >= 200 && res.status < 300;
|
|
1761
1838
|
} catch {
|
|
1762
1839
|
return false;
|
|
@@ -1764,7 +1841,7 @@ async function deleteSession(port, id) {
|
|
|
1764
1841
|
}
|
|
1765
1842
|
async function sessionExists(port, id) {
|
|
1766
1843
|
try {
|
|
1767
|
-
const res = await
|
|
1844
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
|
|
1768
1845
|
if (res.status >= 200 && res.status < 300) return true;
|
|
1769
1846
|
if (res.status === 404) return false;
|
|
1770
1847
|
return null;
|
|
@@ -1774,7 +1851,7 @@ async function sessionExists(port, id) {
|
|
|
1774
1851
|
}
|
|
1775
1852
|
async function getSessionStatuses(port) {
|
|
1776
1853
|
try {
|
|
1777
|
-
const res = await
|
|
1854
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/status`);
|
|
1778
1855
|
if (!res.ok) {
|
|
1779
1856
|
console.error(
|
|
1780
1857
|
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
@@ -1807,7 +1884,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1807
1884
|
if (directory && directory.trim()) {
|
|
1808
1885
|
url.searchParams.set("directory", directory.trim());
|
|
1809
1886
|
}
|
|
1810
|
-
const response = await
|
|
1887
|
+
const response = await timedFetch(url, {
|
|
1811
1888
|
method: "POST",
|
|
1812
1889
|
headers: { "Content-Type": "application/json" },
|
|
1813
1890
|
body: JSON.stringify({})
|
|
@@ -1821,7 +1898,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1821
1898
|
}
|
|
1822
1899
|
async function getModelAttachmentCapability(port, model) {
|
|
1823
1900
|
try {
|
|
1824
|
-
const res = await
|
|
1901
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1825
1902
|
if (!res.ok) {
|
|
1826
1903
|
console.error(
|
|
1827
1904
|
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -1954,7 +2031,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1954
2031
|
};
|
|
1955
2032
|
}
|
|
1956
2033
|
}
|
|
1957
|
-
const res = await
|
|
2034
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1958
2035
|
method: "POST",
|
|
1959
2036
|
headers: { "Content-Type": "application/json" },
|
|
1960
2037
|
body: JSON.stringify(body)
|
|
@@ -2203,7 +2280,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
2203
2280
|
}
|
|
2204
2281
|
async function hasAnyConfiguredProvider(port) {
|
|
2205
2282
|
try {
|
|
2206
|
-
const res = await
|
|
2283
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2207
2284
|
if (!res.ok) {
|
|
2208
2285
|
console.error(
|
|
2209
2286
|
`[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -2645,6 +2722,17 @@ var StreamForwarder = class {
|
|
|
2645
2722
|
};
|
|
2646
2723
|
|
|
2647
2724
|
// src/lib/tunnel/connection.ts
|
|
2725
|
+
var FAILURE_REASON_HEADER_LC = FORWARD_FAILURE_REASON_HEADER.toLowerCase();
|
|
2726
|
+
var TunnelUpgradeRejectedError = class extends Error {
|
|
2727
|
+
constructor(message, reason) {
|
|
2728
|
+
super(message);
|
|
2729
|
+
this.reason = reason;
|
|
2730
|
+
}
|
|
2731
|
+
};
|
|
2732
|
+
function classifyUpgradeRejection(headers) {
|
|
2733
|
+
const value = headers[FAILURE_REASON_HEADER_LC];
|
|
2734
|
+
return value === "do_code_updated" ? "do_code_updated" : "unknown";
|
|
2735
|
+
}
|
|
2648
2736
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
2649
2737
|
var BASE_RECONNECT_DELAY = 500;
|
|
2650
2738
|
function getReconnectDelay(attempt) {
|
|
@@ -2689,6 +2777,7 @@ function connectTunnel(options) {
|
|
|
2689
2777
|
onError,
|
|
2690
2778
|
onResponse,
|
|
2691
2779
|
onInfo,
|
|
2780
|
+
onWarning,
|
|
2692
2781
|
onDrainPing
|
|
2693
2782
|
} = options;
|
|
2694
2783
|
const tunnelUrl = getTunnelUrlConfig();
|
|
@@ -2708,8 +2797,11 @@ function connectTunnel(options) {
|
|
|
2708
2797
|
reject(new Error("Connection timeout"));
|
|
2709
2798
|
}, 3e4);
|
|
2710
2799
|
let upgradeRejection = null;
|
|
2800
|
+
let upgradeRejectionReason = null;
|
|
2711
2801
|
ws.on("unexpected-response", (_req, res) => {
|
|
2712
2802
|
clearTimeout(connectionTimeout);
|
|
2803
|
+
const reason = classifyUpgradeRejection(res.headers);
|
|
2804
|
+
upgradeRejectionReason = reason;
|
|
2713
2805
|
const chunks = [];
|
|
2714
2806
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
2715
2807
|
res.on("end", () => {
|
|
@@ -2723,8 +2815,14 @@ function connectTunnel(options) {
|
|
|
2723
2815
|
}
|
|
2724
2816
|
const statusLine = `HTTP ${res.statusCode}${res.statusMessage ? ` ${res.statusMessage}` : ""}`;
|
|
2725
2817
|
upgradeRejection = detail ? `${statusLine}: ${detail}` : statusLine;
|
|
2726
|
-
|
|
2727
|
-
|
|
2818
|
+
if (reason === "do_code_updated") {
|
|
2819
|
+
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
2820
|
+
} else {
|
|
2821
|
+
onError?.(`Tunnel refused by relay (${upgradeRejection})`);
|
|
2822
|
+
}
|
|
2823
|
+
reject(
|
|
2824
|
+
new TunnelUpgradeRejectedError(`Tunnel handshake rejected: ${upgradeRejection}`, reason)
|
|
2825
|
+
);
|
|
2728
2826
|
});
|
|
2729
2827
|
});
|
|
2730
2828
|
ws.on("open", () => {
|
|
@@ -2770,8 +2868,14 @@ function connectTunnel(options) {
|
|
|
2770
2868
|
ws.on("error", (error2) => {
|
|
2771
2869
|
clearTimeout(connectionTimeout);
|
|
2772
2870
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
2773
|
-
|
|
2774
|
-
|
|
2871
|
+
if (upgradeRejectionReason === "do_code_updated") {
|
|
2872
|
+
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
2873
|
+
} else {
|
|
2874
|
+
onError?.(`Connection error: ${detail}`);
|
|
2875
|
+
}
|
|
2876
|
+
reject(
|
|
2877
|
+
upgradeRejectionReason !== null ? new TunnelUpgradeRejectedError(detail, upgradeRejectionReason) : new Error(detail)
|
|
2878
|
+
);
|
|
2775
2879
|
});
|
|
2776
2880
|
ws.on("close", (code, reason) => {
|
|
2777
2881
|
const reasonStr = reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
@@ -2847,7 +2951,8 @@ var RunnerConnection = class {
|
|
|
2847
2951
|
onError: (error2) => events.onError?.(error2),
|
|
2848
2952
|
onResponse: () => events.onResponse?.(),
|
|
2849
2953
|
onDrainPing: () => events.onDrainPing?.(),
|
|
2850
|
-
onInfo: (message) => events.onInfo?.(message)
|
|
2954
|
+
onInfo: (message) => events.onInfo?.(message),
|
|
2955
|
+
onWarning: (message) => events.onWarning?.(message)
|
|
2851
2956
|
});
|
|
2852
2957
|
return;
|
|
2853
2958
|
} catch (error2) {
|
|
@@ -2858,7 +2963,12 @@ var RunnerConnection = class {
|
|
|
2858
2963
|
}
|
|
2859
2964
|
const delay = getReconnectDelay(this.reconnectAttempt);
|
|
2860
2965
|
events.onReconnecting?.(this.reconnectAttempt);
|
|
2861
|
-
|
|
2966
|
+
const retryMessage = `Connection failed, retrying in ${Math.round(delay / 1e3)}s...`;
|
|
2967
|
+
if (error2 instanceof TunnelUpgradeRejectedError && error2.reason === "do_code_updated") {
|
|
2968
|
+
events.onWarning?.(retryMessage);
|
|
2969
|
+
} else {
|
|
2970
|
+
events.onError?.(retryMessage);
|
|
2971
|
+
}
|
|
2862
2972
|
await this.sleep(delay);
|
|
2863
2973
|
}
|
|
2864
2974
|
}
|
|
@@ -2878,6 +2988,21 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
2878
2988
|
}
|
|
2879
2989
|
}
|
|
2880
2990
|
|
|
2991
|
+
// src/lib/reporting-schedule.ts
|
|
2992
|
+
function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
2993
|
+
const jitterRangeMs = baseMs * jitterFraction;
|
|
2994
|
+
return baseMs - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
2995
|
+
}
|
|
2996
|
+
function firstReportDelayMs(random = Math.random) {
|
|
2997
|
+
return 5e3 + random() * 1e4;
|
|
2998
|
+
}
|
|
2999
|
+
function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
|
|
3000
|
+
return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
|
|
3001
|
+
}
|
|
3002
|
+
function failureStreakSuffix(consecutiveFailures) {
|
|
3003
|
+
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
3004
|
+
}
|
|
3005
|
+
|
|
2881
3006
|
// src/lib/claude-usage-reporting.ts
|
|
2882
3007
|
var VALID_MODES = ["auto", "on", "off"];
|
|
2883
3008
|
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
@@ -2900,18 +3025,79 @@ function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
|
2900
3025
|
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
2901
3026
|
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
2902
3027
|
function nextReportDelayMs(random = Math.random) {
|
|
2903
|
-
|
|
2904
|
-
return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
3028
|
+
return jitteredDelayMs(BASE_REPORT_DELAY_MS, REPORT_DELAY_JITTER_FRACTION, random);
|
|
2905
3029
|
}
|
|
2906
|
-
var FIRST_REPORT_DELAY_MS =
|
|
3030
|
+
var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
|
|
2907
3031
|
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
2908
3032
|
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
2909
|
-
return consecutiveFailures
|
|
3033
|
+
return reportFailureLogLevel(consecutiveFailures, CLAUDE_USAGE_FAILURE_REESCALATION_TICKS);
|
|
3034
|
+
}
|
|
3035
|
+
|
|
3036
|
+
// src/lib/resource-usage-reporting.ts
|
|
3037
|
+
var ENABLED_VALUES = /* @__PURE__ */ new Set(["on", "true", "1"]);
|
|
3038
|
+
var DISABLED_VALUES = /* @__PURE__ */ new Set(["off", "false", "0"]);
|
|
3039
|
+
function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
3040
|
+
if (flagValue === false) {
|
|
3041
|
+
return { enabled: false, warnings: [] };
|
|
3042
|
+
}
|
|
3043
|
+
const raw = env.EVIDENT_RESOURCE_USAGE_REPORTING;
|
|
3044
|
+
if (raw === void 0 || raw === "") {
|
|
3045
|
+
return { enabled: true, warnings: [] };
|
|
3046
|
+
}
|
|
3047
|
+
const normalized = raw.trim().toLowerCase();
|
|
3048
|
+
if (DISABLED_VALUES.has(normalized)) {
|
|
3049
|
+
return { enabled: false, warnings: [] };
|
|
3050
|
+
}
|
|
3051
|
+
if (ENABLED_VALUES.has(normalized)) {
|
|
3052
|
+
return { enabled: true, warnings: [] };
|
|
3053
|
+
}
|
|
3054
|
+
return {
|
|
3055
|
+
enabled: true,
|
|
3056
|
+
warnings: [
|
|
3057
|
+
`Ignoring invalid EVIDENT_RESOURCE_USAGE_REPORTING "${raw}": expected on or off; leaving reporting on`
|
|
3058
|
+
]
|
|
3059
|
+
};
|
|
3060
|
+
}
|
|
3061
|
+
|
|
3062
|
+
// src/lib/resource-usage.ts
|
|
3063
|
+
import { cpus, totalmem, freemem } from "os";
|
|
3064
|
+
function readCpuSample() {
|
|
3065
|
+
let busyMs = 0;
|
|
3066
|
+
let idleMs = 0;
|
|
3067
|
+
for (const cpu of cpus()) {
|
|
3068
|
+
busyMs += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.irq;
|
|
3069
|
+
idleMs += cpu.times.idle;
|
|
3070
|
+
}
|
|
3071
|
+
return { busyMs, idleMs };
|
|
3072
|
+
}
|
|
3073
|
+
function cpuPercentBetween(previous, current) {
|
|
3074
|
+
const deltaBusy = current.busyMs - previous.busyMs;
|
|
3075
|
+
const deltaIdle = current.idleMs - previous.idleMs;
|
|
3076
|
+
const total = deltaBusy + deltaIdle;
|
|
3077
|
+
if (total === 0) return null;
|
|
3078
|
+
return Math.round((deltaBusy / total * 100 + Number.EPSILON) * 100) / 100;
|
|
3079
|
+
}
|
|
3080
|
+
function createResourceUsageCollector() {
|
|
3081
|
+
let previous = readCpuSample();
|
|
3082
|
+
return () => {
|
|
3083
|
+
const current = readCpuSample();
|
|
3084
|
+
const cpuPercent = cpuPercentBetween(previous, current);
|
|
3085
|
+
previous = current;
|
|
3086
|
+
return {
|
|
3087
|
+
cpuPercent,
|
|
3088
|
+
cpuCount: cpus().length,
|
|
3089
|
+
memoryTotalBytes: totalmem(),
|
|
3090
|
+
memoryAvailableBytes: freemem()
|
|
3091
|
+
};
|
|
3092
|
+
};
|
|
2910
3093
|
}
|
|
2911
3094
|
|
|
2912
3095
|
// src/lib/channels/driver.ts
|
|
2913
3096
|
import { homedir as homedir2 } from "os";
|
|
2914
3097
|
|
|
3098
|
+
// src/lib/runner-file-sync.ts
|
|
3099
|
+
import { join as join4 } from "path";
|
|
3100
|
+
|
|
2915
3101
|
// src/lib/file-push.ts
|
|
2916
3102
|
import { randomUUID } from "crypto";
|
|
2917
3103
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
@@ -3105,17 +3291,20 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3105
3291
|
for (const id of options.ackFailures.keys()) {
|
|
3106
3292
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3107
3293
|
}
|
|
3108
|
-
if (pending.length === 0) return 0;
|
|
3294
|
+
if (pending.length === 0) return { applied: 0, claudeCredentialApplied: false };
|
|
3109
3295
|
options.log({
|
|
3110
3296
|
level: "info",
|
|
3111
3297
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3112
3298
|
});
|
|
3113
3299
|
let applied = 0;
|
|
3300
|
+
let claudeCredentialApplied = false;
|
|
3114
3301
|
for (const file of pending) {
|
|
3115
3302
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3116
|
-
|
|
3303
|
+
const outcome = await applyOne(options, file);
|
|
3304
|
+
if (outcome.applied) applied += 1;
|
|
3305
|
+
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
3117
3306
|
}
|
|
3118
|
-
return applied;
|
|
3307
|
+
return { applied, claudeCredentialApplied };
|
|
3119
3308
|
}
|
|
3120
3309
|
async function listPendingFiles(options) {
|
|
3121
3310
|
let res;
|
|
@@ -3176,6 +3365,11 @@ function asPendingFile(entry) {
|
|
|
3176
3365
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3177
3366
|
return { id, path, size };
|
|
3178
3367
|
}
|
|
3368
|
+
var NOT_APPLIED = { applied: false, claudeCredentialApplied: false };
|
|
3369
|
+
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3370
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join4(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3371
|
+
return expanded === join4(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3372
|
+
}
|
|
3179
3373
|
async function applyOne(options, file) {
|
|
3180
3374
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
3181
3375
|
if (options.allowedDirectories.length === 0) {
|
|
@@ -3184,7 +3378,7 @@ async function applyOne(options, file) {
|
|
|
3184
3378
|
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
3185
3379
|
});
|
|
3186
3380
|
await ack(options, file, "rejected", "file_sync_disabled");
|
|
3187
|
-
return
|
|
3381
|
+
return NOT_APPLIED;
|
|
3188
3382
|
}
|
|
3189
3383
|
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
3190
3384
|
options.log({
|
|
@@ -3192,12 +3386,12 @@ async function applyOne(options, file) {
|
|
|
3192
3386
|
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
3193
3387
|
});
|
|
3194
3388
|
await ack(options, file, "rejected", "file_too_large");
|
|
3195
|
-
return
|
|
3389
|
+
return NOT_APPLIED;
|
|
3196
3390
|
}
|
|
3197
3391
|
const download = await downloadContent(options, file, label);
|
|
3198
3392
|
if (!download.ok) {
|
|
3199
3393
|
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
3200
|
-
return
|
|
3394
|
+
return NOT_APPLIED;
|
|
3201
3395
|
}
|
|
3202
3396
|
let outcome;
|
|
3203
3397
|
try {
|
|
@@ -3213,7 +3407,7 @@ async function applyOne(options, file) {
|
|
|
3213
3407
|
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
3214
3408
|
});
|
|
3215
3409
|
await ack(options, file, "rejected", "write_failed");
|
|
3216
|
-
return
|
|
3410
|
+
return NOT_APPLIED;
|
|
3217
3411
|
}
|
|
3218
3412
|
if (!outcome.ok) {
|
|
3219
3413
|
options.log({
|
|
@@ -3221,14 +3415,17 @@ async function applyOne(options, file) {
|
|
|
3221
3415
|
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
3222
3416
|
});
|
|
3223
3417
|
await ack(options, file, "rejected", outcome.code);
|
|
3224
|
-
return
|
|
3418
|
+
return NOT_APPLIED;
|
|
3225
3419
|
}
|
|
3226
3420
|
options.log({
|
|
3227
3421
|
level: "info",
|
|
3228
3422
|
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
3229
3423
|
});
|
|
3230
3424
|
await ack(options, file, "applied");
|
|
3231
|
-
return
|
|
3425
|
+
return {
|
|
3426
|
+
applied: true,
|
|
3427
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir)
|
|
3428
|
+
};
|
|
3232
3429
|
}
|
|
3233
3430
|
function durableDownloadCode(status2) {
|
|
3234
3431
|
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
@@ -3339,8 +3536,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
|
3339
3536
|
var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
|
|
3340
3537
|
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
3341
3538
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
3539
|
+
var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
|
|
3540
|
+
var MAX_WATCHER_STALL_RESTARTS = 3;
|
|
3541
|
+
var MAX_RELEASED_OPENCODE_IDS = 256;
|
|
3342
3542
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3343
3543
|
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
3544
|
+
var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
|
|
3545
|
+
var MAX_WEDGED_CONVERSATIONS = 256;
|
|
3344
3546
|
var ChannelAuthError = class extends Error {
|
|
3345
3547
|
constructor(message) {
|
|
3346
3548
|
super(message);
|
|
@@ -3384,6 +3586,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3384
3586
|
fileSyncDirectories;
|
|
3385
3587
|
homeDir;
|
|
3386
3588
|
maxActiveSessions;
|
|
3589
|
+
watcherStallMs;
|
|
3590
|
+
wedgeWarningIntervalMs;
|
|
3387
3591
|
/** Cache of conversationId → opencode sessionId. */
|
|
3388
3592
|
sessions = /* @__PURE__ */ new Map();
|
|
3389
3593
|
/**
|
|
@@ -3414,6 +3618,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3414
3618
|
* bounded cost.
|
|
3415
3619
|
*/
|
|
3416
3620
|
supersededSessions = /* @__PURE__ */ new Map();
|
|
3621
|
+
/**
|
|
3622
|
+
* Local re-drive fence for a message force-released by the stall watchdog
|
|
3623
|
+
* (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
|
|
3624
|
+
* see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
|
|
3625
|
+
* is `null` for exactly this shape (its `markProcessing` never landed), so
|
|
3626
|
+
* without a local record of the id the driver last knew, the next drain's
|
|
3627
|
+
* `if (message.opencode_message_id)` re-drive-fence check at
|
|
3628
|
+
* `processConversation` would not engage and it would blind-`prompt_async`
|
|
3629
|
+
* a turn that may still be running in opencode — the one duplicate-turn
|
|
3630
|
+
* hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
|
|
3631
|
+
* `processConversation` reads `message.opencode_message_id ?? this
|
|
3632
|
+
* .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
|
|
3633
|
+
* threads it into `resolveRedrive`, which asks opencode itself whether the
|
|
3634
|
+
* turn is still ongoing before ever dispatching.
|
|
3635
|
+
*
|
|
3636
|
+
* Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
|
|
3637
|
+
* `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
|
|
3638
|
+
* non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
|
|
3639
|
+
* dispatch) and at the top-level fresh-dispatch site, so it does not outlive
|
|
3640
|
+
* the row it was recorded for.
|
|
3641
|
+
*/
|
|
3642
|
+
releasedOpencodeIds = /* @__PURE__ */ new Map();
|
|
3643
|
+
/**
|
|
3644
|
+
* Per-conversation throttle state for the #183 recurrence warning (#1618
|
|
3645
|
+
* WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
|
|
3646
|
+
* `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
|
|
3647
|
+
* log line and the `dispatch_wedged` signal to at most once per
|
|
3648
|
+
* `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
|
|
3649
|
+
* so the operator sees magnitude, not repetition. Cleared the moment the
|
|
3650
|
+
* conversation dispatches anything (a fresh wedge, if it recurs, is a new
|
|
3651
|
+
* incident). Bounded FIFO, mirroring `supersededSessions`
|
|
3652
|
+
* (`MAX_WEDGED_CONVERSATIONS`).
|
|
3653
|
+
*/
|
|
3654
|
+
wedgeWarnings = /* @__PURE__ */ new Map();
|
|
3417
3655
|
/**
|
|
3418
3656
|
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
3419
3657
|
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
@@ -3632,6 +3870,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3632
3870
|
* same trick `lastProxiedActivityAt` uses.
|
|
3633
3871
|
*/
|
|
3634
3872
|
appliedFileCount = 0;
|
|
3873
|
+
/**
|
|
3874
|
+
* Generation counter, NOT a tally (#1656): advances by exactly one per sync
|
|
3875
|
+
* batch that applied the Claude CLI credential file, not by how many
|
|
3876
|
+
* credential files were in that batch. `run.ts` only ever tests inequality
|
|
3877
|
+
* against the value it saw last cycle, so magnitude is meaningless — keep it
|
|
3878
|
+
* that way rather than "fixing" it into a count.
|
|
3879
|
+
*/
|
|
3880
|
+
claudeCredentialApplyCount = 0;
|
|
3635
3881
|
/**
|
|
3636
3882
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3637
3883
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -3656,7 +3902,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3656
3902
|
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3657
3903
|
this.log = config.log ?? (() => {
|
|
3658
3904
|
});
|
|
3659
|
-
this.fetchImpl =
|
|
3905
|
+
this.fetchImpl = withRequestTimeout(
|
|
3906
|
+
config.fetchImpl ?? fetch,
|
|
3907
|
+
config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
|
|
3908
|
+
);
|
|
3660
3909
|
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3661
3910
|
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3662
3911
|
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
@@ -3665,6 +3914,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3665
3914
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3666
3915
|
this.homeDir = config.homeDir ?? homedir2();
|
|
3667
3916
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
3917
|
+
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
3918
|
+
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
3668
3919
|
}
|
|
3669
3920
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
3670
3921
|
get opencodeBase() {
|
|
@@ -3678,6 +3929,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3678
3929
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
3679
3930
|
*/
|
|
3680
3931
|
async drainPending() {
|
|
3932
|
+
try {
|
|
3933
|
+
this.reconcileWatchers();
|
|
3934
|
+
} catch (err) {
|
|
3935
|
+
this.log({
|
|
3936
|
+
level: "error",
|
|
3937
|
+
message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
|
|
3938
|
+
});
|
|
3939
|
+
}
|
|
3681
3940
|
if (this.stopped) return 0;
|
|
3682
3941
|
if (this.draining) return 0;
|
|
3683
3942
|
this.draining = true;
|
|
@@ -3711,7 +3970,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3711
3970
|
if (this.syncingFiles) return 0;
|
|
3712
3971
|
this.syncingFiles = true;
|
|
3713
3972
|
try {
|
|
3714
|
-
const
|
|
3973
|
+
const result = await syncPendingRunnerFiles({
|
|
3715
3974
|
agentId: this.agentId,
|
|
3716
3975
|
apiUrl: this.apiUrl,
|
|
3717
3976
|
getAuthHeader: this.getAuthHeader,
|
|
@@ -3721,8 +3980,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3721
3980
|
ackFailures: this.fileAckFailures,
|
|
3722
3981
|
log: this.log
|
|
3723
3982
|
});
|
|
3724
|
-
this.appliedFileCount += applied;
|
|
3725
|
-
|
|
3983
|
+
this.appliedFileCount += result.applied;
|
|
3984
|
+
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
3985
|
+
return result.applied;
|
|
3726
3986
|
} catch (err) {
|
|
3727
3987
|
this.log({
|
|
3728
3988
|
level: "error",
|
|
@@ -3813,12 +4073,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3813
4073
|
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3814
4074
|
* idle checks still shows up as an advance.
|
|
3815
4075
|
*
|
|
4076
|
+
* A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
|
|
4077
|
+
* (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
|
|
4078
|
+
* reporting on it advancing, so an unrelated file sync can never disturb a
|
|
4079
|
+
* healthy reporting cadence (#1627) — it never even reaches that trigger, let
|
|
4080
|
+
* alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
|
|
4081
|
+
* whose consumer is idle-timeout suppression and must key on ANY file, not
|
|
4082
|
+
* just a Claude credential.
|
|
4083
|
+
*
|
|
3816
4084
|
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3817
4085
|
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3818
4086
|
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3819
4087
|
*/
|
|
3820
4088
|
fileSyncActivity() {
|
|
3821
|
-
return {
|
|
4089
|
+
return {
|
|
4090
|
+
appliedFiles: this.appliedFileCount,
|
|
4091
|
+
inFlight: this.syncingFiles,
|
|
4092
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount
|
|
4093
|
+
};
|
|
3822
4094
|
}
|
|
3823
4095
|
/**
|
|
3824
4096
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
@@ -3931,8 +4203,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3931
4203
|
skippedAlreadyDispatched += 1;
|
|
3932
4204
|
continue;
|
|
3933
4205
|
}
|
|
3934
|
-
|
|
3935
|
-
|
|
4206
|
+
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
4207
|
+
if (effectiveOpencodeMessageId) {
|
|
4208
|
+
const outcome = await this.resolveRedrive(
|
|
4209
|
+
conv,
|
|
4210
|
+
sessionId,
|
|
4211
|
+
message,
|
|
4212
|
+
sessionCreated,
|
|
4213
|
+
effectiveOpencodeMessageId
|
|
4214
|
+
);
|
|
3936
4215
|
if (outcome === "abandoned") {
|
|
3937
4216
|
continue;
|
|
3938
4217
|
}
|
|
@@ -4043,21 +4322,102 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4043
4322
|
}
|
|
4044
4323
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4045
4324
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
4325
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
4046
4326
|
this.dispatched.add(message.id);
|
|
4047
4327
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
4048
4328
|
dispatched += 1;
|
|
4049
4329
|
void this.postSignal(conv.id, message.id, "dispatched");
|
|
4050
4330
|
}
|
|
4051
4331
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
4052
|
-
this.
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
conversation_id: conv.id
|
|
4056
|
-
});
|
|
4332
|
+
this.reportWedgedConversation(conv, messages);
|
|
4333
|
+
} else if (dispatched > 0) {
|
|
4334
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4057
4335
|
}
|
|
4058
4336
|
this.ensureWatcherRunning(sessionId);
|
|
4059
4337
|
return dispatched;
|
|
4060
4338
|
}
|
|
4339
|
+
/**
|
|
4340
|
+
* The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
|
|
4341
|
+
* escalated (#1618 WI-4). `messages` is the conversation's full pending list
|
|
4342
|
+
* on THIS tick — the caller has already confirmed every one of them is a
|
|
4343
|
+
* skip-because-already-`dispatched`, the exact signature of a message stuck
|
|
4344
|
+
* acknowledged-but-never-worked.
|
|
4345
|
+
*
|
|
4346
|
+
* Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
|
|
4347
|
+
* (52,843 occurrences observed in one incident) — burning the GLOBAL
|
|
4348
|
+
* 30-events/60s `runner-activity-telemetry.ts` budget that was itself
|
|
4349
|
+
* suppressing the diagnostics needed to debug the wedge. The `warn` log (and
|
|
4350
|
+
* the `dispatch_wedged` signal once the wedge has persisted past the same
|
|
4351
|
+
* interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
|
|
4352
|
+
* naming the consecutive-tick count so the operator sees magnitude rather
|
|
4353
|
+
* than repetition.
|
|
4354
|
+
*
|
|
4355
|
+
* Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
|
|
4356
|
+
* unconditionally on this same tick and is already recovering anything it
|
|
4357
|
+
* can see. This is reporting only — see `countUntrackedIds`'s doc for the
|
|
4358
|
+
* one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
|
|
4359
|
+
*/
|
|
4360
|
+
reportWedgedConversation(conv, messages) {
|
|
4361
|
+
const now = this.now();
|
|
4362
|
+
const existing = this.wedgeWarnings.get(conv.id);
|
|
4363
|
+
const firstWedgedAt = existing?.firstWedgedAt ?? now;
|
|
4364
|
+
const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
|
|
4365
|
+
const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
|
|
4366
|
+
if (!dueForWarn) {
|
|
4367
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4368
|
+
this.wedgeWarnings.set(conv.id, {
|
|
4369
|
+
firstWedgedAt,
|
|
4370
|
+
lastWarnedAt: existing.lastWarnedAt,
|
|
4371
|
+
consecutiveTicks
|
|
4372
|
+
});
|
|
4373
|
+
return;
|
|
4374
|
+
}
|
|
4375
|
+
const stuckForMs = now - firstWedgedAt;
|
|
4376
|
+
const untracked = this.countUntrackedIds(messages);
|
|
4377
|
+
this.log({
|
|
4378
|
+
level: "warn",
|
|
4379
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
4380
|
+
conversation_id: conv.id
|
|
4381
|
+
});
|
|
4382
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4383
|
+
this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
|
|
4384
|
+
while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
|
|
4385
|
+
const oldest = this.wedgeWarnings.keys().next().value;
|
|
4386
|
+
if (oldest === void 0) break;
|
|
4387
|
+
this.wedgeWarnings.delete(oldest);
|
|
4388
|
+
}
|
|
4389
|
+
if (stuckForMs >= this.wedgeWarningIntervalMs) {
|
|
4390
|
+
void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
|
|
4391
|
+
stuck_for_ms: stuckForMs,
|
|
4392
|
+
untracked
|
|
4393
|
+
});
|
|
4394
|
+
}
|
|
4395
|
+
}
|
|
4396
|
+
/**
|
|
4397
|
+
* How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
|
|
4398
|
+
* WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
|
|
4399
|
+
* `dispatched`/`inFlight` pairing invariant holds by construction across
|
|
4400
|
+
* every `dispatched.add` site (see its own doc comment), so `> 0` here means
|
|
4401
|
+
* that invariant has actually been violated for this conversation: there is
|
|
4402
|
+
* no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
|
|
4403
|
+
* `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
|
|
4404
|
+
* recovering. One pass over `this.watchers`, called only when the throttled
|
|
4405
|
+
* warning above is due to fire — not every tick.
|
|
4406
|
+
*/
|
|
4407
|
+
countUntrackedIds(messages) {
|
|
4408
|
+
let untracked = 0;
|
|
4409
|
+
for (const message of messages) {
|
|
4410
|
+
let tracked = false;
|
|
4411
|
+
for (const watcher of this.watchers.values()) {
|
|
4412
|
+
if (watcher.inFlight.has(message.id)) {
|
|
4413
|
+
tracked = true;
|
|
4414
|
+
break;
|
|
4415
|
+
}
|
|
4416
|
+
}
|
|
4417
|
+
if (!tracked) untracked += 1;
|
|
4418
|
+
}
|
|
4419
|
+
return untracked;
|
|
4420
|
+
}
|
|
4061
4421
|
/**
|
|
4062
4422
|
* Poll a session's message list for the re-drive fence (#965), via the
|
|
4063
4423
|
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
@@ -4126,9 +4486,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4126
4486
|
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
4127
4487
|
* other failure resolves to `unresolved` and is retried whole on the next
|
|
4128
4488
|
* ~2s drain tick.
|
|
4489
|
+
*
|
|
4490
|
+
* `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
|
|
4491
|
+
* server `opencode_message_id` when present, else the stall watchdog's local
|
|
4492
|
+
* `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
|
|
4493
|
+
* `message` so every line below — and the signals this method posts —
|
|
4494
|
+
* keeps reporting the REAL server row; a shadow-copied `message` would
|
|
4495
|
+
* silently diverge from it.
|
|
4129
4496
|
*/
|
|
4130
|
-
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
4131
|
-
const ocId =
|
|
4497
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
|
|
4498
|
+
const ocId = effectiveOpencodeMessageId;
|
|
4132
4499
|
if (sessionCreated) {
|
|
4133
4500
|
this.clearRedriveUnresolved(message.id);
|
|
4134
4501
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
@@ -4358,7 +4725,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4358
4725
|
}
|
|
4359
4726
|
return "unresolved";
|
|
4360
4727
|
}
|
|
4361
|
-
/**
|
|
4728
|
+
/**
|
|
4729
|
+
* Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
|
|
4730
|
+
* outcome) — including the stall watchdog's local re-drive fence (#1618): once
|
|
4731
|
+
* `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
|
|
4732
|
+
* a real server-side `opencode_message_id` again or is no longer pending, so
|
|
4733
|
+
* the fence entry is no longer needed.
|
|
4734
|
+
*/
|
|
4362
4735
|
clearRedriveUnresolved(messageId) {
|
|
4363
4736
|
this.redriveUnresolvedSince.delete(messageId);
|
|
4364
4737
|
this.redriveUnresolvedSignalled.delete(messageId);
|
|
@@ -4366,6 +4739,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4366
4739
|
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4367
4740
|
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4368
4741
|
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
4742
|
+
this.releasedOpencodeIds.delete(messageId);
|
|
4369
4743
|
}
|
|
4370
4744
|
/**
|
|
4371
4745
|
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
@@ -4520,6 +4894,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4520
4894
|
this.supersededSessions.delete(oldest);
|
|
4521
4895
|
}
|
|
4522
4896
|
}
|
|
4897
|
+
/**
|
|
4898
|
+
* Record the local re-drive fence for a message force-released without
|
|
4899
|
+
* completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
|
|
4900
|
+
* `removeInFlight`, which is about to drop the `InFlightMessage` this reads
|
|
4901
|
+
* `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
|
|
4902
|
+
*/
|
|
4903
|
+
recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
|
|
4904
|
+
this.releasedOpencodeIds.delete(evidentMessageId);
|
|
4905
|
+
this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
|
|
4906
|
+
while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
|
|
4907
|
+
const oldest = this.releasedOpencodeIds.keys().next().value;
|
|
4908
|
+
if (oldest === void 0) return;
|
|
4909
|
+
this.releasedOpencodeIds.delete(oldest);
|
|
4910
|
+
}
|
|
4911
|
+
}
|
|
4523
4912
|
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
4524
4913
|
isSuperseded(conversationId, sessionId) {
|
|
4525
4914
|
return this.supersededSessions.get(conversationId) === sessionId;
|
|
@@ -4561,6 +4950,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4561
4950
|
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
4562
4951
|
conversation_id: conv.id
|
|
4563
4952
|
});
|
|
4953
|
+
const watcher = this.watchers.get(bound);
|
|
4954
|
+
if (watcher) {
|
|
4955
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
4956
|
+
this.recordReleasedOpencodeId(evidentMessageId, bound, inFlight.opencodeMessageId);
|
|
4957
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
4958
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
4959
|
+
recovery: "session_gone_released"
|
|
4960
|
+
});
|
|
4961
|
+
}
|
|
4962
|
+
this.watchers.delete(bound);
|
|
4963
|
+
}
|
|
4564
4964
|
this.sessions.delete(conv.id);
|
|
4565
4965
|
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
4566
4966
|
}
|
|
@@ -4751,15 +5151,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4751
5151
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
4752
5152
|
let watcher = this.watchers.get(sessionId);
|
|
4753
5153
|
if (!watcher) {
|
|
4754
|
-
watcher =
|
|
4755
|
-
conv,
|
|
4756
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4757
|
-
loop: null,
|
|
4758
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4759
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4760
|
-
lastGoodPollAt: this.now(),
|
|
4761
|
-
hadUsablePoll: false
|
|
4762
|
-
};
|
|
5154
|
+
watcher = this.newSessionWatcher(conv);
|
|
4763
5155
|
this.watchers.set(sessionId, watcher);
|
|
4764
5156
|
}
|
|
4765
5157
|
const now = this.now();
|
|
@@ -4790,6 +5182,27 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4790
5182
|
ambiguousResolved: false
|
|
4791
5183
|
});
|
|
4792
5184
|
}
|
|
5185
|
+
/**
|
|
5186
|
+
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
5187
|
+
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
5188
|
+
* misread as stalled by the very first reconciliation that sees it.
|
|
5189
|
+
*/
|
|
5190
|
+
newSessionWatcher(conv) {
|
|
5191
|
+
const now = this.now();
|
|
5192
|
+
return {
|
|
5193
|
+
conv,
|
|
5194
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
5195
|
+
loop: null,
|
|
5196
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
5197
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
5198
|
+
lastGoodPollAt: now,
|
|
5199
|
+
hadUsablePoll: false,
|
|
5200
|
+
generation: 0,
|
|
5201
|
+
lastTickAt: now,
|
|
5202
|
+
lastObservedTickAt: now,
|
|
5203
|
+
consecutiveStallRestarts: 0
|
|
5204
|
+
};
|
|
5205
|
+
}
|
|
4793
5206
|
/**
|
|
4794
5207
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
4795
5208
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
@@ -4819,15 +5232,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4819
5232
|
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
4820
5233
|
let watcher = this.watchers.get(sessionId);
|
|
4821
5234
|
if (!watcher) {
|
|
4822
|
-
watcher =
|
|
4823
|
-
conv,
|
|
4824
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4825
|
-
loop: null,
|
|
4826
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4827
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4828
|
-
lastGoodPollAt: this.now(),
|
|
4829
|
-
hadUsablePoll: false
|
|
4830
|
-
};
|
|
5235
|
+
watcher = this.newSessionWatcher(conv);
|
|
4831
5236
|
this.watchers.set(sessionId, watcher);
|
|
4832
5237
|
}
|
|
4833
5238
|
watcher.inFlight.set(message.id, {
|
|
@@ -4871,12 +5276,110 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4871
5276
|
ambiguousResolved: false
|
|
4872
5277
|
});
|
|
4873
5278
|
}
|
|
5279
|
+
/**
|
|
5280
|
+
* Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
|
|
5281
|
+
* restarts any per-session watcher whose loop has exited or stopped ticking
|
|
5282
|
+
* — escalating to a bounded force-release only once
|
|
5283
|
+
* `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
|
|
5284
|
+
* it. Fully synchronous: it only inspects in-memory state and calls the
|
|
5285
|
+
* synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
|
|
5286
|
+
* run from the very top of `drainPending()` — ahead of the un-timed
|
|
5287
|
+
* `getPendingConversations()` await that would otherwise be able to disable
|
|
5288
|
+
* it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
|
|
5289
|
+
* `drainPending()` from being CALLED again at all, not just from finishing).
|
|
5290
|
+
*
|
|
5291
|
+
* Restarts the loop rather than releasing messages directly: a blind release
|
|
5292
|
+
* would let the next drain re-`prompt_async` a turn that may still be
|
|
5293
|
+
* running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
|
|
5294
|
+
* re-polls with each message's `opencodeMessageId` still in hand and lets
|
|
5295
|
+
* the existing, audited `!activelyRunning` give-up decide, same as it always
|
|
5296
|
+
* has.
|
|
5297
|
+
*
|
|
5298
|
+
* Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
|
|
5299
|
+
* that shape has no in-flight entry and therefore no `opencodeMessageId` to
|
|
5300
|
+
* fence a release with, so releasing it here would blind-re-POST a possibly-
|
|
5301
|
+
* running turn — and there is no conversation id in hand to signal with
|
|
5302
|
+
* either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
|
|
5303
|
+
* instead, where a conversation id already exists. If you find yourself
|
|
5304
|
+
* wanting to add a `dispatched` sweep here, don't — read the drain-wedge
|
|
5305
|
+
* plan's §3/D5 first.
|
|
5306
|
+
*/
|
|
5307
|
+
reconcileWatchers() {
|
|
5308
|
+
const now = this.now();
|
|
5309
|
+
for (const [sessionId, watcher] of [...this.watchers]) {
|
|
5310
|
+
if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
|
|
5311
|
+
watcher.consecutiveStallRestarts = 0;
|
|
5312
|
+
}
|
|
5313
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5314
|
+
if (watcher.inFlight.size === 0 && watcher.loop === null) {
|
|
5315
|
+
this.watchers.delete(sessionId);
|
|
5316
|
+
continue;
|
|
5317
|
+
}
|
|
5318
|
+
if (watcher.loop === null && watcher.inFlight.size > 0) {
|
|
5319
|
+
if (now - watcher.lastTickAt < this.watcherStallMs) continue;
|
|
5320
|
+
this.log({
|
|
5321
|
+
level: "warn",
|
|
5322
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
|
|
5323
|
+
conversation_id: watcher.conv.id
|
|
5324
|
+
});
|
|
5325
|
+
this.ensureWatcherRunning(sessionId);
|
|
5326
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5327
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5328
|
+
recovery: "loop_exited"
|
|
5329
|
+
});
|
|
5330
|
+
}
|
|
5331
|
+
continue;
|
|
5332
|
+
}
|
|
5333
|
+
if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
|
|
5334
|
+
const stalledForMs = now - watcher.lastTickAt;
|
|
5335
|
+
watcher.consecutiveStallRestarts += 1;
|
|
5336
|
+
if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
|
|
5337
|
+
this.log({
|
|
5338
|
+
level: "error",
|
|
5339
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
|
|
5340
|
+
conversation_id: watcher.conv.id
|
|
5341
|
+
});
|
|
5342
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5343
|
+
this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
|
|
5344
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5345
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5346
|
+
recovery: "unrecoverable_released"
|
|
5347
|
+
});
|
|
5348
|
+
}
|
|
5349
|
+
watcher.generation += 1;
|
|
5350
|
+
this.watchers.delete(sessionId);
|
|
5351
|
+
continue;
|
|
5352
|
+
}
|
|
5353
|
+
watcher.generation += 1;
|
|
5354
|
+
watcher.loop = null;
|
|
5355
|
+
watcher.lastGoodPollAt = now;
|
|
5356
|
+
watcher.lastTickAt = now;
|
|
5357
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5358
|
+
this.ensureWatcherRunning(sessionId);
|
|
5359
|
+
this.log({
|
|
5360
|
+
level: "warn",
|
|
5361
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
|
|
5362
|
+
conversation_id: watcher.conv.id
|
|
5363
|
+
});
|
|
5364
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5365
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5366
|
+
recovery: "loop_stalled"
|
|
5367
|
+
});
|
|
5368
|
+
}
|
|
5369
|
+
}
|
|
5370
|
+
}
|
|
5371
|
+
}
|
|
4874
5372
|
/**
|
|
4875
5373
|
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
4876
5374
|
* work and is not already running. Single-flight per session. The loop is
|
|
4877
5375
|
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
4878
5376
|
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
4879
5377
|
* stays as the safety net.
|
|
5378
|
+
*
|
|
5379
|
+
* The generation started here (#1618) is captured in the `.finally` closure
|
|
5380
|
+
* so a RETIRED loop settling late — after `reconcileWatchers` has already
|
|
5381
|
+
* restarted this watcher under a newer generation — can neither null the new
|
|
5382
|
+
* loop's handle nor delete a watcher that still has live work.
|
|
4880
5383
|
*/
|
|
4881
5384
|
ensureWatcherRunning(sessionId) {
|
|
4882
5385
|
const watcher = this.watchers.get(sessionId);
|
|
@@ -4886,7 +5389,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4886
5389
|
this.watchers.delete(sessionId);
|
|
4887
5390
|
return;
|
|
4888
5391
|
}
|
|
4889
|
-
const
|
|
5392
|
+
const generation = watcher.generation;
|
|
5393
|
+
const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
|
|
5394
|
+
if (watcher.generation !== generation) return;
|
|
4890
5395
|
watcher.loop = null;
|
|
4891
5396
|
if (watcher.inFlight.size === 0) {
|
|
4892
5397
|
this.watchers.delete(sessionId);
|
|
@@ -4906,11 +5411,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4906
5411
|
* `source_message_id`;
|
|
4907
5412
|
* 4. drops messages that completed or timed out from the in-flight set.
|
|
4908
5413
|
* Exits when the in-flight set empties. Never throws.
|
|
5414
|
+
*
|
|
5415
|
+
* `generation` (#1618) is the incarnation this call was started under.
|
|
5416
|
+
* `reconcileWatchers` can restart a stalled loop by bumping
|
|
5417
|
+
* `watcher.generation` and starting a NEW `runWatcherLoop` over the same
|
|
5418
|
+
* `SessionWatcher` object — the stalled promise itself cannot be cancelled,
|
|
5419
|
+
* so this loop instead checks at the top of every iteration, right after
|
|
5420
|
+
* waking from `sleep`, and right before servicing any message, and quietly
|
|
5421
|
+
* retires (returns without touching anything) the moment it is no longer the
|
|
5422
|
+
* watcher's current generation. Retiring mid-tick can still let ONE
|
|
5423
|
+
* `serviceInFlightMessage` pass complete first — acceptable, since that
|
|
5424
|
+
* method contains no non-idempotent action.
|
|
4909
5425
|
*/
|
|
4910
|
-
async runWatcherLoop(sessionId, watcher) {
|
|
5426
|
+
async runWatcherLoop(sessionId, watcher, generation) {
|
|
4911
5427
|
try {
|
|
4912
5428
|
while (watcher.inFlight.size > 0) {
|
|
5429
|
+
if (watcher.generation !== generation) return;
|
|
5430
|
+
watcher.lastTickAt = this.now();
|
|
4913
5431
|
await this.sleep(this.pausedPollIntervalMs);
|
|
5432
|
+
if (watcher.generation !== generation) return;
|
|
4914
5433
|
let messages = null;
|
|
4915
5434
|
try {
|
|
4916
5435
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
@@ -4931,6 +5450,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4931
5450
|
}
|
|
4932
5451
|
}
|
|
4933
5452
|
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
5453
|
+
if (watcher.generation !== generation) return;
|
|
4934
5454
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
4935
5455
|
await this.serviceInFlightMessage(
|
|
4936
5456
|
sessionId,
|
|
@@ -6906,7 +7426,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
6906
7426
|
if (trimmed === "") {
|
|
6907
7427
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6908
7428
|
}
|
|
6909
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7429
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join5(homeDir, trimmed.slice(2)) : trimmed;
|
|
6910
7430
|
if (!isAbsolute2(expanded)) {
|
|
6911
7431
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6912
7432
|
}
|
|
@@ -7113,6 +7633,7 @@ async function driveChannels(state, driver) {
|
|
|
7113
7633
|
let unreachableMs = 0;
|
|
7114
7634
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7115
7635
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
7636
|
+
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
7116
7637
|
while (state.running) {
|
|
7117
7638
|
const cycleStartedAtMs = performance.now();
|
|
7118
7639
|
let idleThisCycle = false;
|
|
@@ -7136,11 +7657,15 @@ async function driveChannels(state, driver) {
|
|
|
7136
7657
|
state.messageCount += processed;
|
|
7137
7658
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
7138
7659
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7139
|
-
const
|
|
7660
|
+
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
7661
|
+
const appliedFiles = fileActivitySnapshot.appliedFiles;
|
|
7140
7662
|
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
7141
7663
|
const fileActivity = carriedOverFileSync || filesApplied;
|
|
7142
7664
|
lastSeenAppliedFiles = appliedFiles;
|
|
7143
|
-
|
|
7665
|
+
const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
|
|
7666
|
+
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
7667
|
+
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
7668
|
+
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
7144
7669
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7145
7670
|
idlePolls = 0;
|
|
7146
7671
|
idleMs = 0;
|
|
@@ -7209,7 +7734,7 @@ async function driveChannels(state, driver) {
|
|
|
7209
7734
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7210
7735
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7211
7736
|
function sessionDbPath() {
|
|
7212
|
-
return
|
|
7737
|
+
return join5(homedir3(), ".local", "share", "opencode", "opencode.db");
|
|
7213
7738
|
}
|
|
7214
7739
|
async function runSweep(state, driver, config) {
|
|
7215
7740
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7320,9 +7845,6 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7320
7845
|
);
|
|
7321
7846
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
7322
7847
|
}
|
|
7323
|
-
function claudeUsageFailureStreakSuffix(consecutiveFailures) {
|
|
7324
|
-
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
7325
|
-
}
|
|
7326
7848
|
function scheduleClaudeUsageReporting(state, options) {
|
|
7327
7849
|
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
7328
7850
|
options.claudeUsageReporting,
|
|
@@ -7344,23 +7866,44 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7344
7866
|
return null;
|
|
7345
7867
|
}
|
|
7346
7868
|
let consecutiveFailures = 0;
|
|
7347
|
-
let
|
|
7869
|
+
let phase = "dormant";
|
|
7348
7870
|
let rearmRequested = false;
|
|
7871
|
+
const armProbe = () => {
|
|
7872
|
+
phase = "probe-pending";
|
|
7873
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7874
|
+
};
|
|
7349
7875
|
const scheduleNextTick = () => {
|
|
7350
|
-
|
|
7351
|
-
|
|
7876
|
+
if (rearmRequested) {
|
|
7877
|
+
rearmRequested = false;
|
|
7878
|
+
armProbe();
|
|
7879
|
+
return;
|
|
7880
|
+
}
|
|
7881
|
+
phase = "steady-pending";
|
|
7352
7882
|
state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
|
|
7353
7883
|
};
|
|
7354
7884
|
const rearm = () => {
|
|
7355
|
-
|
|
7356
|
-
|
|
7357
|
-
|
|
7885
|
+
switch (phase) {
|
|
7886
|
+
case "tick-in-flight":
|
|
7887
|
+
rearmRequested = true;
|
|
7888
|
+
return;
|
|
7889
|
+
case "probe-pending":
|
|
7890
|
+
return;
|
|
7891
|
+
case "steady-pending":
|
|
7892
|
+
if (state.claudeUsageTimer) {
|
|
7893
|
+
clearTimeout(state.claudeUsageTimer);
|
|
7894
|
+
state.claudeUsageTimer = null;
|
|
7895
|
+
}
|
|
7896
|
+
rearmRequested = false;
|
|
7897
|
+
armProbe();
|
|
7898
|
+
return;
|
|
7899
|
+
case "dormant":
|
|
7900
|
+
rearmRequested = false;
|
|
7901
|
+
armProbe();
|
|
7902
|
+
return;
|
|
7358
7903
|
}
|
|
7359
|
-
rearmRequested = false;
|
|
7360
|
-
armed = true;
|
|
7361
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7362
7904
|
};
|
|
7363
7905
|
const tick = async (isProbe) => {
|
|
7906
|
+
phase = "tick-in-flight";
|
|
7364
7907
|
try {
|
|
7365
7908
|
const usage = await getClaudeUsage();
|
|
7366
7909
|
const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
|
|
@@ -7383,7 +7926,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7383
7926
|
logActivity(state, {
|
|
7384
7927
|
type: "info",
|
|
7385
7928
|
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7386
|
-
message: `Failed to report Claude usage: ${result.error}${
|
|
7929
|
+
message: `Failed to report Claude usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
7387
7930
|
});
|
|
7388
7931
|
}
|
|
7389
7932
|
scheduleNextTick();
|
|
@@ -7402,7 +7945,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7402
7945
|
level: "debug",
|
|
7403
7946
|
message: `Claude usage reporting: ${error2.message}`
|
|
7404
7947
|
});
|
|
7405
|
-
|
|
7948
|
+
phase = "dormant";
|
|
7406
7949
|
if (rearmRequested) rearm();
|
|
7407
7950
|
} else {
|
|
7408
7951
|
logActivity(state, {
|
|
@@ -7418,16 +7961,92 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7418
7961
|
logActivity(state, {
|
|
7419
7962
|
type: "info",
|
|
7420
7963
|
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7421
|
-
message: `Claude usage reporting failed: ${message}${
|
|
7964
|
+
message: `Claude usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
7422
7965
|
});
|
|
7423
7966
|
scheduleNextTick();
|
|
7424
7967
|
}
|
|
7425
7968
|
}
|
|
7426
7969
|
};
|
|
7427
|
-
|
|
7428
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7970
|
+
armProbe();
|
|
7429
7971
|
return rearm;
|
|
7430
7972
|
}
|
|
7973
|
+
var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
7974
|
+
var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
7975
|
+
var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
7976
|
+
function scheduleResourceUsageReporting(state, options) {
|
|
7977
|
+
const { enabled, warnings } = resolveResourceUsageReportingEnabled(
|
|
7978
|
+
options.resourceUsageReporting,
|
|
7979
|
+
process.env
|
|
7980
|
+
);
|
|
7981
|
+
for (const warning2 of warnings) {
|
|
7982
|
+
logActivity(state, {
|
|
7983
|
+
type: "info",
|
|
7984
|
+
level: "warn",
|
|
7985
|
+
message: `Resource usage reporting: ${warning2}`
|
|
7986
|
+
});
|
|
7987
|
+
}
|
|
7988
|
+
if (!enabled) {
|
|
7989
|
+
logActivity(state, {
|
|
7990
|
+
type: "info",
|
|
7991
|
+
level: "debug",
|
|
7992
|
+
message: "Resource usage reporting is off (--no-resource-usage-reporting)"
|
|
7993
|
+
});
|
|
7994
|
+
return;
|
|
7995
|
+
}
|
|
7996
|
+
const collect = createResourceUsageCollector();
|
|
7997
|
+
let consecutiveFailures = 0;
|
|
7998
|
+
const tick = async () => {
|
|
7999
|
+
try {
|
|
8000
|
+
const usage = collect();
|
|
8001
|
+
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
8002
|
+
if (result.ok) {
|
|
8003
|
+
if (consecutiveFailures > 0) {
|
|
8004
|
+
logActivity(state, {
|
|
8005
|
+
type: "info",
|
|
8006
|
+
level: "info",
|
|
8007
|
+
message: "Resource usage reporting recovered"
|
|
8008
|
+
});
|
|
8009
|
+
}
|
|
8010
|
+
consecutiveFailures = 0;
|
|
8011
|
+
logActivity(state, {
|
|
8012
|
+
type: "info",
|
|
8013
|
+
level: "debug",
|
|
8014
|
+
message: "Reported resource usage to Evident"
|
|
8015
|
+
});
|
|
8016
|
+
} else {
|
|
8017
|
+
consecutiveFailures++;
|
|
8018
|
+
logActivity(state, {
|
|
8019
|
+
type: "info",
|
|
8020
|
+
level: reportFailureLogLevel(
|
|
8021
|
+
consecutiveFailures,
|
|
8022
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8023
|
+
),
|
|
8024
|
+
message: `Failed to report resource usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
8025
|
+
});
|
|
8026
|
+
}
|
|
8027
|
+
} catch (error2) {
|
|
8028
|
+
consecutiveFailures++;
|
|
8029
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8030
|
+
logActivity(state, {
|
|
8031
|
+
type: "info",
|
|
8032
|
+
level: reportFailureLogLevel(
|
|
8033
|
+
consecutiveFailures,
|
|
8034
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8035
|
+
),
|
|
8036
|
+
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
8037
|
+
});
|
|
8038
|
+
} finally {
|
|
8039
|
+
state.resourceUsageTimer = setTimeout(
|
|
8040
|
+
() => void tick(),
|
|
8041
|
+
jitteredDelayMs(
|
|
8042
|
+
RESOURCE_USAGE_BASE_REPORT_DELAY_MS,
|
|
8043
|
+
RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION
|
|
8044
|
+
)
|
|
8045
|
+
);
|
|
8046
|
+
}
|
|
8047
|
+
};
|
|
8048
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
8049
|
+
}
|
|
7431
8050
|
async function notifyOffline(state) {
|
|
7432
8051
|
if (!state.agentId || !state.authHeader) return;
|
|
7433
8052
|
if (!state.connected) {
|
|
@@ -7468,6 +8087,10 @@ async function cleanup(state, opts = {}) {
|
|
|
7468
8087
|
state.claudeUsageTimer = null;
|
|
7469
8088
|
}
|
|
7470
8089
|
state.claudeUsageRearm = null;
|
|
8090
|
+
if (state.resourceUsageTimer) {
|
|
8091
|
+
clearTimeout(state.resourceUsageTimer);
|
|
8092
|
+
state.resourceUsageTimer = null;
|
|
8093
|
+
}
|
|
7471
8094
|
if (opts.graceful && state.channelDriver) {
|
|
7472
8095
|
state.channelDriver.stop();
|
|
7473
8096
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -7550,6 +8173,7 @@ async function run(options) {
|
|
|
7550
8173
|
sessionCleanupTimers: [],
|
|
7551
8174
|
claudeUsageTimer: null,
|
|
7552
8175
|
claudeUsageRearm: null,
|
|
8176
|
+
resourceUsageTimer: null,
|
|
7553
8177
|
authHeader: ""
|
|
7554
8178
|
};
|
|
7555
8179
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
@@ -7889,6 +8513,13 @@ async function run(options) {
|
|
|
7889
8513
|
logActivity(state, { type: "error", error: error2 });
|
|
7890
8514
|
if (state.interactive) displayStatus(state);
|
|
7891
8515
|
},
|
|
8516
|
+
// `warn`, not `info`: `forwardRunnerActivity`'s FORWARDED_LEVELS floor is
|
|
8517
|
+
// {'warn','error'}, so an `info` entry would never leave the machine and
|
|
8518
|
+
// an operator couldn't correlate a reconnect storm with a relay deploy.
|
|
8519
|
+
onWarning: (message) => {
|
|
8520
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
8521
|
+
if (state.interactive) displayStatus(state);
|
|
8522
|
+
},
|
|
7892
8523
|
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
7893
8524
|
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
7894
8525
|
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
@@ -7942,6 +8573,7 @@ async function run(options) {
|
|
|
7942
8573
|
}
|
|
7943
8574
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
7944
8575
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
8576
|
+
scheduleResourceUsageReporting(state, options);
|
|
7945
8577
|
if (!interactive || state.json) {
|
|
7946
8578
|
log2(state, "Driving channel messages...");
|
|
7947
8579
|
}
|
|
@@ -8022,6 +8654,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8022
8654
|
).option(
|
|
8023
8655
|
"--claude-usage-reporting <mode>",
|
|
8024
8656
|
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
8657
|
+
).option(
|
|
8658
|
+
"--no-resource-usage-reporting",
|
|
8659
|
+
"Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
|
|
8025
8660
|
).option(
|
|
8026
8661
|
"--enable-file-sync-to <dir>",
|
|
8027
8662
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
@@ -8054,6 +8689,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8054
8689
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
8055
8690
|
// (resolveClaudeUsageReportingMode).
|
|
8056
8691
|
claudeUsageReporting: options.claudeUsageReporting,
|
|
8692
|
+
// Raw value — resolution is single-sourced in run.ts's
|
|
8693
|
+
// resolveResourceUsageReportingEnabled.
|
|
8694
|
+
resourceUsageReporting: options.resourceUsageReporting,
|
|
8057
8695
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
8058
8696
|
// resolveFileSyncDirectories.
|
|
8059
8697
|
enableFileSyncTo: options.enableFileSyncTo,
|