@evident-ai/cli 3.3.1-dev.e98fa27 → 3.4.1-dev.74a16b2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/index.js +744 -81
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -746,6 +746,35 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
746
746
|
return { ok: false, error: describeBestEffortError(error2) };
|
|
747
747
|
}
|
|
748
748
|
}
|
|
749
|
+
async function reportResourceUsage(agentId, authHeader, usage) {
|
|
750
|
+
try {
|
|
751
|
+
const apiUrl = getApiUrlConfig();
|
|
752
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/resource-usage`, {
|
|
753
|
+
method: "POST",
|
|
754
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
755
|
+
body: JSON.stringify({
|
|
756
|
+
cpu_percent: usage.cpuPercent,
|
|
757
|
+
cpu_count: usage.cpuCount,
|
|
758
|
+
memory_total_bytes: usage.memoryTotalBytes,
|
|
759
|
+
memory_available_bytes: usage.memoryAvailableBytes,
|
|
760
|
+
disk_total_bytes: usage.diskTotalBytes,
|
|
761
|
+
disk_free_bytes: usage.diskFreeBytes,
|
|
762
|
+
opencode_db_bytes: usage.opencodeDbBytes
|
|
763
|
+
}),
|
|
764
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
765
|
+
});
|
|
766
|
+
if (!response.ok) {
|
|
767
|
+
const serverMessage = await readErrorMessage(response);
|
|
768
|
+
return {
|
|
769
|
+
ok: false,
|
|
770
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
return { ok: true };
|
|
774
|
+
} catch (error2) {
|
|
775
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
776
|
+
}
|
|
777
|
+
}
|
|
749
778
|
async function getAgentInfo(agentId, authHeader) {
|
|
750
779
|
const apiUrl = getApiUrlConfig();
|
|
751
780
|
try {
|
|
@@ -934,6 +963,7 @@ import { homedir } from "os";
|
|
|
934
963
|
import { join } from "path";
|
|
935
964
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
936
965
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
966
|
+
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
937
967
|
function parseClaudeCliCredentials(raw) {
|
|
938
968
|
let parsed;
|
|
939
969
|
try {
|
|
@@ -967,7 +997,7 @@ function readClaudeCliCredentials() {
|
|
|
967
997
|
}
|
|
968
998
|
}
|
|
969
999
|
try {
|
|
970
|
-
const raw = readFileSync(join(homedir(),
|
|
1000
|
+
const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
|
|
971
1001
|
return parseClaudeCliCredentials(raw);
|
|
972
1002
|
} catch (err) {
|
|
973
1003
|
const code = err.code;
|
|
@@ -1063,7 +1093,7 @@ async function claudeUsage() {
|
|
|
1063
1093
|
|
|
1064
1094
|
// src/commands/run.ts
|
|
1065
1095
|
import { homedir as homedir3 } from "os";
|
|
1066
|
-
import { isAbsolute as isAbsolute2, join as
|
|
1096
|
+
import { isAbsolute as isAbsolute2, join as join5, parse, resolve as resolvePath } from "path";
|
|
1067
1097
|
import chalk6 from "chalk";
|
|
1068
1098
|
|
|
1069
1099
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1700,13 +1730,22 @@ function buildNoProviderWarning(hasProvider) {
|
|
|
1700
1730
|
return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
|
|
1701
1731
|
}
|
|
1702
1732
|
|
|
1733
|
+
// src/lib/http-timeout.ts
|
|
1734
|
+
var REQUEST_TIMEOUT_MS = 6e4;
|
|
1735
|
+
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1736
|
+
return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1703
1739
|
// src/lib/opencode/session.ts
|
|
1740
|
+
function timedFetch(input, init) {
|
|
1741
|
+
return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
|
|
1742
|
+
}
|
|
1704
1743
|
function opencodeBase(port) {
|
|
1705
1744
|
return `http://127.0.0.1:${port}`;
|
|
1706
1745
|
}
|
|
1707
1746
|
async function getOpenCodeDirectory(port) {
|
|
1708
1747
|
try {
|
|
1709
|
-
const res = await
|
|
1748
|
+
const res = await timedFetch(`${opencodeBase(port)}/path`);
|
|
1710
1749
|
if (!res.ok) return null;
|
|
1711
1750
|
const body = await res.json();
|
|
1712
1751
|
const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
|
|
@@ -1757,7 +1796,7 @@ function isAssistantInFlight(m) {
|
|
|
1757
1796
|
}
|
|
1758
1797
|
async function getSessionMessages(port, sessionId) {
|
|
1759
1798
|
try {
|
|
1760
|
-
const res = await
|
|
1799
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1761
1800
|
if (!res.ok) return null;
|
|
1762
1801
|
const body = await res.json();
|
|
1763
1802
|
return Array.isArray(body) ? body : null;
|
|
@@ -1787,7 +1826,7 @@ function sessionLastActivityMs(session) {
|
|
|
1787
1826
|
}
|
|
1788
1827
|
async function listSessions(port) {
|
|
1789
1828
|
try {
|
|
1790
|
-
const res = await
|
|
1829
|
+
const res = await timedFetch(`${opencodeBase(port)}/session`);
|
|
1791
1830
|
if (!res.ok) return null;
|
|
1792
1831
|
const body = await res.json();
|
|
1793
1832
|
return Array.isArray(body) ? body : null;
|
|
@@ -1797,7 +1836,7 @@ async function listSessions(port) {
|
|
|
1797
1836
|
}
|
|
1798
1837
|
async function deleteSession(port, id) {
|
|
1799
1838
|
try {
|
|
1800
|
-
const res = await
|
|
1839
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1801
1840
|
return res.status >= 200 && res.status < 300;
|
|
1802
1841
|
} catch {
|
|
1803
1842
|
return false;
|
|
@@ -1805,7 +1844,7 @@ async function deleteSession(port, id) {
|
|
|
1805
1844
|
}
|
|
1806
1845
|
async function sessionExists(port, id) {
|
|
1807
1846
|
try {
|
|
1808
|
-
const res = await
|
|
1847
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
|
|
1809
1848
|
if (res.status >= 200 && res.status < 300) return true;
|
|
1810
1849
|
if (res.status === 404) return false;
|
|
1811
1850
|
return null;
|
|
@@ -1815,7 +1854,7 @@ async function sessionExists(port, id) {
|
|
|
1815
1854
|
}
|
|
1816
1855
|
async function getSessionStatuses(port) {
|
|
1817
1856
|
try {
|
|
1818
|
-
const res = await
|
|
1857
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/status`);
|
|
1819
1858
|
if (!res.ok) {
|
|
1820
1859
|
console.error(
|
|
1821
1860
|
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
@@ -1848,7 +1887,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1848
1887
|
if (directory && directory.trim()) {
|
|
1849
1888
|
url.searchParams.set("directory", directory.trim());
|
|
1850
1889
|
}
|
|
1851
|
-
const response = await
|
|
1890
|
+
const response = await timedFetch(url, {
|
|
1852
1891
|
method: "POST",
|
|
1853
1892
|
headers: { "Content-Type": "application/json" },
|
|
1854
1893
|
body: JSON.stringify({})
|
|
@@ -1862,7 +1901,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1862
1901
|
}
|
|
1863
1902
|
async function getModelAttachmentCapability(port, model) {
|
|
1864
1903
|
try {
|
|
1865
|
-
const res = await
|
|
1904
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1866
1905
|
if (!res.ok) {
|
|
1867
1906
|
console.error(
|
|
1868
1907
|
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -1995,7 +2034,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1995
2034
|
};
|
|
1996
2035
|
}
|
|
1997
2036
|
}
|
|
1998
|
-
const res = await
|
|
2037
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1999
2038
|
method: "POST",
|
|
2000
2039
|
headers: { "Content-Type": "application/json" },
|
|
2001
2040
|
body: JSON.stringify(body)
|
|
@@ -2244,7 +2283,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
2244
2283
|
}
|
|
2245
2284
|
async function hasAnyConfiguredProvider(port) {
|
|
2246
2285
|
try {
|
|
2247
|
-
const res = await
|
|
2286
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2248
2287
|
if (!res.ok) {
|
|
2249
2288
|
console.error(
|
|
2250
2289
|
`[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -2952,6 +2991,21 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
2952
2991
|
}
|
|
2953
2992
|
}
|
|
2954
2993
|
|
|
2994
|
+
// src/lib/reporting-schedule.ts
|
|
2995
|
+
function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
2996
|
+
const jitterRangeMs = baseMs * jitterFraction;
|
|
2997
|
+
return baseMs - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
2998
|
+
}
|
|
2999
|
+
function firstReportDelayMs(random = Math.random) {
|
|
3000
|
+
return 5e3 + random() * 1e4;
|
|
3001
|
+
}
|
|
3002
|
+
function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
|
|
3003
|
+
return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
|
|
3004
|
+
}
|
|
3005
|
+
function failureStreakSuffix(consecutiveFailures) {
|
|
3006
|
+
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
3007
|
+
}
|
|
3008
|
+
|
|
2955
3009
|
// src/lib/claude-usage-reporting.ts
|
|
2956
3010
|
var VALID_MODES = ["auto", "on", "off"];
|
|
2957
3011
|
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
@@ -2974,18 +3028,175 @@ function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
|
2974
3028
|
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
2975
3029
|
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
2976
3030
|
function nextReportDelayMs(random = Math.random) {
|
|
2977
|
-
|
|
2978
|
-
return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
3031
|
+
return jitteredDelayMs(BASE_REPORT_DELAY_MS, REPORT_DELAY_JITTER_FRACTION, random);
|
|
2979
3032
|
}
|
|
2980
|
-
var FIRST_REPORT_DELAY_MS =
|
|
3033
|
+
var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
|
|
2981
3034
|
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
2982
3035
|
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
2983
|
-
return consecutiveFailures
|
|
3036
|
+
return reportFailureLogLevel(consecutiveFailures, CLAUDE_USAGE_FAILURE_REESCALATION_TICKS);
|
|
3037
|
+
}
|
|
3038
|
+
|
|
3039
|
+
// src/lib/resource-usage-reporting.ts
|
|
3040
|
+
var ENABLED_VALUES = /* @__PURE__ */ new Set(["on", "true", "1"]);
|
|
3041
|
+
var DISABLED_VALUES = /* @__PURE__ */ new Set(["off", "false", "0"]);
|
|
3042
|
+
function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
3043
|
+
if (flagValue === false) {
|
|
3044
|
+
return { enabled: false, warnings: [] };
|
|
3045
|
+
}
|
|
3046
|
+
const raw = env.EVIDENT_RESOURCE_USAGE_REPORTING;
|
|
3047
|
+
if (raw === void 0 || raw === "") {
|
|
3048
|
+
return { enabled: true, warnings: [] };
|
|
3049
|
+
}
|
|
3050
|
+
const normalized = raw.trim().toLowerCase();
|
|
3051
|
+
if (DISABLED_VALUES.has(normalized)) {
|
|
3052
|
+
return { enabled: false, warnings: [] };
|
|
3053
|
+
}
|
|
3054
|
+
if (ENABLED_VALUES.has(normalized)) {
|
|
3055
|
+
return { enabled: true, warnings: [] };
|
|
3056
|
+
}
|
|
3057
|
+
return {
|
|
3058
|
+
enabled: true,
|
|
3059
|
+
warnings: [
|
|
3060
|
+
`Ignoring invalid EVIDENT_RESOURCE_USAGE_REPORTING "${raw}": expected on or off; leaving reporting on`
|
|
3061
|
+
]
|
|
3062
|
+
};
|
|
3063
|
+
}
|
|
3064
|
+
|
|
3065
|
+
// src/lib/resource-usage.ts
|
|
3066
|
+
import { cpus, totalmem, freemem } from "os";
|
|
3067
|
+
import { statfsSync as statfsSync2 } from "fs";
|
|
3068
|
+
|
|
3069
|
+
// src/lib/ecs-task-metadata.ts
|
|
3070
|
+
var ECS_METADATA_TIMEOUT_MS = 2e3;
|
|
3071
|
+
function parseEcsTaskLimits(payload) {
|
|
3072
|
+
if (typeof payload !== "object" || payload === null) return null;
|
|
3073
|
+
const limits = payload.Limits;
|
|
3074
|
+
if (typeof limits !== "object" || limits === null) return null;
|
|
3075
|
+
const cpu = limits.CPU;
|
|
3076
|
+
const memory = limits.Memory;
|
|
3077
|
+
if (typeof cpu !== "number" || !Number.isFinite(cpu) || cpu <= 0) return null;
|
|
3078
|
+
if (typeof memory !== "number" || !Number.isFinite(memory) || memory <= 0) return null;
|
|
3079
|
+
return {
|
|
3080
|
+
cpuCount: Math.max(1, Math.round(cpu)),
|
|
3081
|
+
memoryTotalBytes: memory * 1024 * 1024
|
|
3082
|
+
};
|
|
3083
|
+
}
|
|
3084
|
+
async function readEcsTaskLimits(env) {
|
|
3085
|
+
const uri = env.ECS_CONTAINER_METADATA_URI_V4;
|
|
3086
|
+
if (!uri) {
|
|
3087
|
+
return { limits: null };
|
|
3088
|
+
}
|
|
3089
|
+
const url = `${uri}/task`;
|
|
3090
|
+
try {
|
|
3091
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(ECS_METADATA_TIMEOUT_MS) });
|
|
3092
|
+
if (!response.ok) {
|
|
3093
|
+
return {
|
|
3094
|
+
limits: null,
|
|
3095
|
+
warning: `ECS task metadata fetch (${url}) returned HTTP ${response.status}`
|
|
3096
|
+
};
|
|
3097
|
+
}
|
|
3098
|
+
const payload = await response.json();
|
|
3099
|
+
const limits = parseEcsTaskLimits(payload);
|
|
3100
|
+
if (limits === null) {
|
|
3101
|
+
return {
|
|
3102
|
+
limits: null,
|
|
3103
|
+
warning: `ECS task metadata fetch (${url}) returned an unexpected payload`
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
return { limits };
|
|
3107
|
+
} catch (error2) {
|
|
3108
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3109
|
+
return { limits: null, warning: `ECS task metadata fetch (${url}) failed: ${message}` };
|
|
3110
|
+
}
|
|
3111
|
+
}
|
|
3112
|
+
|
|
3113
|
+
// src/lib/resource-usage.ts
|
|
3114
|
+
function readCpuSample() {
|
|
3115
|
+
let busyMs = 0;
|
|
3116
|
+
let idleMs = 0;
|
|
3117
|
+
for (const cpu of cpus()) {
|
|
3118
|
+
busyMs += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.irq;
|
|
3119
|
+
idleMs += cpu.times.idle;
|
|
3120
|
+
}
|
|
3121
|
+
return { busyMs, idleMs };
|
|
3122
|
+
}
|
|
3123
|
+
function cpuPercentBetween(previous, current) {
|
|
3124
|
+
const deltaBusy = current.busyMs - previous.busyMs;
|
|
3125
|
+
const deltaIdle = current.idleMs - previous.idleMs;
|
|
3126
|
+
const total = deltaBusy + deltaIdle;
|
|
3127
|
+
if (total === 0) return null;
|
|
3128
|
+
return Math.round((deltaBusy / total * 100 + Number.EPSILON) * 100) / 100;
|
|
3129
|
+
}
|
|
3130
|
+
function clamp(value, min, max) {
|
|
3131
|
+
return Math.min(Math.max(value, min), max);
|
|
3132
|
+
}
|
|
3133
|
+
function round2(value) {
|
|
3134
|
+
return Math.round((value + Number.EPSILON) * 100) / 100;
|
|
3135
|
+
}
|
|
3136
|
+
function readDisk(homeDir) {
|
|
3137
|
+
try {
|
|
3138
|
+
const stats = statfsSync2(homeDir);
|
|
3139
|
+
return {
|
|
3140
|
+
totalBytes: stats.bsize * stats.blocks,
|
|
3141
|
+
freeBytes: stats.bsize * stats.bavail
|
|
3142
|
+
};
|
|
3143
|
+
} catch (error2) {
|
|
3144
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3145
|
+
return {
|
|
3146
|
+
totalBytes: null,
|
|
3147
|
+
freeBytes: null,
|
|
3148
|
+
warning: `Could not read disk usage for ${homeDir}: ${message}`
|
|
3149
|
+
};
|
|
3150
|
+
}
|
|
3151
|
+
}
|
|
3152
|
+
function createResourceUsageCollector(homeDir) {
|
|
3153
|
+
let previous = readCpuSample();
|
|
3154
|
+
return async () => {
|
|
3155
|
+
const current = readCpuSample();
|
|
3156
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
3157
|
+
const hostCpuCount = cpus().length;
|
|
3158
|
+
previous = current;
|
|
3159
|
+
const disk = readDisk(homeDir);
|
|
3160
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
3161
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
3162
|
+
const warnings = [];
|
|
3163
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
3164
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
3165
|
+
let cpuPercent = hostCpuPercent;
|
|
3166
|
+
let cpuCount = hostCpuCount;
|
|
3167
|
+
let memoryTotalBytes = totalmem();
|
|
3168
|
+
let memoryAvailableBytes = freemem();
|
|
3169
|
+
if (limits !== null) {
|
|
3170
|
+
cpuCount = limits.cpuCount;
|
|
3171
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3172
|
+
memoryAvailableBytes = clamp(
|
|
3173
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3174
|
+
0,
|
|
3175
|
+
limits.memoryTotalBytes
|
|
3176
|
+
);
|
|
3177
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
3178
|
+
}
|
|
3179
|
+
return {
|
|
3180
|
+
usage: {
|
|
3181
|
+
cpuPercent,
|
|
3182
|
+
cpuCount,
|
|
3183
|
+
memoryTotalBytes,
|
|
3184
|
+
memoryAvailableBytes,
|
|
3185
|
+
diskTotalBytes: disk.totalBytes,
|
|
3186
|
+
diskFreeBytes: disk.freeBytes,
|
|
3187
|
+
opencodeDbBytes
|
|
3188
|
+
},
|
|
3189
|
+
warnings
|
|
3190
|
+
};
|
|
3191
|
+
};
|
|
2984
3192
|
}
|
|
2985
3193
|
|
|
2986
3194
|
// src/lib/channels/driver.ts
|
|
2987
3195
|
import { homedir as homedir2 } from "os";
|
|
2988
3196
|
|
|
3197
|
+
// src/lib/runner-file-sync.ts
|
|
3198
|
+
import { join as join4 } from "path";
|
|
3199
|
+
|
|
2989
3200
|
// src/lib/file-push.ts
|
|
2990
3201
|
import { randomUUID } from "crypto";
|
|
2991
3202
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
@@ -3179,17 +3390,20 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3179
3390
|
for (const id of options.ackFailures.keys()) {
|
|
3180
3391
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3181
3392
|
}
|
|
3182
|
-
if (pending.length === 0) return 0;
|
|
3393
|
+
if (pending.length === 0) return { applied: 0, claudeCredentialApplied: false };
|
|
3183
3394
|
options.log({
|
|
3184
3395
|
level: "info",
|
|
3185
3396
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3186
3397
|
});
|
|
3187
3398
|
let applied = 0;
|
|
3399
|
+
let claudeCredentialApplied = false;
|
|
3188
3400
|
for (const file of pending) {
|
|
3189
3401
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3190
|
-
|
|
3402
|
+
const outcome = await applyOne(options, file);
|
|
3403
|
+
if (outcome.applied) applied += 1;
|
|
3404
|
+
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
3191
3405
|
}
|
|
3192
|
-
return applied;
|
|
3406
|
+
return { applied, claudeCredentialApplied };
|
|
3193
3407
|
}
|
|
3194
3408
|
async function listPendingFiles(options) {
|
|
3195
3409
|
let res;
|
|
@@ -3250,6 +3464,11 @@ function asPendingFile(entry) {
|
|
|
3250
3464
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3251
3465
|
return { id, path, size };
|
|
3252
3466
|
}
|
|
3467
|
+
var NOT_APPLIED = { applied: false, claudeCredentialApplied: false };
|
|
3468
|
+
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3469
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join4(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3470
|
+
return expanded === join4(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3471
|
+
}
|
|
3253
3472
|
async function applyOne(options, file) {
|
|
3254
3473
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
3255
3474
|
if (options.allowedDirectories.length === 0) {
|
|
@@ -3258,7 +3477,7 @@ async function applyOne(options, file) {
|
|
|
3258
3477
|
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
3259
3478
|
});
|
|
3260
3479
|
await ack(options, file, "rejected", "file_sync_disabled");
|
|
3261
|
-
return
|
|
3480
|
+
return NOT_APPLIED;
|
|
3262
3481
|
}
|
|
3263
3482
|
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
3264
3483
|
options.log({
|
|
@@ -3266,12 +3485,12 @@ async function applyOne(options, file) {
|
|
|
3266
3485
|
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
3267
3486
|
});
|
|
3268
3487
|
await ack(options, file, "rejected", "file_too_large");
|
|
3269
|
-
return
|
|
3488
|
+
return NOT_APPLIED;
|
|
3270
3489
|
}
|
|
3271
3490
|
const download = await downloadContent(options, file, label);
|
|
3272
3491
|
if (!download.ok) {
|
|
3273
3492
|
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
3274
|
-
return
|
|
3493
|
+
return NOT_APPLIED;
|
|
3275
3494
|
}
|
|
3276
3495
|
let outcome;
|
|
3277
3496
|
try {
|
|
@@ -3287,7 +3506,7 @@ async function applyOne(options, file) {
|
|
|
3287
3506
|
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
3288
3507
|
});
|
|
3289
3508
|
await ack(options, file, "rejected", "write_failed");
|
|
3290
|
-
return
|
|
3509
|
+
return NOT_APPLIED;
|
|
3291
3510
|
}
|
|
3292
3511
|
if (!outcome.ok) {
|
|
3293
3512
|
options.log({
|
|
@@ -3295,14 +3514,17 @@ async function applyOne(options, file) {
|
|
|
3295
3514
|
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
3296
3515
|
});
|
|
3297
3516
|
await ack(options, file, "rejected", outcome.code);
|
|
3298
|
-
return
|
|
3517
|
+
return NOT_APPLIED;
|
|
3299
3518
|
}
|
|
3300
3519
|
options.log({
|
|
3301
3520
|
level: "info",
|
|
3302
3521
|
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
3303
3522
|
});
|
|
3304
3523
|
await ack(options, file, "applied");
|
|
3305
|
-
return
|
|
3524
|
+
return {
|
|
3525
|
+
applied: true,
|
|
3526
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir)
|
|
3527
|
+
};
|
|
3306
3528
|
}
|
|
3307
3529
|
function durableDownloadCode(status2) {
|
|
3308
3530
|
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
@@ -3413,8 +3635,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
|
3413
3635
|
var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
|
|
3414
3636
|
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
3415
3637
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
3638
|
+
var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
|
|
3639
|
+
var MAX_WATCHER_STALL_RESTARTS = 3;
|
|
3640
|
+
var MAX_RELEASED_OPENCODE_IDS = 256;
|
|
3416
3641
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3417
3642
|
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
3643
|
+
var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
|
|
3644
|
+
var MAX_WEDGED_CONVERSATIONS = 256;
|
|
3418
3645
|
var ChannelAuthError = class extends Error {
|
|
3419
3646
|
constructor(message) {
|
|
3420
3647
|
super(message);
|
|
@@ -3458,6 +3685,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3458
3685
|
fileSyncDirectories;
|
|
3459
3686
|
homeDir;
|
|
3460
3687
|
maxActiveSessions;
|
|
3688
|
+
watcherStallMs;
|
|
3689
|
+
wedgeWarningIntervalMs;
|
|
3461
3690
|
/** Cache of conversationId → opencode sessionId. */
|
|
3462
3691
|
sessions = /* @__PURE__ */ new Map();
|
|
3463
3692
|
/**
|
|
@@ -3488,6 +3717,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3488
3717
|
* bounded cost.
|
|
3489
3718
|
*/
|
|
3490
3719
|
supersededSessions = /* @__PURE__ */ new Map();
|
|
3720
|
+
/**
|
|
3721
|
+
* Local re-drive fence for a message force-released by the stall watchdog
|
|
3722
|
+
* (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
|
|
3723
|
+
* see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
|
|
3724
|
+
* is `null` for exactly this shape (its `markProcessing` never landed), so
|
|
3725
|
+
* without a local record of the id the driver last knew, the next drain's
|
|
3726
|
+
* `if (message.opencode_message_id)` re-drive-fence check at
|
|
3727
|
+
* `processConversation` would not engage and it would blind-`prompt_async`
|
|
3728
|
+
* a turn that may still be running in opencode — the one duplicate-turn
|
|
3729
|
+
* hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
|
|
3730
|
+
* `processConversation` reads `message.opencode_message_id ?? this
|
|
3731
|
+
* .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
|
|
3732
|
+
* threads it into `resolveRedrive`, which asks opencode itself whether the
|
|
3733
|
+
* turn is still ongoing before ever dispatching.
|
|
3734
|
+
*
|
|
3735
|
+
* Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
|
|
3736
|
+
* `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
|
|
3737
|
+
* non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
|
|
3738
|
+
* dispatch) and at the top-level fresh-dispatch site, so it does not outlive
|
|
3739
|
+
* the row it was recorded for.
|
|
3740
|
+
*/
|
|
3741
|
+
releasedOpencodeIds = /* @__PURE__ */ new Map();
|
|
3742
|
+
/**
|
|
3743
|
+
* Per-conversation throttle state for the #183 recurrence warning (#1618
|
|
3744
|
+
* WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
|
|
3745
|
+
* `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
|
|
3746
|
+
* log line and the `dispatch_wedged` signal to at most once per
|
|
3747
|
+
* `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
|
|
3748
|
+
* so the operator sees magnitude, not repetition. Cleared the moment the
|
|
3749
|
+
* conversation dispatches anything (a fresh wedge, if it recurs, is a new
|
|
3750
|
+
* incident). Bounded FIFO, mirroring `supersededSessions`
|
|
3751
|
+
* (`MAX_WEDGED_CONVERSATIONS`).
|
|
3752
|
+
*/
|
|
3753
|
+
wedgeWarnings = /* @__PURE__ */ new Map();
|
|
3491
3754
|
/**
|
|
3492
3755
|
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
3493
3756
|
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
@@ -3706,6 +3969,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3706
3969
|
* same trick `lastProxiedActivityAt` uses.
|
|
3707
3970
|
*/
|
|
3708
3971
|
appliedFileCount = 0;
|
|
3972
|
+
/**
|
|
3973
|
+
* Generation counter, NOT a tally (#1656): advances by exactly one per sync
|
|
3974
|
+
* batch that applied the Claude CLI credential file, not by how many
|
|
3975
|
+
* credential files were in that batch. `run.ts` only ever tests inequality
|
|
3976
|
+
* against the value it saw last cycle, so magnitude is meaningless — keep it
|
|
3977
|
+
* that way rather than "fixing" it into a count.
|
|
3978
|
+
*/
|
|
3979
|
+
claudeCredentialApplyCount = 0;
|
|
3709
3980
|
/**
|
|
3710
3981
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3711
3982
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -3730,7 +4001,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3730
4001
|
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3731
4002
|
this.log = config.log ?? (() => {
|
|
3732
4003
|
});
|
|
3733
|
-
this.fetchImpl =
|
|
4004
|
+
this.fetchImpl = withRequestTimeout(
|
|
4005
|
+
config.fetchImpl ?? fetch,
|
|
4006
|
+
config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
|
|
4007
|
+
);
|
|
3734
4008
|
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3735
4009
|
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3736
4010
|
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
@@ -3739,6 +4013,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3739
4013
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3740
4014
|
this.homeDir = config.homeDir ?? homedir2();
|
|
3741
4015
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4016
|
+
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4017
|
+
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
3742
4018
|
}
|
|
3743
4019
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
3744
4020
|
get opencodeBase() {
|
|
@@ -3752,6 +4028,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3752
4028
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
3753
4029
|
*/
|
|
3754
4030
|
async drainPending() {
|
|
4031
|
+
try {
|
|
4032
|
+
this.reconcileWatchers();
|
|
4033
|
+
} catch (err) {
|
|
4034
|
+
this.log({
|
|
4035
|
+
level: "error",
|
|
4036
|
+
message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
|
|
4037
|
+
});
|
|
4038
|
+
}
|
|
3755
4039
|
if (this.stopped) return 0;
|
|
3756
4040
|
if (this.draining) return 0;
|
|
3757
4041
|
this.draining = true;
|
|
@@ -3785,7 +4069,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3785
4069
|
if (this.syncingFiles) return 0;
|
|
3786
4070
|
this.syncingFiles = true;
|
|
3787
4071
|
try {
|
|
3788
|
-
const
|
|
4072
|
+
const result = await syncPendingRunnerFiles({
|
|
3789
4073
|
agentId: this.agentId,
|
|
3790
4074
|
apiUrl: this.apiUrl,
|
|
3791
4075
|
getAuthHeader: this.getAuthHeader,
|
|
@@ -3795,8 +4079,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3795
4079
|
ackFailures: this.fileAckFailures,
|
|
3796
4080
|
log: this.log
|
|
3797
4081
|
});
|
|
3798
|
-
this.appliedFileCount += applied;
|
|
3799
|
-
|
|
4082
|
+
this.appliedFileCount += result.applied;
|
|
4083
|
+
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
4084
|
+
return result.applied;
|
|
3800
4085
|
} catch (err) {
|
|
3801
4086
|
this.log({
|
|
3802
4087
|
level: "error",
|
|
@@ -3887,12 +4172,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3887
4172
|
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3888
4173
|
* idle checks still shows up as an advance.
|
|
3889
4174
|
*
|
|
4175
|
+
* A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
|
|
4176
|
+
* (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
|
|
4177
|
+
* reporting on it advancing, so an unrelated file sync can never disturb a
|
|
4178
|
+
* healthy reporting cadence (#1627) — it never even reaches that trigger, let
|
|
4179
|
+
* alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
|
|
4180
|
+
* whose consumer is idle-timeout suppression and must key on ANY file, not
|
|
4181
|
+
* just a Claude credential.
|
|
4182
|
+
*
|
|
3890
4183
|
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3891
4184
|
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3892
4185
|
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3893
4186
|
*/
|
|
3894
4187
|
fileSyncActivity() {
|
|
3895
|
-
return {
|
|
4188
|
+
return {
|
|
4189
|
+
appliedFiles: this.appliedFileCount,
|
|
4190
|
+
inFlight: this.syncingFiles,
|
|
4191
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount
|
|
4192
|
+
};
|
|
3896
4193
|
}
|
|
3897
4194
|
/**
|
|
3898
4195
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
@@ -4005,8 +4302,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4005
4302
|
skippedAlreadyDispatched += 1;
|
|
4006
4303
|
continue;
|
|
4007
4304
|
}
|
|
4008
|
-
|
|
4009
|
-
|
|
4305
|
+
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
4306
|
+
if (effectiveOpencodeMessageId) {
|
|
4307
|
+
const outcome = await this.resolveRedrive(
|
|
4308
|
+
conv,
|
|
4309
|
+
sessionId,
|
|
4310
|
+
message,
|
|
4311
|
+
sessionCreated,
|
|
4312
|
+
effectiveOpencodeMessageId
|
|
4313
|
+
);
|
|
4010
4314
|
if (outcome === "abandoned") {
|
|
4011
4315
|
continue;
|
|
4012
4316
|
}
|
|
@@ -4117,21 +4421,102 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4117
4421
|
}
|
|
4118
4422
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4119
4423
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
4424
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
4120
4425
|
this.dispatched.add(message.id);
|
|
4121
4426
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
4122
4427
|
dispatched += 1;
|
|
4123
4428
|
void this.postSignal(conv.id, message.id, "dispatched");
|
|
4124
4429
|
}
|
|
4125
4430
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
4126
|
-
this.
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
conversation_id: conv.id
|
|
4130
|
-
});
|
|
4431
|
+
this.reportWedgedConversation(conv, messages);
|
|
4432
|
+
} else if (dispatched > 0) {
|
|
4433
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4131
4434
|
}
|
|
4132
4435
|
this.ensureWatcherRunning(sessionId);
|
|
4133
4436
|
return dispatched;
|
|
4134
4437
|
}
|
|
4438
|
+
/**
|
|
4439
|
+
* The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
|
|
4440
|
+
* escalated (#1618 WI-4). `messages` is the conversation's full pending list
|
|
4441
|
+
* on THIS tick — the caller has already confirmed every one of them is a
|
|
4442
|
+
* skip-because-already-`dispatched`, the exact signature of a message stuck
|
|
4443
|
+
* acknowledged-but-never-worked.
|
|
4444
|
+
*
|
|
4445
|
+
* Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
|
|
4446
|
+
* (52,843 occurrences observed in one incident) — burning the GLOBAL
|
|
4447
|
+
* 30-events/60s `runner-activity-telemetry.ts` budget that was itself
|
|
4448
|
+
* suppressing the diagnostics needed to debug the wedge. The `warn` log (and
|
|
4449
|
+
* the `dispatch_wedged` signal once the wedge has persisted past the same
|
|
4450
|
+
* interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
|
|
4451
|
+
* naming the consecutive-tick count so the operator sees magnitude rather
|
|
4452
|
+
* than repetition.
|
|
4453
|
+
*
|
|
4454
|
+
* Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
|
|
4455
|
+
* unconditionally on this same tick and is already recovering anything it
|
|
4456
|
+
* can see. This is reporting only — see `countUntrackedIds`'s doc for the
|
|
4457
|
+
* one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
|
|
4458
|
+
*/
|
|
4459
|
+
reportWedgedConversation(conv, messages) {
|
|
4460
|
+
const now = this.now();
|
|
4461
|
+
const existing = this.wedgeWarnings.get(conv.id);
|
|
4462
|
+
const firstWedgedAt = existing?.firstWedgedAt ?? now;
|
|
4463
|
+
const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
|
|
4464
|
+
const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
|
|
4465
|
+
if (!dueForWarn) {
|
|
4466
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4467
|
+
this.wedgeWarnings.set(conv.id, {
|
|
4468
|
+
firstWedgedAt,
|
|
4469
|
+
lastWarnedAt: existing.lastWarnedAt,
|
|
4470
|
+
consecutiveTicks
|
|
4471
|
+
});
|
|
4472
|
+
return;
|
|
4473
|
+
}
|
|
4474
|
+
const stuckForMs = now - firstWedgedAt;
|
|
4475
|
+
const untracked = this.countUntrackedIds(messages);
|
|
4476
|
+
this.log({
|
|
4477
|
+
level: "warn",
|
|
4478
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
4479
|
+
conversation_id: conv.id
|
|
4480
|
+
});
|
|
4481
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4482
|
+
this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
|
|
4483
|
+
while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
|
|
4484
|
+
const oldest = this.wedgeWarnings.keys().next().value;
|
|
4485
|
+
if (oldest === void 0) break;
|
|
4486
|
+
this.wedgeWarnings.delete(oldest);
|
|
4487
|
+
}
|
|
4488
|
+
if (stuckForMs >= this.wedgeWarningIntervalMs) {
|
|
4489
|
+
void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
|
|
4490
|
+
stuck_for_ms: stuckForMs,
|
|
4491
|
+
untracked
|
|
4492
|
+
});
|
|
4493
|
+
}
|
|
4494
|
+
}
|
|
4495
|
+
/**
|
|
4496
|
+
* How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
|
|
4497
|
+
* WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
|
|
4498
|
+
* `dispatched`/`inFlight` pairing invariant holds by construction across
|
|
4499
|
+
* every `dispatched.add` site (see its own doc comment), so `> 0` here means
|
|
4500
|
+
* that invariant has actually been violated for this conversation: there is
|
|
4501
|
+
* no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
|
|
4502
|
+
* `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
|
|
4503
|
+
* recovering. One pass over `this.watchers`, called only when the throttled
|
|
4504
|
+
* warning above is due to fire — not every tick.
|
|
4505
|
+
*/
|
|
4506
|
+
countUntrackedIds(messages) {
|
|
4507
|
+
let untracked = 0;
|
|
4508
|
+
for (const message of messages) {
|
|
4509
|
+
let tracked = false;
|
|
4510
|
+
for (const watcher of this.watchers.values()) {
|
|
4511
|
+
if (watcher.inFlight.has(message.id)) {
|
|
4512
|
+
tracked = true;
|
|
4513
|
+
break;
|
|
4514
|
+
}
|
|
4515
|
+
}
|
|
4516
|
+
if (!tracked) untracked += 1;
|
|
4517
|
+
}
|
|
4518
|
+
return untracked;
|
|
4519
|
+
}
|
|
4135
4520
|
/**
|
|
4136
4521
|
* Poll a session's message list for the re-drive fence (#965), via the
|
|
4137
4522
|
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
@@ -4200,9 +4585,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4200
4585
|
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
4201
4586
|
* other failure resolves to `unresolved` and is retried whole on the next
|
|
4202
4587
|
* ~2s drain tick.
|
|
4588
|
+
*
|
|
4589
|
+
* `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
|
|
4590
|
+
* server `opencode_message_id` when present, else the stall watchdog's local
|
|
4591
|
+
* `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
|
|
4592
|
+
* `message` so every line below — and the signals this method posts —
|
|
4593
|
+
* keeps reporting the REAL server row; a shadow-copied `message` would
|
|
4594
|
+
* silently diverge from it.
|
|
4203
4595
|
*/
|
|
4204
|
-
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
4205
|
-
const ocId =
|
|
4596
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
|
|
4597
|
+
const ocId = effectiveOpencodeMessageId;
|
|
4206
4598
|
if (sessionCreated) {
|
|
4207
4599
|
this.clearRedriveUnresolved(message.id);
|
|
4208
4600
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
@@ -4432,7 +4824,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4432
4824
|
}
|
|
4433
4825
|
return "unresolved";
|
|
4434
4826
|
}
|
|
4435
|
-
/**
|
|
4827
|
+
/**
|
|
4828
|
+
* Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
|
|
4829
|
+
* outcome) — including the stall watchdog's local re-drive fence (#1618): once
|
|
4830
|
+
* `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
|
|
4831
|
+
* a real server-side `opencode_message_id` again or is no longer pending, so
|
|
4832
|
+
* the fence entry is no longer needed.
|
|
4833
|
+
*/
|
|
4436
4834
|
clearRedriveUnresolved(messageId) {
|
|
4437
4835
|
this.redriveUnresolvedSince.delete(messageId);
|
|
4438
4836
|
this.redriveUnresolvedSignalled.delete(messageId);
|
|
@@ -4440,6 +4838,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4440
4838
|
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4441
4839
|
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4442
4840
|
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
4841
|
+
this.releasedOpencodeIds.delete(messageId);
|
|
4443
4842
|
}
|
|
4444
4843
|
/**
|
|
4445
4844
|
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
@@ -4594,6 +4993,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4594
4993
|
this.supersededSessions.delete(oldest);
|
|
4595
4994
|
}
|
|
4596
4995
|
}
|
|
4996
|
+
/**
|
|
4997
|
+
* Record the local re-drive fence for a message force-released without
|
|
4998
|
+
* completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
|
|
4999
|
+
* `removeInFlight`, which is about to drop the `InFlightMessage` this reads
|
|
5000
|
+
* `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
|
|
5001
|
+
*/
|
|
5002
|
+
recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
|
|
5003
|
+
this.releasedOpencodeIds.delete(evidentMessageId);
|
|
5004
|
+
this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
|
|
5005
|
+
while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
|
|
5006
|
+
const oldest = this.releasedOpencodeIds.keys().next().value;
|
|
5007
|
+
if (oldest === void 0) return;
|
|
5008
|
+
this.releasedOpencodeIds.delete(oldest);
|
|
5009
|
+
}
|
|
5010
|
+
}
|
|
4597
5011
|
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
4598
5012
|
isSuperseded(conversationId, sessionId) {
|
|
4599
5013
|
return this.supersededSessions.get(conversationId) === sessionId;
|
|
@@ -4635,6 +5049,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4635
5049
|
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
4636
5050
|
conversation_id: conv.id
|
|
4637
5051
|
});
|
|
5052
|
+
const watcher = this.watchers.get(bound);
|
|
5053
|
+
if (watcher) {
|
|
5054
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5055
|
+
this.recordReleasedOpencodeId(evidentMessageId, bound, inFlight.opencodeMessageId);
|
|
5056
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5057
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5058
|
+
recovery: "session_gone_released"
|
|
5059
|
+
});
|
|
5060
|
+
}
|
|
5061
|
+
this.watchers.delete(bound);
|
|
5062
|
+
}
|
|
4638
5063
|
this.sessions.delete(conv.id);
|
|
4639
5064
|
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
4640
5065
|
}
|
|
@@ -4825,15 +5250,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4825
5250
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
4826
5251
|
let watcher = this.watchers.get(sessionId);
|
|
4827
5252
|
if (!watcher) {
|
|
4828
|
-
watcher =
|
|
4829
|
-
conv,
|
|
4830
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4831
|
-
loop: null,
|
|
4832
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4833
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4834
|
-
lastGoodPollAt: this.now(),
|
|
4835
|
-
hadUsablePoll: false
|
|
4836
|
-
};
|
|
5253
|
+
watcher = this.newSessionWatcher(conv);
|
|
4837
5254
|
this.watchers.set(sessionId, watcher);
|
|
4838
5255
|
}
|
|
4839
5256
|
const now = this.now();
|
|
@@ -4864,6 +5281,27 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4864
5281
|
ambiguousResolved: false
|
|
4865
5282
|
});
|
|
4866
5283
|
}
|
|
5284
|
+
/**
|
|
5285
|
+
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
5286
|
+
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
5287
|
+
* misread as stalled by the very first reconciliation that sees it.
|
|
5288
|
+
*/
|
|
5289
|
+
newSessionWatcher(conv) {
|
|
5290
|
+
const now = this.now();
|
|
5291
|
+
return {
|
|
5292
|
+
conv,
|
|
5293
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
5294
|
+
loop: null,
|
|
5295
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
5296
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
5297
|
+
lastGoodPollAt: now,
|
|
5298
|
+
hadUsablePoll: false,
|
|
5299
|
+
generation: 0,
|
|
5300
|
+
lastTickAt: now,
|
|
5301
|
+
lastObservedTickAt: now,
|
|
5302
|
+
consecutiveStallRestarts: 0
|
|
5303
|
+
};
|
|
5304
|
+
}
|
|
4867
5305
|
/**
|
|
4868
5306
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
4869
5307
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
@@ -4893,15 +5331,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4893
5331
|
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
4894
5332
|
let watcher = this.watchers.get(sessionId);
|
|
4895
5333
|
if (!watcher) {
|
|
4896
|
-
watcher =
|
|
4897
|
-
conv,
|
|
4898
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4899
|
-
loop: null,
|
|
4900
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4901
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4902
|
-
lastGoodPollAt: this.now(),
|
|
4903
|
-
hadUsablePoll: false
|
|
4904
|
-
};
|
|
5334
|
+
watcher = this.newSessionWatcher(conv);
|
|
4905
5335
|
this.watchers.set(sessionId, watcher);
|
|
4906
5336
|
}
|
|
4907
5337
|
watcher.inFlight.set(message.id, {
|
|
@@ -4945,12 +5375,110 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4945
5375
|
ambiguousResolved: false
|
|
4946
5376
|
});
|
|
4947
5377
|
}
|
|
5378
|
+
/**
|
|
5379
|
+
* Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
|
|
5380
|
+
* restarts any per-session watcher whose loop has exited or stopped ticking
|
|
5381
|
+
* — escalating to a bounded force-release only once
|
|
5382
|
+
* `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
|
|
5383
|
+
* it. Fully synchronous: it only inspects in-memory state and calls the
|
|
5384
|
+
* synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
|
|
5385
|
+
* run from the very top of `drainPending()` — ahead of the un-timed
|
|
5386
|
+
* `getPendingConversations()` await that would otherwise be able to disable
|
|
5387
|
+
* it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
|
|
5388
|
+
* `drainPending()` from being CALLED again at all, not just from finishing).
|
|
5389
|
+
*
|
|
5390
|
+
* Restarts the loop rather than releasing messages directly: a blind release
|
|
5391
|
+
* would let the next drain re-`prompt_async` a turn that may still be
|
|
5392
|
+
* running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
|
|
5393
|
+
* re-polls with each message's `opencodeMessageId` still in hand and lets
|
|
5394
|
+
* the existing, audited `!activelyRunning` give-up decide, same as it always
|
|
5395
|
+
* has.
|
|
5396
|
+
*
|
|
5397
|
+
* Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
|
|
5398
|
+
* that shape has no in-flight entry and therefore no `opencodeMessageId` to
|
|
5399
|
+
* fence a release with, so releasing it here would blind-re-POST a possibly-
|
|
5400
|
+
* running turn — and there is no conversation id in hand to signal with
|
|
5401
|
+
* either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
|
|
5402
|
+
* instead, where a conversation id already exists. If you find yourself
|
|
5403
|
+
* wanting to add a `dispatched` sweep here, don't — read the drain-wedge
|
|
5404
|
+
* plan's §3/D5 first.
|
|
5405
|
+
*/
|
|
5406
|
+
reconcileWatchers() {
|
|
5407
|
+
const now = this.now();
|
|
5408
|
+
for (const [sessionId, watcher] of [...this.watchers]) {
|
|
5409
|
+
if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
|
|
5410
|
+
watcher.consecutiveStallRestarts = 0;
|
|
5411
|
+
}
|
|
5412
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5413
|
+
if (watcher.inFlight.size === 0 && watcher.loop === null) {
|
|
5414
|
+
this.watchers.delete(sessionId);
|
|
5415
|
+
continue;
|
|
5416
|
+
}
|
|
5417
|
+
if (watcher.loop === null && watcher.inFlight.size > 0) {
|
|
5418
|
+
if (now - watcher.lastTickAt < this.watcherStallMs) continue;
|
|
5419
|
+
this.log({
|
|
5420
|
+
level: "warn",
|
|
5421
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
|
|
5422
|
+
conversation_id: watcher.conv.id
|
|
5423
|
+
});
|
|
5424
|
+
this.ensureWatcherRunning(sessionId);
|
|
5425
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5426
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5427
|
+
recovery: "loop_exited"
|
|
5428
|
+
});
|
|
5429
|
+
}
|
|
5430
|
+
continue;
|
|
5431
|
+
}
|
|
5432
|
+
if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
|
|
5433
|
+
const stalledForMs = now - watcher.lastTickAt;
|
|
5434
|
+
watcher.consecutiveStallRestarts += 1;
|
|
5435
|
+
if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
|
|
5436
|
+
this.log({
|
|
5437
|
+
level: "error",
|
|
5438
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
|
|
5439
|
+
conversation_id: watcher.conv.id
|
|
5440
|
+
});
|
|
5441
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5442
|
+
this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
|
|
5443
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5444
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5445
|
+
recovery: "unrecoverable_released"
|
|
5446
|
+
});
|
|
5447
|
+
}
|
|
5448
|
+
watcher.generation += 1;
|
|
5449
|
+
this.watchers.delete(sessionId);
|
|
5450
|
+
continue;
|
|
5451
|
+
}
|
|
5452
|
+
watcher.generation += 1;
|
|
5453
|
+
watcher.loop = null;
|
|
5454
|
+
watcher.lastGoodPollAt = now;
|
|
5455
|
+
watcher.lastTickAt = now;
|
|
5456
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5457
|
+
this.ensureWatcherRunning(sessionId);
|
|
5458
|
+
this.log({
|
|
5459
|
+
level: "warn",
|
|
5460
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
|
|
5461
|
+
conversation_id: watcher.conv.id
|
|
5462
|
+
});
|
|
5463
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5464
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5465
|
+
recovery: "loop_stalled"
|
|
5466
|
+
});
|
|
5467
|
+
}
|
|
5468
|
+
}
|
|
5469
|
+
}
|
|
5470
|
+
}
|
|
4948
5471
|
/**
|
|
4949
5472
|
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
4950
5473
|
* work and is not already running. Single-flight per session. The loop is
|
|
4951
5474
|
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
4952
5475
|
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
4953
5476
|
* stays as the safety net.
|
|
5477
|
+
*
|
|
5478
|
+
* The generation started here (#1618) is captured in the `.finally` closure
|
|
5479
|
+
* so a RETIRED loop settling late — after `reconcileWatchers` has already
|
|
5480
|
+
* restarted this watcher under a newer generation — can neither null the new
|
|
5481
|
+
* loop's handle nor delete a watcher that still has live work.
|
|
4954
5482
|
*/
|
|
4955
5483
|
ensureWatcherRunning(sessionId) {
|
|
4956
5484
|
const watcher = this.watchers.get(sessionId);
|
|
@@ -4960,7 +5488,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4960
5488
|
this.watchers.delete(sessionId);
|
|
4961
5489
|
return;
|
|
4962
5490
|
}
|
|
4963
|
-
const
|
|
5491
|
+
const generation = watcher.generation;
|
|
5492
|
+
const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
|
|
5493
|
+
if (watcher.generation !== generation) return;
|
|
4964
5494
|
watcher.loop = null;
|
|
4965
5495
|
if (watcher.inFlight.size === 0) {
|
|
4966
5496
|
this.watchers.delete(sessionId);
|
|
@@ -4980,11 +5510,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4980
5510
|
* `source_message_id`;
|
|
4981
5511
|
* 4. drops messages that completed or timed out from the in-flight set.
|
|
4982
5512
|
* Exits when the in-flight set empties. Never throws.
|
|
5513
|
+
*
|
|
5514
|
+
* `generation` (#1618) is the incarnation this call was started under.
|
|
5515
|
+
* `reconcileWatchers` can restart a stalled loop by bumping
|
|
5516
|
+
* `watcher.generation` and starting a NEW `runWatcherLoop` over the same
|
|
5517
|
+
* `SessionWatcher` object — the stalled promise itself cannot be cancelled,
|
|
5518
|
+
* so this loop instead checks at the top of every iteration, right after
|
|
5519
|
+
* waking from `sleep`, and right before servicing any message, and quietly
|
|
5520
|
+
* retires (returns without touching anything) the moment it is no longer the
|
|
5521
|
+
* watcher's current generation. Retiring mid-tick can still let ONE
|
|
5522
|
+
* `serviceInFlightMessage` pass complete first — acceptable, since that
|
|
5523
|
+
* method contains no non-idempotent action.
|
|
4983
5524
|
*/
|
|
4984
|
-
async runWatcherLoop(sessionId, watcher) {
|
|
5525
|
+
async runWatcherLoop(sessionId, watcher, generation) {
|
|
4985
5526
|
try {
|
|
4986
5527
|
while (watcher.inFlight.size > 0) {
|
|
5528
|
+
if (watcher.generation !== generation) return;
|
|
5529
|
+
watcher.lastTickAt = this.now();
|
|
4987
5530
|
await this.sleep(this.pausedPollIntervalMs);
|
|
5531
|
+
if (watcher.generation !== generation) return;
|
|
4988
5532
|
let messages = null;
|
|
4989
5533
|
try {
|
|
4990
5534
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
@@ -5005,6 +5549,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5005
5549
|
}
|
|
5006
5550
|
}
|
|
5007
5551
|
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
5552
|
+
if (watcher.generation !== generation) return;
|
|
5008
5553
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
5009
5554
|
await this.serviceInFlightMessage(
|
|
5010
5555
|
sessionId,
|
|
@@ -6980,7 +7525,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
6980
7525
|
if (trimmed === "") {
|
|
6981
7526
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6982
7527
|
}
|
|
6983
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7528
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join5(homeDir, trimmed.slice(2)) : trimmed;
|
|
6984
7529
|
if (!isAbsolute2(expanded)) {
|
|
6985
7530
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6986
7531
|
}
|
|
@@ -7187,6 +7732,7 @@ async function driveChannels(state, driver) {
|
|
|
7187
7732
|
let unreachableMs = 0;
|
|
7188
7733
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7189
7734
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
7735
|
+
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
7190
7736
|
while (state.running) {
|
|
7191
7737
|
const cycleStartedAtMs = performance.now();
|
|
7192
7738
|
let idleThisCycle = false;
|
|
@@ -7210,11 +7756,15 @@ async function driveChannels(state, driver) {
|
|
|
7210
7756
|
state.messageCount += processed;
|
|
7211
7757
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
7212
7758
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7213
|
-
const
|
|
7759
|
+
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
7760
|
+
const appliedFiles = fileActivitySnapshot.appliedFiles;
|
|
7214
7761
|
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
7215
7762
|
const fileActivity = carriedOverFileSync || filesApplied;
|
|
7216
7763
|
lastSeenAppliedFiles = appliedFiles;
|
|
7217
|
-
|
|
7764
|
+
const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
|
|
7765
|
+
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
7766
|
+
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
7767
|
+
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
7218
7768
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7219
7769
|
idlePolls = 0;
|
|
7220
7770
|
idleMs = 0;
|
|
@@ -7283,7 +7833,7 @@ async function driveChannels(state, driver) {
|
|
|
7283
7833
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7284
7834
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7285
7835
|
function sessionDbPath() {
|
|
7286
|
-
return
|
|
7836
|
+
return join5(homedir3(), ".local", "share", "opencode", "opencode.db");
|
|
7287
7837
|
}
|
|
7288
7838
|
async function runSweep(state, driver, config) {
|
|
7289
7839
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7394,9 +7944,6 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7394
7944
|
);
|
|
7395
7945
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
7396
7946
|
}
|
|
7397
|
-
function claudeUsageFailureStreakSuffix(consecutiveFailures) {
|
|
7398
|
-
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
7399
|
-
}
|
|
7400
7947
|
function scheduleClaudeUsageReporting(state, options) {
|
|
7401
7948
|
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
7402
7949
|
options.claudeUsageReporting,
|
|
@@ -7418,23 +7965,44 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7418
7965
|
return null;
|
|
7419
7966
|
}
|
|
7420
7967
|
let consecutiveFailures = 0;
|
|
7421
|
-
let
|
|
7968
|
+
let phase = "dormant";
|
|
7422
7969
|
let rearmRequested = false;
|
|
7970
|
+
const armProbe = () => {
|
|
7971
|
+
phase = "probe-pending";
|
|
7972
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7973
|
+
};
|
|
7423
7974
|
const scheduleNextTick = () => {
|
|
7424
|
-
|
|
7425
|
-
|
|
7975
|
+
if (rearmRequested) {
|
|
7976
|
+
rearmRequested = false;
|
|
7977
|
+
armProbe();
|
|
7978
|
+
return;
|
|
7979
|
+
}
|
|
7980
|
+
phase = "steady-pending";
|
|
7426
7981
|
state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
|
|
7427
7982
|
};
|
|
7428
7983
|
const rearm = () => {
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7984
|
+
switch (phase) {
|
|
7985
|
+
case "tick-in-flight":
|
|
7986
|
+
rearmRequested = true;
|
|
7987
|
+
return;
|
|
7988
|
+
case "probe-pending":
|
|
7989
|
+
return;
|
|
7990
|
+
case "steady-pending":
|
|
7991
|
+
if (state.claudeUsageTimer) {
|
|
7992
|
+
clearTimeout(state.claudeUsageTimer);
|
|
7993
|
+
state.claudeUsageTimer = null;
|
|
7994
|
+
}
|
|
7995
|
+
rearmRequested = false;
|
|
7996
|
+
armProbe();
|
|
7997
|
+
return;
|
|
7998
|
+
case "dormant":
|
|
7999
|
+
rearmRequested = false;
|
|
8000
|
+
armProbe();
|
|
8001
|
+
return;
|
|
7432
8002
|
}
|
|
7433
|
-
rearmRequested = false;
|
|
7434
|
-
armed = true;
|
|
7435
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7436
8003
|
};
|
|
7437
8004
|
const tick = async (isProbe) => {
|
|
8005
|
+
phase = "tick-in-flight";
|
|
7438
8006
|
try {
|
|
7439
8007
|
const usage = await getClaudeUsage();
|
|
7440
8008
|
const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
|
|
@@ -7457,7 +8025,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7457
8025
|
logActivity(state, {
|
|
7458
8026
|
type: "info",
|
|
7459
8027
|
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7460
|
-
message: `Failed to report Claude usage: ${result.error}${
|
|
8028
|
+
message: `Failed to report Claude usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
7461
8029
|
});
|
|
7462
8030
|
}
|
|
7463
8031
|
scheduleNextTick();
|
|
@@ -7476,7 +8044,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7476
8044
|
level: "debug",
|
|
7477
8045
|
message: `Claude usage reporting: ${error2.message}`
|
|
7478
8046
|
});
|
|
7479
|
-
|
|
8047
|
+
phase = "dormant";
|
|
7480
8048
|
if (rearmRequested) rearm();
|
|
7481
8049
|
} else {
|
|
7482
8050
|
logActivity(state, {
|
|
@@ -7492,16 +8060,99 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7492
8060
|
logActivity(state, {
|
|
7493
8061
|
type: "info",
|
|
7494
8062
|
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7495
|
-
message: `Claude usage reporting failed: ${message}${
|
|
8063
|
+
message: `Claude usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
7496
8064
|
});
|
|
7497
8065
|
scheduleNextTick();
|
|
7498
8066
|
}
|
|
7499
8067
|
}
|
|
7500
8068
|
};
|
|
7501
|
-
|
|
7502
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
8069
|
+
armProbe();
|
|
7503
8070
|
return rearm;
|
|
7504
8071
|
}
|
|
8072
|
+
var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
8073
|
+
var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
8074
|
+
var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
8075
|
+
function scheduleResourceUsageReporting(state, options) {
|
|
8076
|
+
const { enabled, warnings } = resolveResourceUsageReportingEnabled(
|
|
8077
|
+
options.resourceUsageReporting,
|
|
8078
|
+
process.env
|
|
8079
|
+
);
|
|
8080
|
+
for (const warning2 of warnings) {
|
|
8081
|
+
logActivity(state, {
|
|
8082
|
+
type: "info",
|
|
8083
|
+
level: "warn",
|
|
8084
|
+
message: `Resource usage reporting: ${warning2}`
|
|
8085
|
+
});
|
|
8086
|
+
}
|
|
8087
|
+
if (!enabled) {
|
|
8088
|
+
logActivity(state, {
|
|
8089
|
+
type: "info",
|
|
8090
|
+
level: "debug",
|
|
8091
|
+
message: "Resource usage reporting is off (--no-resource-usage-reporting)"
|
|
8092
|
+
});
|
|
8093
|
+
return;
|
|
8094
|
+
}
|
|
8095
|
+
const collect = createResourceUsageCollector(homedir3());
|
|
8096
|
+
let consecutiveFailures = 0;
|
|
8097
|
+
const tick = async () => {
|
|
8098
|
+
try {
|
|
8099
|
+
const { usage, warnings: collectWarnings } = await collect();
|
|
8100
|
+
for (const warning2 of collectWarnings) {
|
|
8101
|
+
logActivity(state, {
|
|
8102
|
+
type: "info",
|
|
8103
|
+
level: "debug",
|
|
8104
|
+
message: `Resource usage collection: ${warning2}`
|
|
8105
|
+
});
|
|
8106
|
+
}
|
|
8107
|
+
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
8108
|
+
if (result.ok) {
|
|
8109
|
+
if (consecutiveFailures > 0) {
|
|
8110
|
+
logActivity(state, {
|
|
8111
|
+
type: "info",
|
|
8112
|
+
level: "info",
|
|
8113
|
+
message: "Resource usage reporting recovered"
|
|
8114
|
+
});
|
|
8115
|
+
}
|
|
8116
|
+
consecutiveFailures = 0;
|
|
8117
|
+
logActivity(state, {
|
|
8118
|
+
type: "info",
|
|
8119
|
+
level: "debug",
|
|
8120
|
+
message: "Reported resource usage to Evident"
|
|
8121
|
+
});
|
|
8122
|
+
} else {
|
|
8123
|
+
consecutiveFailures++;
|
|
8124
|
+
logActivity(state, {
|
|
8125
|
+
type: "info",
|
|
8126
|
+
level: reportFailureLogLevel(
|
|
8127
|
+
consecutiveFailures,
|
|
8128
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8129
|
+
),
|
|
8130
|
+
message: `Failed to report resource usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
8131
|
+
});
|
|
8132
|
+
}
|
|
8133
|
+
} catch (error2) {
|
|
8134
|
+
consecutiveFailures++;
|
|
8135
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8136
|
+
logActivity(state, {
|
|
8137
|
+
type: "info",
|
|
8138
|
+
level: reportFailureLogLevel(
|
|
8139
|
+
consecutiveFailures,
|
|
8140
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8141
|
+
),
|
|
8142
|
+
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
8143
|
+
});
|
|
8144
|
+
} finally {
|
|
8145
|
+
state.resourceUsageTimer = setTimeout(
|
|
8146
|
+
() => void tick(),
|
|
8147
|
+
jitteredDelayMs(
|
|
8148
|
+
RESOURCE_USAGE_BASE_REPORT_DELAY_MS,
|
|
8149
|
+
RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION
|
|
8150
|
+
)
|
|
8151
|
+
);
|
|
8152
|
+
}
|
|
8153
|
+
};
|
|
8154
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
8155
|
+
}
|
|
7505
8156
|
async function notifyOffline(state) {
|
|
7506
8157
|
if (!state.agentId || !state.authHeader) return;
|
|
7507
8158
|
if (!state.connected) {
|
|
@@ -7542,6 +8193,10 @@ async function cleanup(state, opts = {}) {
|
|
|
7542
8193
|
state.claudeUsageTimer = null;
|
|
7543
8194
|
}
|
|
7544
8195
|
state.claudeUsageRearm = null;
|
|
8196
|
+
if (state.resourceUsageTimer) {
|
|
8197
|
+
clearTimeout(state.resourceUsageTimer);
|
|
8198
|
+
state.resourceUsageTimer = null;
|
|
8199
|
+
}
|
|
7545
8200
|
if (opts.graceful && state.channelDriver) {
|
|
7546
8201
|
state.channelDriver.stop();
|
|
7547
8202
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -7624,6 +8279,7 @@ async function run(options) {
|
|
|
7624
8279
|
sessionCleanupTimers: [],
|
|
7625
8280
|
claudeUsageTimer: null,
|
|
7626
8281
|
claudeUsageRearm: null,
|
|
8282
|
+
resourceUsageTimer: null,
|
|
7627
8283
|
authHeader: ""
|
|
7628
8284
|
};
|
|
7629
8285
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
@@ -8023,6 +8679,7 @@ async function run(options) {
|
|
|
8023
8679
|
}
|
|
8024
8680
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
8025
8681
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
8682
|
+
scheduleResourceUsageReporting(state, options);
|
|
8026
8683
|
if (!interactive || state.json) {
|
|
8027
8684
|
log2(state, "Driving channel messages...");
|
|
8028
8685
|
}
|
|
@@ -8103,6 +8760,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8103
8760
|
).option(
|
|
8104
8761
|
"--claude-usage-reporting <mode>",
|
|
8105
8762
|
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
8763
|
+
).option(
|
|
8764
|
+
"--no-resource-usage-reporting",
|
|
8765
|
+
"Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
|
|
8106
8766
|
).option(
|
|
8107
8767
|
"--enable-file-sync-to <dir>",
|
|
8108
8768
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
@@ -8135,6 +8795,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8135
8795
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
8136
8796
|
// (resolveClaudeUsageReportingMode).
|
|
8137
8797
|
claudeUsageReporting: options.claudeUsageReporting,
|
|
8798
|
+
// Raw value — resolution is single-sourced in run.ts's
|
|
8799
|
+
// resolveResourceUsageReportingEnabled.
|
|
8800
|
+
resourceUsageReporting: options.resourceUsageReporting,
|
|
8138
8801
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
8139
8802
|
// resolveFileSyncDirectories.
|
|
8140
8803
|
enableFileSyncTo: options.enableFileSyncTo,
|