@evident-ai/cli 3.3.1-dev.e98fa27 → 3.4.1-dev.15755a4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/dist/index.js +1264 -126
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -746,6 +746,69 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
746
746
|
return { ok: false, error: describeBestEffortError(error2) };
|
|
747
747
|
}
|
|
748
748
|
}
|
|
749
|
+
function toReportedOpenAiWindow(window) {
|
|
750
|
+
if (!window) return null;
|
|
751
|
+
return {
|
|
752
|
+
utilization: window.utilization,
|
|
753
|
+
window_minutes: window.windowMinutes,
|
|
754
|
+
resets_at: window.resetsAt
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
758
|
+
try {
|
|
759
|
+
const apiUrl = getApiUrlConfig();
|
|
760
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/openai-usage`, {
|
|
761
|
+
method: "POST",
|
|
762
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
763
|
+
body: JSON.stringify({
|
|
764
|
+
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
765
|
+
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
766
|
+
has_credits: snapshot.hasCredits,
|
|
767
|
+
credits_unlimited: snapshot.creditsUnlimited
|
|
768
|
+
}),
|
|
769
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
770
|
+
});
|
|
771
|
+
if (!response.ok) {
|
|
772
|
+
const serverMessage = await readErrorMessage(response);
|
|
773
|
+
return {
|
|
774
|
+
ok: false,
|
|
775
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
return { ok: true };
|
|
779
|
+
} catch (error2) {
|
|
780
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
async function reportResourceUsage(agentId, authHeader, usage) {
|
|
784
|
+
try {
|
|
785
|
+
const apiUrl = getApiUrlConfig();
|
|
786
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/resource-usage`, {
|
|
787
|
+
method: "POST",
|
|
788
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
789
|
+
body: JSON.stringify({
|
|
790
|
+
cpu_percent: usage.cpuPercent,
|
|
791
|
+
cpu_count: usage.cpuCount,
|
|
792
|
+
memory_total_bytes: usage.memoryTotalBytes,
|
|
793
|
+
memory_available_bytes: usage.memoryAvailableBytes,
|
|
794
|
+
disk_total_bytes: usage.diskTotalBytes,
|
|
795
|
+
disk_free_bytes: usage.diskFreeBytes,
|
|
796
|
+
opencode_db_bytes: usage.opencodeDbBytes
|
|
797
|
+
}),
|
|
798
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
799
|
+
});
|
|
800
|
+
if (!response.ok) {
|
|
801
|
+
const serverMessage = await readErrorMessage(response);
|
|
802
|
+
return {
|
|
803
|
+
ok: false,
|
|
804
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
805
|
+
};
|
|
806
|
+
}
|
|
807
|
+
return { ok: true };
|
|
808
|
+
} catch (error2) {
|
|
809
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
810
|
+
}
|
|
811
|
+
}
|
|
749
812
|
async function getAgentInfo(agentId, authHeader) {
|
|
750
813
|
const apiUrl = getApiUrlConfig();
|
|
751
814
|
try {
|
|
@@ -934,6 +997,7 @@ import { homedir } from "os";
|
|
|
934
997
|
import { join } from "path";
|
|
935
998
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
936
999
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
1000
|
+
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
937
1001
|
function parseClaudeCliCredentials(raw) {
|
|
938
1002
|
let parsed;
|
|
939
1003
|
try {
|
|
@@ -967,7 +1031,7 @@ function readClaudeCliCredentials() {
|
|
|
967
1031
|
}
|
|
968
1032
|
}
|
|
969
1033
|
try {
|
|
970
|
-
const raw = readFileSync(join(homedir(),
|
|
1034
|
+
const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
|
|
971
1035
|
return parseClaudeCliCredentials(raw);
|
|
972
1036
|
} catch (err) {
|
|
973
1037
|
const code = err.code;
|
|
@@ -1062,8 +1126,8 @@ async function claudeUsage() {
|
|
|
1062
1126
|
}
|
|
1063
1127
|
|
|
1064
1128
|
// src/commands/run.ts
|
|
1065
|
-
import { homedir as
|
|
1066
|
-
import { isAbsolute as isAbsolute2, join as
|
|
1129
|
+
import { homedir as homedir4 } from "os";
|
|
1130
|
+
import { isAbsolute as isAbsolute2, join as join7, parse, resolve as resolvePath } from "path";
|
|
1067
1131
|
import chalk6 from "chalk";
|
|
1068
1132
|
|
|
1069
1133
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1296,6 +1360,8 @@ var SEVERITY_BY_LEVEL = {
|
|
|
1296
1360
|
error: "error"
|
|
1297
1361
|
};
|
|
1298
1362
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1363
|
+
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1364
|
+
var MAX_METADATA_ENTRIES = 20;
|
|
1299
1365
|
var TRUNCATION_MARKER = "\u2026";
|
|
1300
1366
|
function redact(message) {
|
|
1301
1367
|
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
@@ -1304,6 +1370,24 @@ function truncate(message) {
|
|
|
1304
1370
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1305
1371
|
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
1306
1372
|
}
|
|
1373
|
+
function sanitiseMetadata(metadata) {
|
|
1374
|
+
if (!metadata) return {};
|
|
1375
|
+
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1376
|
+
if (Object.keys(metadata).length > entries.length) {
|
|
1377
|
+
console.error(
|
|
1378
|
+
`[runner-activity-telemetry] dropped ${Object.keys(metadata).length - entries.length} metadata entries`
|
|
1379
|
+
);
|
|
1380
|
+
}
|
|
1381
|
+
const sanitised = [];
|
|
1382
|
+
for (const [key, value] of entries) {
|
|
1383
|
+
if (typeof value === "string") {
|
|
1384
|
+
sanitised.push([key, truncate(redact(value).slice(0, MAX_METADATA_VALUE_LENGTH))]);
|
|
1385
|
+
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1386
|
+
sanitised.push([key, value]);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
return Object.fromEntries(sanitised);
|
|
1390
|
+
}
|
|
1307
1391
|
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1308
1392
|
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1309
1393
|
var windowStartedAt = 0;
|
|
@@ -1342,7 +1426,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1342
1426
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1343
1427
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1344
1428
|
message,
|
|
1345
|
-
metadata: { source: "cli.run" },
|
|
1429
|
+
metadata: { ...sanitiseMetadata(entry.metadata), source: "cli.run" },
|
|
1346
1430
|
agentId: context.agentId
|
|
1347
1431
|
});
|
|
1348
1432
|
} catch (err) {
|
|
@@ -1352,6 +1436,150 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1352
1436
|
}
|
|
1353
1437
|
}
|
|
1354
1438
|
|
|
1439
|
+
// src/lib/opencode/session-db-recovery-report.ts
|
|
1440
|
+
import { readFileSync as readFileSync2, unlinkSync } from "fs";
|
|
1441
|
+
import { join as join2 } from "path";
|
|
1442
|
+
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1443
|
+
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1444
|
+
return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1445
|
+
}
|
|
1446
|
+
function drainSessionDbRecoveryReport({
|
|
1447
|
+
homeDir,
|
|
1448
|
+
env
|
|
1449
|
+
}) {
|
|
1450
|
+
const path = sessionDbRecoveryReportPath(homeDir, env);
|
|
1451
|
+
let content;
|
|
1452
|
+
try {
|
|
1453
|
+
content = readFileSync2(path, "utf8");
|
|
1454
|
+
} catch (error2) {
|
|
1455
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")
|
|
1456
|
+
return { path, records: [], skippedLines: 0, readError: null };
|
|
1457
|
+
const readError = error2 instanceof Error ? error2.message : String(error2);
|
|
1458
|
+
console.error(`[session-db-recovery-report] could not read ${path}: ${readError}`);
|
|
1459
|
+
return { path, records: [], skippedLines: 0, readError };
|
|
1460
|
+
}
|
|
1461
|
+
let skippedLines = 0;
|
|
1462
|
+
const records = content.split("\n").flatMap((line) => {
|
|
1463
|
+
if (!line.trim()) return [];
|
|
1464
|
+
try {
|
|
1465
|
+
const value = JSON.parse(line);
|
|
1466
|
+
if (!isSessionDbRecoveryRecord(value)) {
|
|
1467
|
+
skippedLines++;
|
|
1468
|
+
return [];
|
|
1469
|
+
}
|
|
1470
|
+
return [value];
|
|
1471
|
+
} catch (error2) {
|
|
1472
|
+
skippedLines++;
|
|
1473
|
+
console.error(
|
|
1474
|
+
`[session-db-recovery-report] skipped malformed record in ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1475
|
+
);
|
|
1476
|
+
return [];
|
|
1477
|
+
}
|
|
1478
|
+
});
|
|
1479
|
+
return { path, records, skippedLines, readError: null };
|
|
1480
|
+
}
|
|
1481
|
+
function acknowledgeSessionDbRecoveryReport(path) {
|
|
1482
|
+
try {
|
|
1483
|
+
unlinkSync(path);
|
|
1484
|
+
} catch (error2) {
|
|
1485
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1486
|
+
console.error(
|
|
1487
|
+
`[session-db-recovery-report] could not remove ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1488
|
+
);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
function buildSessionDbRecoveryActivity(record) {
|
|
1492
|
+
const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
|
|
1493
|
+
if (!level) return null;
|
|
1494
|
+
const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
|
|
1495
|
+
switch (record.outcome) {
|
|
1496
|
+
case "fresh_session_db":
|
|
1497
|
+
return {
|
|
1498
|
+
level,
|
|
1499
|
+
metadata: withoutContractFields(record),
|
|
1500
|
+
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.`
|
|
1501
|
+
};
|
|
1502
|
+
case "restore_retried":
|
|
1503
|
+
return {
|
|
1504
|
+
level,
|
|
1505
|
+
metadata: withoutContractFields(record),
|
|
1506
|
+
message: "The initial session database restore failed; the runner will retry it. Monitor the runner backup for another restore failure."
|
|
1507
|
+
};
|
|
1508
|
+
case "replica_recovered":
|
|
1509
|
+
if (record.reason === "quarantine")
|
|
1510
|
+
return {
|
|
1511
|
+
level,
|
|
1512
|
+
metadata: withoutContractFields(record),
|
|
1513
|
+
message: `Session database recovery quarantined ${record.quarantined_objects ?? "unknown"} objects (${record.quarantined_bytes ?? "unknown"} bytes); ${record.quarantine_failed_objects ?? "unknown"} moves failed. Review the preserved backup at ${record.quarantine_destination ?? "an unknown destination"} before deleting it.`
|
|
1514
|
+
};
|
|
1515
|
+
if (record.reason === "prune")
|
|
1516
|
+
return {
|
|
1517
|
+
level,
|
|
1518
|
+
metadata: withoutContractFields(record),
|
|
1519
|
+
message: "Session database recovery discarded a damaged newest backup and retried. Sessions recorded after the previous backup point may be unavailable. Review the runner backup for another restore failure."
|
|
1520
|
+
};
|
|
1521
|
+
if (record.reason === "clear")
|
|
1522
|
+
return {
|
|
1523
|
+
level,
|
|
1524
|
+
metadata: withoutContractFields(record),
|
|
1525
|
+
message: "Session database recovery deleted the damaged backup and prior session history is unavailable. Review the runner backup configuration before relying on restored session history."
|
|
1526
|
+
};
|
|
1527
|
+
return null;
|
|
1528
|
+
case "history_rolled_back":
|
|
1529
|
+
return {
|
|
1530
|
+
level,
|
|
1531
|
+
metadata: withoutContractFields(record),
|
|
1532
|
+
message: `Session history was rolled back to verified restore point ${record.verified_restore_point ?? "unknown"}; everything after it is unavailable. Review the runner backup for another restore failure.`
|
|
1533
|
+
};
|
|
1534
|
+
case "restore_misconfigured":
|
|
1535
|
+
return {
|
|
1536
|
+
level,
|
|
1537
|
+
metadata: withoutContractFields(record),
|
|
1538
|
+
message: "This runner started with a fresh session database because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions."
|
|
1539
|
+
};
|
|
1540
|
+
case "session_db_boot_refused":
|
|
1541
|
+
return {
|
|
1542
|
+
level,
|
|
1543
|
+
metadata: withoutContractFields(record),
|
|
1544
|
+
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1545
|
+
};
|
|
1546
|
+
default:
|
|
1547
|
+
return null;
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
function withoutContractFields(record) {
|
|
1551
|
+
const { v: _v, event: _event, ...metadata } = record;
|
|
1552
|
+
return metadata;
|
|
1553
|
+
}
|
|
1554
|
+
var OUTCOMES = /* @__PURE__ */ new Set([
|
|
1555
|
+
"replica_recovered",
|
|
1556
|
+
"restore_retried",
|
|
1557
|
+
"fresh_session_db",
|
|
1558
|
+
"history_rolled_back",
|
|
1559
|
+
"restore_misconfigured",
|
|
1560
|
+
"session_db_boot_refused"
|
|
1561
|
+
]);
|
|
1562
|
+
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1563
|
+
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
1564
|
+
var NUMBER_FIELDS = [
|
|
1565
|
+
"litestream_exit_code",
|
|
1566
|
+
"attempt",
|
|
1567
|
+
"replica_objects",
|
|
1568
|
+
"replica_bytes",
|
|
1569
|
+
"quarantined_objects",
|
|
1570
|
+
"quarantine_failed_objects",
|
|
1571
|
+
"quarantined_bytes",
|
|
1572
|
+
"restore_points_tried"
|
|
1573
|
+
];
|
|
1574
|
+
var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"];
|
|
1575
|
+
function isSessionDbRecoveryRecord(value) {
|
|
1576
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1577
|
+
const record = value;
|
|
1578
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1579
|
+
(field) => record[field] === null || typeof record[field] === "string"
|
|
1580
|
+
);
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1355
1583
|
// src/lib/opencode/health.ts
|
|
1356
1584
|
async function checkOpenCodeHealth(port) {
|
|
1357
1585
|
try {
|
|
@@ -1700,13 +1928,22 @@ function buildNoProviderWarning(hasProvider) {
|
|
|
1700
1928
|
return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
|
|
1701
1929
|
}
|
|
1702
1930
|
|
|
1931
|
+
// src/lib/http-timeout.ts
|
|
1932
|
+
var REQUEST_TIMEOUT_MS = 6e4;
|
|
1933
|
+
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1934
|
+
return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1703
1937
|
// src/lib/opencode/session.ts
|
|
1938
|
+
function timedFetch(input, init) {
|
|
1939
|
+
return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
|
|
1940
|
+
}
|
|
1704
1941
|
function opencodeBase(port) {
|
|
1705
1942
|
return `http://127.0.0.1:${port}`;
|
|
1706
1943
|
}
|
|
1707
1944
|
async function getOpenCodeDirectory(port) {
|
|
1708
1945
|
try {
|
|
1709
|
-
const res = await
|
|
1946
|
+
const res = await timedFetch(`${opencodeBase(port)}/path`);
|
|
1710
1947
|
if (!res.ok) return null;
|
|
1711
1948
|
const body = await res.json();
|
|
1712
1949
|
const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
|
|
@@ -1757,7 +1994,7 @@ function isAssistantInFlight(m) {
|
|
|
1757
1994
|
}
|
|
1758
1995
|
async function getSessionMessages(port, sessionId) {
|
|
1759
1996
|
try {
|
|
1760
|
-
const res = await
|
|
1997
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1761
1998
|
if (!res.ok) return null;
|
|
1762
1999
|
const body = await res.json();
|
|
1763
2000
|
return Array.isArray(body) ? body : null;
|
|
@@ -1787,7 +2024,7 @@ function sessionLastActivityMs(session) {
|
|
|
1787
2024
|
}
|
|
1788
2025
|
async function listSessions(port) {
|
|
1789
2026
|
try {
|
|
1790
|
-
const res = await
|
|
2027
|
+
const res = await timedFetch(`${opencodeBase(port)}/session`);
|
|
1791
2028
|
if (!res.ok) return null;
|
|
1792
2029
|
const body = await res.json();
|
|
1793
2030
|
return Array.isArray(body) ? body : null;
|
|
@@ -1797,7 +2034,7 @@ async function listSessions(port) {
|
|
|
1797
2034
|
}
|
|
1798
2035
|
async function deleteSession(port, id) {
|
|
1799
2036
|
try {
|
|
1800
|
-
const res = await
|
|
2037
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1801
2038
|
return res.status >= 200 && res.status < 300;
|
|
1802
2039
|
} catch {
|
|
1803
2040
|
return false;
|
|
@@ -1805,7 +2042,7 @@ async function deleteSession(port, id) {
|
|
|
1805
2042
|
}
|
|
1806
2043
|
async function sessionExists(port, id) {
|
|
1807
2044
|
try {
|
|
1808
|
-
const res = await
|
|
2045
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
|
|
1809
2046
|
if (res.status >= 200 && res.status < 300) return true;
|
|
1810
2047
|
if (res.status === 404) return false;
|
|
1811
2048
|
return null;
|
|
@@ -1815,7 +2052,7 @@ async function sessionExists(port, id) {
|
|
|
1815
2052
|
}
|
|
1816
2053
|
async function getSessionStatuses(port) {
|
|
1817
2054
|
try {
|
|
1818
|
-
const res = await
|
|
2055
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/status`);
|
|
1819
2056
|
if (!res.ok) {
|
|
1820
2057
|
console.error(
|
|
1821
2058
|
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
@@ -1848,7 +2085,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1848
2085
|
if (directory && directory.trim()) {
|
|
1849
2086
|
url.searchParams.set("directory", directory.trim());
|
|
1850
2087
|
}
|
|
1851
|
-
const response = await
|
|
2088
|
+
const response = await timedFetch(url, {
|
|
1852
2089
|
method: "POST",
|
|
1853
2090
|
headers: { "Content-Type": "application/json" },
|
|
1854
2091
|
body: JSON.stringify({})
|
|
@@ -1862,7 +2099,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1862
2099
|
}
|
|
1863
2100
|
async function getModelAttachmentCapability(port, model) {
|
|
1864
2101
|
try {
|
|
1865
|
-
const res = await
|
|
2102
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1866
2103
|
if (!res.ok) {
|
|
1867
2104
|
console.error(
|
|
1868
2105
|
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -1995,7 +2232,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1995
2232
|
};
|
|
1996
2233
|
}
|
|
1997
2234
|
}
|
|
1998
|
-
const res = await
|
|
2235
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1999
2236
|
method: "POST",
|
|
2000
2237
|
headers: { "Content-Type": "application/json" },
|
|
2001
2238
|
body: JSON.stringify(body)
|
|
@@ -2244,7 +2481,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
2244
2481
|
}
|
|
2245
2482
|
async function hasAnyConfiguredProvider(port) {
|
|
2246
2483
|
try {
|
|
2247
|
-
const res = await
|
|
2484
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2248
2485
|
if (!res.ok) {
|
|
2249
2486
|
console.error(
|
|
2250
2487
|
`[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -2380,10 +2617,10 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2380
2617
|
|
|
2381
2618
|
// src/lib/opencode/session-db-size.ts
|
|
2382
2619
|
import { statSync as statSync2 } from "fs";
|
|
2383
|
-
import { join as
|
|
2620
|
+
import { join as join3 } from "path";
|
|
2384
2621
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2385
2622
|
function statSessionDbBytes(homeDir) {
|
|
2386
|
-
const dbPath =
|
|
2623
|
+
const dbPath = join3(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2387
2624
|
try {
|
|
2388
2625
|
return statSync2(dbPath).size;
|
|
2389
2626
|
} catch (err) {
|
|
@@ -2952,44 +3189,405 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
2952
3189
|
}
|
|
2953
3190
|
}
|
|
2954
3191
|
|
|
2955
|
-
// src/lib/
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
3192
|
+
// src/lib/openai-usage.ts
|
|
3193
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
3194
|
+
import { homedir as homedir2 } from "os";
|
|
3195
|
+
import { join as join4 } from "path";
|
|
3196
|
+
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3197
|
+
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3198
|
+
var OpenAiUsageError = class extends Error {
|
|
3199
|
+
constructor(message, reason) {
|
|
3200
|
+
super(message);
|
|
3201
|
+
this.reason = reason;
|
|
3202
|
+
}
|
|
3203
|
+
};
|
|
3204
|
+
function isLocalCredentialProblem2(err) {
|
|
3205
|
+
return err instanceof OpenAiUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
|
|
3206
|
+
}
|
|
3207
|
+
function readOpenCodeChatGptCredentials() {
|
|
3208
|
+
try {
|
|
3209
|
+
const raw = readFileSync3(join4(homedir2(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3210
|
+
let parsed;
|
|
3211
|
+
try {
|
|
3212
|
+
parsed = JSON.parse(raw);
|
|
3213
|
+
} catch {
|
|
3214
|
+
return null;
|
|
3215
|
+
}
|
|
3216
|
+
const entry = parsed.openai;
|
|
3217
|
+
if (entry?.type !== "oauth" || typeof entry.access !== "string" || !entry.access || typeof entry.expires !== "number") {
|
|
3218
|
+
return null;
|
|
3219
|
+
}
|
|
3220
|
+
return { accessToken: entry.access, expiresAt: entry.expires };
|
|
3221
|
+
} catch (err) {
|
|
3222
|
+
const code = err.code;
|
|
3223
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
3224
|
+
console.warn(
|
|
3225
|
+
`readOpenCodeChatGptCredentials: reading auth.json failed (${code ?? "unknown"})`
|
|
3226
|
+
);
|
|
3227
|
+
}
|
|
3228
|
+
return null;
|
|
3229
|
+
}
|
|
3230
|
+
}
|
|
3231
|
+
function toWindow2(headers, name) {
|
|
3232
|
+
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
3233
|
+
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
3234
|
+
if (utilizationHeader == null || utilizationHeader === "" || windowMinutesHeader == null) {
|
|
3235
|
+
return null;
|
|
3236
|
+
}
|
|
3237
|
+
const utilization = Number(utilizationHeader);
|
|
3238
|
+
const windowMinutes = Number(windowMinutesHeader);
|
|
3239
|
+
if (!Number.isFinite(utilization) || !Number.isFinite(windowMinutes) || windowMinutes <= 0) {
|
|
3240
|
+
return null;
|
|
3241
|
+
}
|
|
3242
|
+
const resetAtHeader = headers.get(`x-codex-${name}-reset-at`);
|
|
3243
|
+
const resetSeconds = resetAtHeader == null || resetAtHeader === "" ? NaN : Number(resetAtHeader);
|
|
3244
|
+
const resetsAt = Number.isFinite(resetSeconds) ? new Date(resetSeconds * 1e3).toISOString() : null;
|
|
3245
|
+
return { utilization: Math.min(100, Math.max(0, utilization)), windowMinutes, resetsAt };
|
|
3246
|
+
}
|
|
3247
|
+
function parseCodexUsageHeaders(headers) {
|
|
3248
|
+
return {
|
|
3249
|
+
primary: toWindow2(headers, "primary"),
|
|
3250
|
+
secondary: toWindow2(headers, "secondary"),
|
|
3251
|
+
hasCredits: headers.has("x-codex-credits-has-credits") ? headers.get("x-codex-credits-has-credits")?.toLowerCase() === "true" : null,
|
|
3252
|
+
creditsUnlimited: headers.has("x-codex-credits-unlimited") ? headers.get("x-codex-credits-unlimited")?.toLowerCase() === "true" : null
|
|
3253
|
+
};
|
|
3254
|
+
}
|
|
3255
|
+
function normalizeProbeModel(model) {
|
|
3256
|
+
return model.endsWith("-fast") ? model.slice(0, -"-fast".length) : model;
|
|
3257
|
+
}
|
|
3258
|
+
async function resolveProbeModels(port) {
|
|
3259
|
+
try {
|
|
3260
|
+
const res = await withRequestTimeout(
|
|
3261
|
+
fetch,
|
|
3262
|
+
REQUEST_TIMEOUT_MS
|
|
3263
|
+
)(`${opencodeBase(port)}/config/providers`);
|
|
3264
|
+
if (!res.ok) {
|
|
3265
|
+
console.error(
|
|
3266
|
+
`[resolveProbeModels] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
3267
|
+
);
|
|
3268
|
+
return [];
|
|
3269
|
+
}
|
|
3270
|
+
const body = await res.json();
|
|
3271
|
+
const provider = body?.providers?.find((candidate) => candidate?.id === "openai");
|
|
3272
|
+
if (!provider || !provider.models || typeof provider.models !== "object") return [];
|
|
3273
|
+
const candidates = [
|
|
3274
|
+
...typeof body?.default?.openai === "string" ? [body.default.openai] : [],
|
|
3275
|
+
...Object.keys(provider.models)
|
|
3276
|
+
].map(normalizeProbeModel);
|
|
3277
|
+
return [...new Set(candidates)].slice(0, 4);
|
|
3278
|
+
} catch (err) {
|
|
3279
|
+
console.error(
|
|
3280
|
+
`[resolveProbeModels] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
3281
|
+
);
|
|
3282
|
+
return [];
|
|
3283
|
+
}
|
|
3284
|
+
}
|
|
3285
|
+
function hasPrimaryHeaders(headers) {
|
|
3286
|
+
return [
|
|
3287
|
+
"x-codex-primary-used-percent",
|
|
3288
|
+
"x-codex-primary-window-minutes",
|
|
3289
|
+
"x-codex-primary-reset-at"
|
|
3290
|
+
].some((name) => headers.has(name));
|
|
3291
|
+
}
|
|
3292
|
+
async function getOpenAiUsage(port) {
|
|
3293
|
+
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3294
|
+
if (!credentials2) {
|
|
3295
|
+
throw new OpenAiUsageError(
|
|
3296
|
+
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3297
|
+
"no_credentials"
|
|
3298
|
+
);
|
|
3299
|
+
}
|
|
3300
|
+
if (credentials2.expiresAt < Date.now()) {
|
|
3301
|
+
throw new OpenAiUsageError(
|
|
3302
|
+
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3303
|
+
"credentials_expired"
|
|
3304
|
+
);
|
|
3305
|
+
}
|
|
3306
|
+
const models = await resolveProbeModels(port);
|
|
3307
|
+
if (models.length === 0) {
|
|
3308
|
+
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
2961
3309
|
}
|
|
3310
|
+
let lastStatus;
|
|
3311
|
+
for (const model of models) {
|
|
3312
|
+
let res;
|
|
3313
|
+
try {
|
|
3314
|
+
res = await withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(CODEX_RESPONSES_URL, {
|
|
3315
|
+
method: "POST",
|
|
3316
|
+
headers: {
|
|
3317
|
+
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
3318
|
+
"Content-Type": "application/json"
|
|
3319
|
+
},
|
|
3320
|
+
body: JSON.stringify({ model, store: false, stream: true })
|
|
3321
|
+
});
|
|
3322
|
+
} catch (err) {
|
|
3323
|
+
throw new OpenAiUsageError(
|
|
3324
|
+
`OpenAI usage probe request failed: ${err instanceof Error ? err.message : String(err)}.`,
|
|
3325
|
+
"request_failed"
|
|
3326
|
+
);
|
|
3327
|
+
}
|
|
3328
|
+
try {
|
|
3329
|
+
lastStatus = res.status;
|
|
3330
|
+
if (hasPrimaryHeaders(res.headers)) {
|
|
3331
|
+
const usage = parseCodexUsageHeaders(res.headers);
|
|
3332
|
+
if (!usage.primary && !usage.secondary) {
|
|
3333
|
+
throw new OpenAiUsageError(
|
|
3334
|
+
`OpenAI usage probe returned no usable window (HTTP ${res.status}).`,
|
|
3335
|
+
"no_usable_window"
|
|
3336
|
+
);
|
|
3337
|
+
}
|
|
3338
|
+
return usage;
|
|
3339
|
+
}
|
|
3340
|
+
if (res.status === 401) {
|
|
3341
|
+
throw new OpenAiUsageError(
|
|
3342
|
+
"ChatGPT credentials have expired (HTTP 401).",
|
|
3343
|
+
"credentials_expired"
|
|
3344
|
+
);
|
|
3345
|
+
}
|
|
3346
|
+
if (res.status === 403 || res.status === 429) {
|
|
3347
|
+
throw new OpenAiUsageError(
|
|
3348
|
+
`OpenAI usage probe was blocked (HTTP ${res.status}).`,
|
|
3349
|
+
"probe_blocked"
|
|
3350
|
+
);
|
|
3351
|
+
}
|
|
3352
|
+
} finally {
|
|
3353
|
+
await res.body?.cancel().catch(() => {
|
|
3354
|
+
});
|
|
3355
|
+
}
|
|
3356
|
+
}
|
|
3357
|
+
throw new OpenAiUsageError(
|
|
3358
|
+
`OpenAI usage probe failed: HTTP ${lastStatus ?? "unknown"}.`,
|
|
3359
|
+
"request_failed"
|
|
3360
|
+
);
|
|
3361
|
+
}
|
|
3362
|
+
|
|
3363
|
+
// src/lib/reporting-schedule.ts
|
|
3364
|
+
function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
3365
|
+
const jitterRangeMs = baseMs * jitterFraction;
|
|
3366
|
+
return baseMs - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
3367
|
+
}
|
|
3368
|
+
function firstReportDelayMs(random = Math.random) {
|
|
3369
|
+
return 5e3 + random() * 1e4;
|
|
3370
|
+
}
|
|
3371
|
+
var VALID_USAGE_REPORTING_MODES = ["auto", "on", "off"];
|
|
3372
|
+
function resolveUsageReportingMode(flagValue, env, names) {
|
|
3373
|
+
const raw = flagValue ?? env[names.envVar];
|
|
3374
|
+
if (raw === void 0 || raw === "") return { mode: "auto", warnings: [] };
|
|
2962
3375
|
const normalized = raw.trim().toLowerCase();
|
|
2963
|
-
if (
|
|
3376
|
+
if (VALID_USAGE_REPORTING_MODES.includes(normalized)) {
|
|
2964
3377
|
return { mode: normalized, warnings: [] };
|
|
2965
3378
|
}
|
|
2966
|
-
const source = flagValue !== void 0 ?
|
|
3379
|
+
const source = flagValue !== void 0 ? names.flagName : names.envVar;
|
|
2967
3380
|
return {
|
|
2968
3381
|
mode: "auto",
|
|
2969
3382
|
warnings: [
|
|
2970
|
-
`Ignoring invalid ${source} "${raw}": expected one of ${
|
|
3383
|
+
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_USAGE_REPORTING_MODES.join(", ")}; using auto`
|
|
2971
3384
|
]
|
|
2972
3385
|
};
|
|
2973
3386
|
}
|
|
2974
|
-
var
|
|
2975
|
-
var
|
|
3387
|
+
var BASE_USAGE_REPORT_DELAY_MS = 10 * 6e4;
|
|
3388
|
+
var USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
3389
|
+
function usageReportDelayMs(random = Math.random) {
|
|
3390
|
+
return jitteredDelayMs(BASE_USAGE_REPORT_DELAY_MS, USAGE_REPORT_DELAY_JITTER_FRACTION, random);
|
|
3391
|
+
}
|
|
3392
|
+
var USAGE_REPORT_FAILURE_REESCALATION_TICKS = 6;
|
|
3393
|
+
function usageReportFailureLogLevel(consecutiveFailures) {
|
|
3394
|
+
return reportFailureLogLevel(consecutiveFailures, USAGE_REPORT_FAILURE_REESCALATION_TICKS);
|
|
3395
|
+
}
|
|
3396
|
+
function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
|
|
3397
|
+
return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
|
|
3398
|
+
}
|
|
3399
|
+
function failureStreakSuffix(consecutiveFailures) {
|
|
3400
|
+
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
3401
|
+
}
|
|
3402
|
+
|
|
3403
|
+
// src/lib/claude-usage-reporting.ts
|
|
3404
|
+
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
3405
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
3406
|
+
flagName: "--claude-usage-reporting",
|
|
3407
|
+
envVar: "EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
3408
|
+
});
|
|
3409
|
+
}
|
|
2976
3410
|
function nextReportDelayMs(random = Math.random) {
|
|
2977
|
-
|
|
2978
|
-
return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
3411
|
+
return usageReportDelayMs(random);
|
|
2979
3412
|
}
|
|
2980
|
-
var FIRST_REPORT_DELAY_MS =
|
|
2981
|
-
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
3413
|
+
var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
|
|
2982
3414
|
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
2983
|
-
return consecutiveFailures
|
|
3415
|
+
return usageReportFailureLogLevel(consecutiveFailures);
|
|
3416
|
+
}
|
|
3417
|
+
|
|
3418
|
+
// src/lib/openai-usage-reporting.ts
|
|
3419
|
+
function resolveOpenAiUsageReportingMode(flagValue, env) {
|
|
3420
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
3421
|
+
flagName: "--openai-usage-reporting",
|
|
3422
|
+
envVar: "EVIDENT_OPENAI_USAGE_REPORTING"
|
|
3423
|
+
});
|
|
3424
|
+
}
|
|
3425
|
+
|
|
3426
|
+
// src/lib/resource-usage-reporting.ts
|
|
3427
|
+
var ENABLED_VALUES = /* @__PURE__ */ new Set(["on", "true", "1"]);
|
|
3428
|
+
var DISABLED_VALUES = /* @__PURE__ */ new Set(["off", "false", "0"]);
|
|
3429
|
+
function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
3430
|
+
if (flagValue === false) {
|
|
3431
|
+
return { enabled: false, warnings: [] };
|
|
3432
|
+
}
|
|
3433
|
+
const raw = env.EVIDENT_RESOURCE_USAGE_REPORTING;
|
|
3434
|
+
if (raw === void 0 || raw === "") {
|
|
3435
|
+
return { enabled: true, warnings: [] };
|
|
3436
|
+
}
|
|
3437
|
+
const normalized = raw.trim().toLowerCase();
|
|
3438
|
+
if (DISABLED_VALUES.has(normalized)) {
|
|
3439
|
+
return { enabled: false, warnings: [] };
|
|
3440
|
+
}
|
|
3441
|
+
if (ENABLED_VALUES.has(normalized)) {
|
|
3442
|
+
return { enabled: true, warnings: [] };
|
|
3443
|
+
}
|
|
3444
|
+
return {
|
|
3445
|
+
enabled: true,
|
|
3446
|
+
warnings: [
|
|
3447
|
+
`Ignoring invalid EVIDENT_RESOURCE_USAGE_REPORTING "${raw}": expected on or off; leaving reporting on`
|
|
3448
|
+
]
|
|
3449
|
+
};
|
|
3450
|
+
}
|
|
3451
|
+
|
|
3452
|
+
// src/lib/resource-usage.ts
|
|
3453
|
+
import { cpus, totalmem, freemem } from "os";
|
|
3454
|
+
import { statfsSync as statfsSync2 } from "fs";
|
|
3455
|
+
|
|
3456
|
+
// src/lib/ecs-task-metadata.ts
|
|
3457
|
+
var ECS_METADATA_TIMEOUT_MS = 2e3;
|
|
3458
|
+
function parseEcsTaskLimits(payload) {
|
|
3459
|
+
if (typeof payload !== "object" || payload === null) return null;
|
|
3460
|
+
const limits = payload.Limits;
|
|
3461
|
+
if (typeof limits !== "object" || limits === null) return null;
|
|
3462
|
+
const cpu = limits.CPU;
|
|
3463
|
+
const memory = limits.Memory;
|
|
3464
|
+
if (typeof cpu !== "number" || !Number.isFinite(cpu) || cpu <= 0) return null;
|
|
3465
|
+
if (typeof memory !== "number" || !Number.isFinite(memory) || memory <= 0) return null;
|
|
3466
|
+
return {
|
|
3467
|
+
cpuCount: Math.max(1, Math.round(cpu)),
|
|
3468
|
+
memoryTotalBytes: memory * 1024 * 1024
|
|
3469
|
+
};
|
|
3470
|
+
}
|
|
3471
|
+
async function readEcsTaskLimits(env) {
|
|
3472
|
+
const uri = env.ECS_CONTAINER_METADATA_URI_V4;
|
|
3473
|
+
if (!uri) {
|
|
3474
|
+
return { limits: null };
|
|
3475
|
+
}
|
|
3476
|
+
const url = `${uri}/task`;
|
|
3477
|
+
try {
|
|
3478
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(ECS_METADATA_TIMEOUT_MS) });
|
|
3479
|
+
if (!response.ok) {
|
|
3480
|
+
return {
|
|
3481
|
+
limits: null,
|
|
3482
|
+
warning: `ECS task metadata fetch (${url}) returned HTTP ${response.status}`
|
|
3483
|
+
};
|
|
3484
|
+
}
|
|
3485
|
+
const payload = await response.json();
|
|
3486
|
+
const limits = parseEcsTaskLimits(payload);
|
|
3487
|
+
if (limits === null) {
|
|
3488
|
+
return {
|
|
3489
|
+
limits: null,
|
|
3490
|
+
warning: `ECS task metadata fetch (${url}) returned an unexpected payload`
|
|
3491
|
+
};
|
|
3492
|
+
}
|
|
3493
|
+
return { limits };
|
|
3494
|
+
} catch (error2) {
|
|
3495
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3496
|
+
return { limits: null, warning: `ECS task metadata fetch (${url}) failed: ${message}` };
|
|
3497
|
+
}
|
|
3498
|
+
}
|
|
3499
|
+
|
|
3500
|
+
// src/lib/resource-usage.ts
|
|
3501
|
+
function readCpuSample() {
|
|
3502
|
+
let busyMs = 0;
|
|
3503
|
+
let idleMs = 0;
|
|
3504
|
+
for (const cpu of cpus()) {
|
|
3505
|
+
busyMs += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.irq;
|
|
3506
|
+
idleMs += cpu.times.idle;
|
|
3507
|
+
}
|
|
3508
|
+
return { busyMs, idleMs };
|
|
3509
|
+
}
|
|
3510
|
+
function cpuPercentBetween(previous, current) {
|
|
3511
|
+
const deltaBusy = current.busyMs - previous.busyMs;
|
|
3512
|
+
const deltaIdle = current.idleMs - previous.idleMs;
|
|
3513
|
+
const total = deltaBusy + deltaIdle;
|
|
3514
|
+
if (total === 0) return null;
|
|
3515
|
+
return Math.round((deltaBusy / total * 100 + Number.EPSILON) * 100) / 100;
|
|
3516
|
+
}
|
|
3517
|
+
function clamp(value, min, max) {
|
|
3518
|
+
return Math.min(Math.max(value, min), max);
|
|
3519
|
+
}
|
|
3520
|
+
function round2(value) {
|
|
3521
|
+
return Math.round((value + Number.EPSILON) * 100) / 100;
|
|
3522
|
+
}
|
|
3523
|
+
function readDisk(homeDir) {
|
|
3524
|
+
try {
|
|
3525
|
+
const stats = statfsSync2(homeDir);
|
|
3526
|
+
return {
|
|
3527
|
+
totalBytes: stats.bsize * stats.blocks,
|
|
3528
|
+
freeBytes: stats.bsize * stats.bavail
|
|
3529
|
+
};
|
|
3530
|
+
} catch (error2) {
|
|
3531
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3532
|
+
return {
|
|
3533
|
+
totalBytes: null,
|
|
3534
|
+
freeBytes: null,
|
|
3535
|
+
warning: `Could not read disk usage for ${homeDir}: ${message}`
|
|
3536
|
+
};
|
|
3537
|
+
}
|
|
3538
|
+
}
|
|
3539
|
+
function createResourceUsageCollector(homeDir) {
|
|
3540
|
+
let previous = readCpuSample();
|
|
3541
|
+
return async () => {
|
|
3542
|
+
const current = readCpuSample();
|
|
3543
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
3544
|
+
const hostCpuCount = cpus().length;
|
|
3545
|
+
previous = current;
|
|
3546
|
+
const disk = readDisk(homeDir);
|
|
3547
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
3548
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
3549
|
+
const warnings = [];
|
|
3550
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
3551
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
3552
|
+
let cpuPercent = hostCpuPercent;
|
|
3553
|
+
let cpuCount = hostCpuCount;
|
|
3554
|
+
let memoryTotalBytes = totalmem();
|
|
3555
|
+
let memoryAvailableBytes = freemem();
|
|
3556
|
+
if (limits !== null) {
|
|
3557
|
+
cpuCount = limits.cpuCount;
|
|
3558
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3559
|
+
memoryAvailableBytes = clamp(
|
|
3560
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3561
|
+
0,
|
|
3562
|
+
limits.memoryTotalBytes
|
|
3563
|
+
);
|
|
3564
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
3565
|
+
}
|
|
3566
|
+
return {
|
|
3567
|
+
usage: {
|
|
3568
|
+
cpuPercent,
|
|
3569
|
+
cpuCount,
|
|
3570
|
+
memoryTotalBytes,
|
|
3571
|
+
memoryAvailableBytes,
|
|
3572
|
+
diskTotalBytes: disk.totalBytes,
|
|
3573
|
+
diskFreeBytes: disk.freeBytes,
|
|
3574
|
+
opencodeDbBytes
|
|
3575
|
+
},
|
|
3576
|
+
warnings
|
|
3577
|
+
};
|
|
3578
|
+
};
|
|
2984
3579
|
}
|
|
2985
3580
|
|
|
2986
3581
|
// src/lib/channels/driver.ts
|
|
2987
|
-
import { homedir as
|
|
3582
|
+
import { homedir as homedir3 } from "os";
|
|
3583
|
+
|
|
3584
|
+
// src/lib/runner-file-sync.ts
|
|
3585
|
+
import { join as join6 } from "path";
|
|
2988
3586
|
|
|
2989
3587
|
// src/lib/file-push.ts
|
|
2990
3588
|
import { randomUUID } from "crypto";
|
|
2991
3589
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2992
|
-
import { basename, dirname as dirname3, isAbsolute, join as
|
|
3590
|
+
import { basename, dirname as dirname3, isAbsolute, join as join5, relative, resolve as resolve2, sep } from "path";
|
|
2993
3591
|
var FILE_MODE = 384;
|
|
2994
3592
|
var DIRECTORY_MODE = 448;
|
|
2995
3593
|
async function writePushedFile(request) {
|
|
@@ -3022,7 +3620,7 @@ async function writePushedFile(request) {
|
|
|
3022
3620
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3023
3621
|
dirname3(candidate)
|
|
3024
3622
|
);
|
|
3025
|
-
const realTarget =
|
|
3623
|
+
const realTarget = join5(existingAncestor, ...missingSegments, basename(candidate));
|
|
3026
3624
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3027
3625
|
if (allowedDirectory === null) {
|
|
3028
3626
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3058,7 +3656,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3058
3656
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3059
3657
|
return null;
|
|
3060
3658
|
}
|
|
3061
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3659
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join5(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3062
3660
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3063
3661
|
return null;
|
|
3064
3662
|
}
|
|
@@ -3131,13 +3729,13 @@ function contains(realDirectory, realTarget) {
|
|
|
3131
3729
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3132
3730
|
let current = existingAncestor;
|
|
3133
3731
|
for (const segment of missingSegments) {
|
|
3134
|
-
current =
|
|
3732
|
+
current = join5(current, segment);
|
|
3135
3733
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3136
3734
|
await chmod(current, DIRECTORY_MODE);
|
|
3137
3735
|
}
|
|
3138
3736
|
}
|
|
3139
3737
|
async function writeAtomically(realTarget, content) {
|
|
3140
|
-
const temporaryPath =
|
|
3738
|
+
const temporaryPath = join5(dirname3(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3141
3739
|
let handle;
|
|
3142
3740
|
try {
|
|
3143
3741
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -3179,17 +3777,28 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3179
3777
|
for (const id of options.ackFailures.keys()) {
|
|
3180
3778
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3181
3779
|
}
|
|
3182
|
-
if (pending.length === 0)
|
|
3780
|
+
if (pending.length === 0) {
|
|
3781
|
+
return { applied: 0, claudeCredentialApplied: false, opencodeAuthApplied: false };
|
|
3782
|
+
}
|
|
3183
3783
|
options.log({
|
|
3184
3784
|
level: "info",
|
|
3185
3785
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3186
3786
|
});
|
|
3187
3787
|
let applied = 0;
|
|
3788
|
+
let claudeCredentialApplied = false;
|
|
3789
|
+
let opencodeAuthApplied = false;
|
|
3188
3790
|
for (const file of pending) {
|
|
3189
3791
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3190
|
-
|
|
3792
|
+
const outcome = await applyOne(options, file);
|
|
3793
|
+
if (outcome.applied) applied += 1;
|
|
3794
|
+
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
3795
|
+
if (outcome.opencodeAuthApplied) opencodeAuthApplied = true;
|
|
3191
3796
|
}
|
|
3192
|
-
return
|
|
3797
|
+
return {
|
|
3798
|
+
applied,
|
|
3799
|
+
claudeCredentialApplied,
|
|
3800
|
+
opencodeAuthApplied
|
|
3801
|
+
};
|
|
3193
3802
|
}
|
|
3194
3803
|
async function listPendingFiles(options) {
|
|
3195
3804
|
let res;
|
|
@@ -3250,6 +3859,19 @@ function asPendingFile(entry) {
|
|
|
3250
3859
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3251
3860
|
return { id, path, size };
|
|
3252
3861
|
}
|
|
3862
|
+
var NOT_APPLIED = {
|
|
3863
|
+
applied: false,
|
|
3864
|
+
claudeCredentialApplied: false,
|
|
3865
|
+
opencodeAuthApplied: false
|
|
3866
|
+
};
|
|
3867
|
+
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3868
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3869
|
+
return expanded === join6(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3870
|
+
}
|
|
3871
|
+
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
3872
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3873
|
+
return expanded === join6(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
3874
|
+
}
|
|
3253
3875
|
async function applyOne(options, file) {
|
|
3254
3876
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
3255
3877
|
if (options.allowedDirectories.length === 0) {
|
|
@@ -3258,7 +3880,7 @@ async function applyOne(options, file) {
|
|
|
3258
3880
|
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
3259
3881
|
});
|
|
3260
3882
|
await ack(options, file, "rejected", "file_sync_disabled");
|
|
3261
|
-
return
|
|
3883
|
+
return NOT_APPLIED;
|
|
3262
3884
|
}
|
|
3263
3885
|
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
3264
3886
|
options.log({
|
|
@@ -3266,12 +3888,12 @@ async function applyOne(options, file) {
|
|
|
3266
3888
|
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
3267
3889
|
});
|
|
3268
3890
|
await ack(options, file, "rejected", "file_too_large");
|
|
3269
|
-
return
|
|
3891
|
+
return NOT_APPLIED;
|
|
3270
3892
|
}
|
|
3271
3893
|
const download = await downloadContent(options, file, label);
|
|
3272
3894
|
if (!download.ok) {
|
|
3273
3895
|
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
3274
|
-
return
|
|
3896
|
+
return NOT_APPLIED;
|
|
3275
3897
|
}
|
|
3276
3898
|
let outcome;
|
|
3277
3899
|
try {
|
|
@@ -3287,7 +3909,7 @@ async function applyOne(options, file) {
|
|
|
3287
3909
|
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
3288
3910
|
});
|
|
3289
3911
|
await ack(options, file, "rejected", "write_failed");
|
|
3290
|
-
return
|
|
3912
|
+
return NOT_APPLIED;
|
|
3291
3913
|
}
|
|
3292
3914
|
if (!outcome.ok) {
|
|
3293
3915
|
options.log({
|
|
@@ -3295,14 +3917,18 @@ async function applyOne(options, file) {
|
|
|
3295
3917
|
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
3296
3918
|
});
|
|
3297
3919
|
await ack(options, file, "rejected", outcome.code);
|
|
3298
|
-
return
|
|
3920
|
+
return NOT_APPLIED;
|
|
3299
3921
|
}
|
|
3300
3922
|
options.log({
|
|
3301
3923
|
level: "info",
|
|
3302
3924
|
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
3303
3925
|
});
|
|
3304
3926
|
await ack(options, file, "applied");
|
|
3305
|
-
return
|
|
3927
|
+
return {
|
|
3928
|
+
applied: true,
|
|
3929
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir),
|
|
3930
|
+
opencodeAuthApplied: isOpenCodeAuthPath(file.path, options.homeDir)
|
|
3931
|
+
};
|
|
3306
3932
|
}
|
|
3307
3933
|
function durableDownloadCode(status2) {
|
|
3308
3934
|
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
@@ -3413,8 +4039,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
|
3413
4039
|
var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
|
|
3414
4040
|
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
3415
4041
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
4042
|
+
var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
|
|
4043
|
+
var MAX_WATCHER_STALL_RESTARTS = 3;
|
|
4044
|
+
var MAX_RELEASED_OPENCODE_IDS = 256;
|
|
3416
4045
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3417
4046
|
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
4047
|
+
var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
|
|
4048
|
+
var MAX_WEDGED_CONVERSATIONS = 256;
|
|
3418
4049
|
var ChannelAuthError = class extends Error {
|
|
3419
4050
|
constructor(message) {
|
|
3420
4051
|
super(message);
|
|
@@ -3458,6 +4089,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3458
4089
|
fileSyncDirectories;
|
|
3459
4090
|
homeDir;
|
|
3460
4091
|
maxActiveSessions;
|
|
4092
|
+
watcherStallMs;
|
|
4093
|
+
wedgeWarningIntervalMs;
|
|
3461
4094
|
/** Cache of conversationId → opencode sessionId. */
|
|
3462
4095
|
sessions = /* @__PURE__ */ new Map();
|
|
3463
4096
|
/**
|
|
@@ -3488,6 +4121,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3488
4121
|
* bounded cost.
|
|
3489
4122
|
*/
|
|
3490
4123
|
supersededSessions = /* @__PURE__ */ new Map();
|
|
4124
|
+
/**
|
|
4125
|
+
* Local re-drive fence for a message force-released by the stall watchdog
|
|
4126
|
+
* (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
|
|
4127
|
+
* see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
|
|
4128
|
+
* is `null` for exactly this shape (its `markProcessing` never landed), so
|
|
4129
|
+
* without a local record of the id the driver last knew, the next drain's
|
|
4130
|
+
* `if (message.opencode_message_id)` re-drive-fence check at
|
|
4131
|
+
* `processConversation` would not engage and it would blind-`prompt_async`
|
|
4132
|
+
* a turn that may still be running in opencode — the one duplicate-turn
|
|
4133
|
+
* hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
|
|
4134
|
+
* `processConversation` reads `message.opencode_message_id ?? this
|
|
4135
|
+
* .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
|
|
4136
|
+
* threads it into `resolveRedrive`, which asks opencode itself whether the
|
|
4137
|
+
* turn is still ongoing before ever dispatching.
|
|
4138
|
+
*
|
|
4139
|
+
* Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
|
|
4140
|
+
* `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
|
|
4141
|
+
* non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
|
|
4142
|
+
* dispatch) and at the top-level fresh-dispatch site, so it does not outlive
|
|
4143
|
+
* the row it was recorded for.
|
|
4144
|
+
*/
|
|
4145
|
+
releasedOpencodeIds = /* @__PURE__ */ new Map();
|
|
4146
|
+
/**
|
|
4147
|
+
* Per-conversation throttle state for the #183 recurrence warning (#1618
|
|
4148
|
+
* WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
|
|
4149
|
+
* `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
|
|
4150
|
+
* log line and the `dispatch_wedged` signal to at most once per
|
|
4151
|
+
* `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
|
|
4152
|
+
* so the operator sees magnitude, not repetition. Cleared the moment the
|
|
4153
|
+
* conversation dispatches anything (a fresh wedge, if it recurs, is a new
|
|
4154
|
+
* incident). Bounded FIFO, mirroring `supersededSessions`
|
|
4155
|
+
* (`MAX_WEDGED_CONVERSATIONS`).
|
|
4156
|
+
*/
|
|
4157
|
+
wedgeWarnings = /* @__PURE__ */ new Map();
|
|
3491
4158
|
/**
|
|
3492
4159
|
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
3493
4160
|
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
@@ -3706,6 +4373,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3706
4373
|
* same trick `lastProxiedActivityAt` uses.
|
|
3707
4374
|
*/
|
|
3708
4375
|
appliedFileCount = 0;
|
|
4376
|
+
/**
|
|
4377
|
+
* Generation counter, NOT a tally (#1656): advances by exactly one per sync
|
|
4378
|
+
* batch that applied the Claude CLI credential file, not by how many
|
|
4379
|
+
* credential files were in that batch. `run.ts` only ever tests inequality
|
|
4380
|
+
* against the value it saw last cycle, so magnitude is meaningless — keep it
|
|
4381
|
+
* that way rather than "fixing" it into a count.
|
|
4382
|
+
*/
|
|
4383
|
+
claudeCredentialApplyCount = 0;
|
|
4384
|
+
opencodeAuthApplyCount = 0;
|
|
3709
4385
|
/**
|
|
3710
4386
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3711
4387
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -3730,15 +4406,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3730
4406
|
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3731
4407
|
this.log = config.log ?? (() => {
|
|
3732
4408
|
});
|
|
3733
|
-
this.fetchImpl =
|
|
4409
|
+
this.fetchImpl = withRequestTimeout(
|
|
4410
|
+
config.fetchImpl ?? fetch,
|
|
4411
|
+
config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
|
|
4412
|
+
);
|
|
3734
4413
|
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3735
4414
|
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3736
4415
|
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
3737
4416
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
3738
4417
|
this.now = config.now ?? (() => Date.now());
|
|
3739
4418
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3740
|
-
this.homeDir = config.homeDir ??
|
|
4419
|
+
this.homeDir = config.homeDir ?? homedir3();
|
|
3741
4420
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4421
|
+
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4422
|
+
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
3742
4423
|
}
|
|
3743
4424
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
3744
4425
|
get opencodeBase() {
|
|
@@ -3752,6 +4433,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3752
4433
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
3753
4434
|
*/
|
|
3754
4435
|
async drainPending() {
|
|
4436
|
+
try {
|
|
4437
|
+
this.reconcileWatchers();
|
|
4438
|
+
} catch (err) {
|
|
4439
|
+
this.log({
|
|
4440
|
+
level: "error",
|
|
4441
|
+
message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
|
|
4442
|
+
});
|
|
4443
|
+
}
|
|
3755
4444
|
if (this.stopped) return 0;
|
|
3756
4445
|
if (this.draining) return 0;
|
|
3757
4446
|
this.draining = true;
|
|
@@ -3785,7 +4474,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3785
4474
|
if (this.syncingFiles) return 0;
|
|
3786
4475
|
this.syncingFiles = true;
|
|
3787
4476
|
try {
|
|
3788
|
-
const
|
|
4477
|
+
const result = await syncPendingRunnerFiles({
|
|
3789
4478
|
agentId: this.agentId,
|
|
3790
4479
|
apiUrl: this.apiUrl,
|
|
3791
4480
|
getAuthHeader: this.getAuthHeader,
|
|
@@ -3795,8 +4484,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3795
4484
|
ackFailures: this.fileAckFailures,
|
|
3796
4485
|
log: this.log
|
|
3797
4486
|
});
|
|
3798
|
-
this.appliedFileCount += applied;
|
|
3799
|
-
|
|
4487
|
+
this.appliedFileCount += result.applied;
|
|
4488
|
+
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
4489
|
+
if (result.opencodeAuthApplied) this.opencodeAuthApplyCount += 1;
|
|
4490
|
+
return result.applied;
|
|
3800
4491
|
} catch (err) {
|
|
3801
4492
|
this.log({
|
|
3802
4493
|
level: "error",
|
|
@@ -3887,12 +4578,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3887
4578
|
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3888
4579
|
* idle checks still shows up as an advance.
|
|
3889
4580
|
*
|
|
4581
|
+
* A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
|
|
4582
|
+
* (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
|
|
4583
|
+
* reporting on it advancing, so an unrelated file sync can never disturb a
|
|
4584
|
+
* healthy reporting cadence (#1627) — it never even reaches that trigger, let
|
|
4585
|
+
* alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
|
|
4586
|
+
* whose consumer is idle-timeout suppression and must key on ANY file, not
|
|
4587
|
+
* just a Claude credential.
|
|
4588
|
+
*
|
|
3890
4589
|
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3891
4590
|
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3892
4591
|
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3893
4592
|
*/
|
|
3894
4593
|
fileSyncActivity() {
|
|
3895
|
-
return {
|
|
4594
|
+
return {
|
|
4595
|
+
appliedFiles: this.appliedFileCount,
|
|
4596
|
+
inFlight: this.syncingFiles,
|
|
4597
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount,
|
|
4598
|
+
opencodeAuthApplyCount: this.opencodeAuthApplyCount
|
|
4599
|
+
};
|
|
3896
4600
|
}
|
|
3897
4601
|
/**
|
|
3898
4602
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
@@ -4005,8 +4709,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4005
4709
|
skippedAlreadyDispatched += 1;
|
|
4006
4710
|
continue;
|
|
4007
4711
|
}
|
|
4008
|
-
|
|
4009
|
-
|
|
4712
|
+
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
4713
|
+
if (effectiveOpencodeMessageId) {
|
|
4714
|
+
const outcome = await this.resolveRedrive(
|
|
4715
|
+
conv,
|
|
4716
|
+
sessionId,
|
|
4717
|
+
message,
|
|
4718
|
+
sessionCreated,
|
|
4719
|
+
effectiveOpencodeMessageId
|
|
4720
|
+
);
|
|
4010
4721
|
if (outcome === "abandoned") {
|
|
4011
4722
|
continue;
|
|
4012
4723
|
}
|
|
@@ -4117,21 +4828,102 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4117
4828
|
}
|
|
4118
4829
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4119
4830
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
4831
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
4120
4832
|
this.dispatched.add(message.id);
|
|
4121
4833
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
4122
4834
|
dispatched += 1;
|
|
4123
4835
|
void this.postSignal(conv.id, message.id, "dispatched");
|
|
4124
4836
|
}
|
|
4125
4837
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
4126
|
-
this.
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
conversation_id: conv.id
|
|
4130
|
-
});
|
|
4838
|
+
this.reportWedgedConversation(conv, messages);
|
|
4839
|
+
} else if (dispatched > 0) {
|
|
4840
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4131
4841
|
}
|
|
4132
4842
|
this.ensureWatcherRunning(sessionId);
|
|
4133
4843
|
return dispatched;
|
|
4134
4844
|
}
|
|
4845
|
+
/**
|
|
4846
|
+
* The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
|
|
4847
|
+
* escalated (#1618 WI-4). `messages` is the conversation's full pending list
|
|
4848
|
+
* on THIS tick — the caller has already confirmed every one of them is a
|
|
4849
|
+
* skip-because-already-`dispatched`, the exact signature of a message stuck
|
|
4850
|
+
* acknowledged-but-never-worked.
|
|
4851
|
+
*
|
|
4852
|
+
* Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
|
|
4853
|
+
* (52,843 occurrences observed in one incident) — burning the GLOBAL
|
|
4854
|
+
* 30-events/60s `runner-activity-telemetry.ts` budget that was itself
|
|
4855
|
+
* suppressing the diagnostics needed to debug the wedge. The `warn` log (and
|
|
4856
|
+
* the `dispatch_wedged` signal once the wedge has persisted past the same
|
|
4857
|
+
* interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
|
|
4858
|
+
* naming the consecutive-tick count so the operator sees magnitude rather
|
|
4859
|
+
* than repetition.
|
|
4860
|
+
*
|
|
4861
|
+
* Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
|
|
4862
|
+
* unconditionally on this same tick and is already recovering anything it
|
|
4863
|
+
* can see. This is reporting only — see `countUntrackedIds`'s doc for the
|
|
4864
|
+
* one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
|
|
4865
|
+
*/
|
|
4866
|
+
reportWedgedConversation(conv, messages) {
|
|
4867
|
+
const now = this.now();
|
|
4868
|
+
const existing = this.wedgeWarnings.get(conv.id);
|
|
4869
|
+
const firstWedgedAt = existing?.firstWedgedAt ?? now;
|
|
4870
|
+
const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
|
|
4871
|
+
const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
|
|
4872
|
+
if (!dueForWarn) {
|
|
4873
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4874
|
+
this.wedgeWarnings.set(conv.id, {
|
|
4875
|
+
firstWedgedAt,
|
|
4876
|
+
lastWarnedAt: existing.lastWarnedAt,
|
|
4877
|
+
consecutiveTicks
|
|
4878
|
+
});
|
|
4879
|
+
return;
|
|
4880
|
+
}
|
|
4881
|
+
const stuckForMs = now - firstWedgedAt;
|
|
4882
|
+
const untracked = this.countUntrackedIds(messages);
|
|
4883
|
+
this.log({
|
|
4884
|
+
level: "warn",
|
|
4885
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
4886
|
+
conversation_id: conv.id
|
|
4887
|
+
});
|
|
4888
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4889
|
+
this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
|
|
4890
|
+
while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
|
|
4891
|
+
const oldest = this.wedgeWarnings.keys().next().value;
|
|
4892
|
+
if (oldest === void 0) break;
|
|
4893
|
+
this.wedgeWarnings.delete(oldest);
|
|
4894
|
+
}
|
|
4895
|
+
if (stuckForMs >= this.wedgeWarningIntervalMs) {
|
|
4896
|
+
void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
|
|
4897
|
+
stuck_for_ms: stuckForMs,
|
|
4898
|
+
untracked
|
|
4899
|
+
});
|
|
4900
|
+
}
|
|
4901
|
+
}
|
|
4902
|
+
/**
|
|
4903
|
+
* How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
|
|
4904
|
+
* WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
|
|
4905
|
+
* `dispatched`/`inFlight` pairing invariant holds by construction across
|
|
4906
|
+
* every `dispatched.add` site (see its own doc comment), so `> 0` here means
|
|
4907
|
+
* that invariant has actually been violated for this conversation: there is
|
|
4908
|
+
* no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
|
|
4909
|
+
* `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
|
|
4910
|
+
* recovering. One pass over `this.watchers`, called only when the throttled
|
|
4911
|
+
* warning above is due to fire — not every tick.
|
|
4912
|
+
*/
|
|
4913
|
+
countUntrackedIds(messages) {
|
|
4914
|
+
let untracked = 0;
|
|
4915
|
+
for (const message of messages) {
|
|
4916
|
+
let tracked = false;
|
|
4917
|
+
for (const watcher of this.watchers.values()) {
|
|
4918
|
+
if (watcher.inFlight.has(message.id)) {
|
|
4919
|
+
tracked = true;
|
|
4920
|
+
break;
|
|
4921
|
+
}
|
|
4922
|
+
}
|
|
4923
|
+
if (!tracked) untracked += 1;
|
|
4924
|
+
}
|
|
4925
|
+
return untracked;
|
|
4926
|
+
}
|
|
4135
4927
|
/**
|
|
4136
4928
|
* Poll a session's message list for the re-drive fence (#965), via the
|
|
4137
4929
|
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
@@ -4200,9 +4992,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4200
4992
|
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
4201
4993
|
* other failure resolves to `unresolved` and is retried whole on the next
|
|
4202
4994
|
* ~2s drain tick.
|
|
4995
|
+
*
|
|
4996
|
+
* `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
|
|
4997
|
+
* server `opencode_message_id` when present, else the stall watchdog's local
|
|
4998
|
+
* `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
|
|
4999
|
+
* `message` so every line below — and the signals this method posts —
|
|
5000
|
+
* keeps reporting the REAL server row; a shadow-copied `message` would
|
|
5001
|
+
* silently diverge from it.
|
|
4203
5002
|
*/
|
|
4204
|
-
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
4205
|
-
const ocId =
|
|
5003
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
|
|
5004
|
+
const ocId = effectiveOpencodeMessageId;
|
|
4206
5005
|
if (sessionCreated) {
|
|
4207
5006
|
this.clearRedriveUnresolved(message.id);
|
|
4208
5007
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
@@ -4432,7 +5231,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4432
5231
|
}
|
|
4433
5232
|
return "unresolved";
|
|
4434
5233
|
}
|
|
4435
|
-
/**
|
|
5234
|
+
/**
|
|
5235
|
+
* Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
|
|
5236
|
+
* outcome) — including the stall watchdog's local re-drive fence (#1618): once
|
|
5237
|
+
* `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
|
|
5238
|
+
* a real server-side `opencode_message_id` again or is no longer pending, so
|
|
5239
|
+
* the fence entry is no longer needed.
|
|
5240
|
+
*/
|
|
4436
5241
|
clearRedriveUnresolved(messageId) {
|
|
4437
5242
|
this.redriveUnresolvedSince.delete(messageId);
|
|
4438
5243
|
this.redriveUnresolvedSignalled.delete(messageId);
|
|
@@ -4440,6 +5245,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4440
5245
|
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4441
5246
|
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4442
5247
|
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
5248
|
+
this.releasedOpencodeIds.delete(messageId);
|
|
4443
5249
|
}
|
|
4444
5250
|
/**
|
|
4445
5251
|
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
@@ -4594,6 +5400,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4594
5400
|
this.supersededSessions.delete(oldest);
|
|
4595
5401
|
}
|
|
4596
5402
|
}
|
|
5403
|
+
/**
|
|
5404
|
+
* Record the local re-drive fence for a message force-released without
|
|
5405
|
+
* completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
|
|
5406
|
+
* `removeInFlight`, which is about to drop the `InFlightMessage` this reads
|
|
5407
|
+
* `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
|
|
5408
|
+
*/
|
|
5409
|
+
recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
|
|
5410
|
+
this.releasedOpencodeIds.delete(evidentMessageId);
|
|
5411
|
+
this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
|
|
5412
|
+
while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
|
|
5413
|
+
const oldest = this.releasedOpencodeIds.keys().next().value;
|
|
5414
|
+
if (oldest === void 0) return;
|
|
5415
|
+
this.releasedOpencodeIds.delete(oldest);
|
|
5416
|
+
}
|
|
5417
|
+
}
|
|
4597
5418
|
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
4598
5419
|
isSuperseded(conversationId, sessionId) {
|
|
4599
5420
|
return this.supersededSessions.get(conversationId) === sessionId;
|
|
@@ -4635,6 +5456,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4635
5456
|
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
4636
5457
|
conversation_id: conv.id
|
|
4637
5458
|
});
|
|
5459
|
+
const watcher = this.watchers.get(bound);
|
|
5460
|
+
if (watcher) {
|
|
5461
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5462
|
+
this.recordReleasedOpencodeId(evidentMessageId, bound, inFlight.opencodeMessageId);
|
|
5463
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5464
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5465
|
+
recovery: "session_gone_released"
|
|
5466
|
+
});
|
|
5467
|
+
}
|
|
5468
|
+
this.watchers.delete(bound);
|
|
5469
|
+
}
|
|
4638
5470
|
this.sessions.delete(conv.id);
|
|
4639
5471
|
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
4640
5472
|
}
|
|
@@ -4825,15 +5657,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4825
5657
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
4826
5658
|
let watcher = this.watchers.get(sessionId);
|
|
4827
5659
|
if (!watcher) {
|
|
4828
|
-
watcher =
|
|
4829
|
-
conv,
|
|
4830
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4831
|
-
loop: null,
|
|
4832
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4833
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4834
|
-
lastGoodPollAt: this.now(),
|
|
4835
|
-
hadUsablePoll: false
|
|
4836
|
-
};
|
|
5660
|
+
watcher = this.newSessionWatcher(conv);
|
|
4837
5661
|
this.watchers.set(sessionId, watcher);
|
|
4838
5662
|
}
|
|
4839
5663
|
const now = this.now();
|
|
@@ -4864,6 +5688,27 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4864
5688
|
ambiguousResolved: false
|
|
4865
5689
|
});
|
|
4866
5690
|
}
|
|
5691
|
+
/**
|
|
5692
|
+
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
5693
|
+
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
5694
|
+
* misread as stalled by the very first reconciliation that sees it.
|
|
5695
|
+
*/
|
|
5696
|
+
newSessionWatcher(conv) {
|
|
5697
|
+
const now = this.now();
|
|
5698
|
+
return {
|
|
5699
|
+
conv,
|
|
5700
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
5701
|
+
loop: null,
|
|
5702
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
5703
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
5704
|
+
lastGoodPollAt: now,
|
|
5705
|
+
hadUsablePoll: false,
|
|
5706
|
+
generation: 0,
|
|
5707
|
+
lastTickAt: now,
|
|
5708
|
+
lastObservedTickAt: now,
|
|
5709
|
+
consecutiveStallRestarts: 0
|
|
5710
|
+
};
|
|
5711
|
+
}
|
|
4867
5712
|
/**
|
|
4868
5713
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
4869
5714
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
@@ -4893,15 +5738,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4893
5738
|
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
4894
5739
|
let watcher = this.watchers.get(sessionId);
|
|
4895
5740
|
if (!watcher) {
|
|
4896
|
-
watcher =
|
|
4897
|
-
conv,
|
|
4898
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4899
|
-
loop: null,
|
|
4900
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4901
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4902
|
-
lastGoodPollAt: this.now(),
|
|
4903
|
-
hadUsablePoll: false
|
|
4904
|
-
};
|
|
5741
|
+
watcher = this.newSessionWatcher(conv);
|
|
4905
5742
|
this.watchers.set(sessionId, watcher);
|
|
4906
5743
|
}
|
|
4907
5744
|
watcher.inFlight.set(message.id, {
|
|
@@ -4945,12 +5782,110 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4945
5782
|
ambiguousResolved: false
|
|
4946
5783
|
});
|
|
4947
5784
|
}
|
|
5785
|
+
/**
|
|
5786
|
+
* Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
|
|
5787
|
+
* restarts any per-session watcher whose loop has exited or stopped ticking
|
|
5788
|
+
* — escalating to a bounded force-release only once
|
|
5789
|
+
* `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
|
|
5790
|
+
* it. Fully synchronous: it only inspects in-memory state and calls the
|
|
5791
|
+
* synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
|
|
5792
|
+
* run from the very top of `drainPending()` — ahead of the un-timed
|
|
5793
|
+
* `getPendingConversations()` await that would otherwise be able to disable
|
|
5794
|
+
* it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
|
|
5795
|
+
* `drainPending()` from being CALLED again at all, not just from finishing).
|
|
5796
|
+
*
|
|
5797
|
+
* Restarts the loop rather than releasing messages directly: a blind release
|
|
5798
|
+
* would let the next drain re-`prompt_async` a turn that may still be
|
|
5799
|
+
* running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
|
|
5800
|
+
* re-polls with each message's `opencodeMessageId` still in hand and lets
|
|
5801
|
+
* the existing, audited `!activelyRunning` give-up decide, same as it always
|
|
5802
|
+
* has.
|
|
5803
|
+
*
|
|
5804
|
+
* Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
|
|
5805
|
+
* that shape has no in-flight entry and therefore no `opencodeMessageId` to
|
|
5806
|
+
* fence a release with, so releasing it here would blind-re-POST a possibly-
|
|
5807
|
+
* running turn — and there is no conversation id in hand to signal with
|
|
5808
|
+
* either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
|
|
5809
|
+
* instead, where a conversation id already exists. If you find yourself
|
|
5810
|
+
* wanting to add a `dispatched` sweep here, don't — read the drain-wedge
|
|
5811
|
+
* plan's §3/D5 first.
|
|
5812
|
+
*/
|
|
5813
|
+
reconcileWatchers() {
|
|
5814
|
+
const now = this.now();
|
|
5815
|
+
for (const [sessionId, watcher] of [...this.watchers]) {
|
|
5816
|
+
if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
|
|
5817
|
+
watcher.consecutiveStallRestarts = 0;
|
|
5818
|
+
}
|
|
5819
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5820
|
+
if (watcher.inFlight.size === 0 && watcher.loop === null) {
|
|
5821
|
+
this.watchers.delete(sessionId);
|
|
5822
|
+
continue;
|
|
5823
|
+
}
|
|
5824
|
+
if (watcher.loop === null && watcher.inFlight.size > 0) {
|
|
5825
|
+
if (now - watcher.lastTickAt < this.watcherStallMs) continue;
|
|
5826
|
+
this.log({
|
|
5827
|
+
level: "warn",
|
|
5828
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
|
|
5829
|
+
conversation_id: watcher.conv.id
|
|
5830
|
+
});
|
|
5831
|
+
this.ensureWatcherRunning(sessionId);
|
|
5832
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5833
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5834
|
+
recovery: "loop_exited"
|
|
5835
|
+
});
|
|
5836
|
+
}
|
|
5837
|
+
continue;
|
|
5838
|
+
}
|
|
5839
|
+
if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
|
|
5840
|
+
const stalledForMs = now - watcher.lastTickAt;
|
|
5841
|
+
watcher.consecutiveStallRestarts += 1;
|
|
5842
|
+
if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
|
|
5843
|
+
this.log({
|
|
5844
|
+
level: "error",
|
|
5845
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
|
|
5846
|
+
conversation_id: watcher.conv.id
|
|
5847
|
+
});
|
|
5848
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5849
|
+
this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
|
|
5850
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5851
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5852
|
+
recovery: "unrecoverable_released"
|
|
5853
|
+
});
|
|
5854
|
+
}
|
|
5855
|
+
watcher.generation += 1;
|
|
5856
|
+
this.watchers.delete(sessionId);
|
|
5857
|
+
continue;
|
|
5858
|
+
}
|
|
5859
|
+
watcher.generation += 1;
|
|
5860
|
+
watcher.loop = null;
|
|
5861
|
+
watcher.lastGoodPollAt = now;
|
|
5862
|
+
watcher.lastTickAt = now;
|
|
5863
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5864
|
+
this.ensureWatcherRunning(sessionId);
|
|
5865
|
+
this.log({
|
|
5866
|
+
level: "warn",
|
|
5867
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
|
|
5868
|
+
conversation_id: watcher.conv.id
|
|
5869
|
+
});
|
|
5870
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5871
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5872
|
+
recovery: "loop_stalled"
|
|
5873
|
+
});
|
|
5874
|
+
}
|
|
5875
|
+
}
|
|
5876
|
+
}
|
|
5877
|
+
}
|
|
4948
5878
|
/**
|
|
4949
5879
|
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
4950
5880
|
* work and is not already running. Single-flight per session. The loop is
|
|
4951
5881
|
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
4952
5882
|
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
4953
5883
|
* stays as the safety net.
|
|
5884
|
+
*
|
|
5885
|
+
* The generation started here (#1618) is captured in the `.finally` closure
|
|
5886
|
+
* so a RETIRED loop settling late — after `reconcileWatchers` has already
|
|
5887
|
+
* restarted this watcher under a newer generation — can neither null the new
|
|
5888
|
+
* loop's handle nor delete a watcher that still has live work.
|
|
4954
5889
|
*/
|
|
4955
5890
|
ensureWatcherRunning(sessionId) {
|
|
4956
5891
|
const watcher = this.watchers.get(sessionId);
|
|
@@ -4960,7 +5895,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4960
5895
|
this.watchers.delete(sessionId);
|
|
4961
5896
|
return;
|
|
4962
5897
|
}
|
|
4963
|
-
const
|
|
5898
|
+
const generation = watcher.generation;
|
|
5899
|
+
const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
|
|
5900
|
+
if (watcher.generation !== generation) return;
|
|
4964
5901
|
watcher.loop = null;
|
|
4965
5902
|
if (watcher.inFlight.size === 0) {
|
|
4966
5903
|
this.watchers.delete(sessionId);
|
|
@@ -4980,11 +5917,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4980
5917
|
* `source_message_id`;
|
|
4981
5918
|
* 4. drops messages that completed or timed out from the in-flight set.
|
|
4982
5919
|
* Exits when the in-flight set empties. Never throws.
|
|
5920
|
+
*
|
|
5921
|
+
* `generation` (#1618) is the incarnation this call was started under.
|
|
5922
|
+
* `reconcileWatchers` can restart a stalled loop by bumping
|
|
5923
|
+
* `watcher.generation` and starting a NEW `runWatcherLoop` over the same
|
|
5924
|
+
* `SessionWatcher` object — the stalled promise itself cannot be cancelled,
|
|
5925
|
+
* so this loop instead checks at the top of every iteration, right after
|
|
5926
|
+
* waking from `sleep`, and right before servicing any message, and quietly
|
|
5927
|
+
* retires (returns without touching anything) the moment it is no longer the
|
|
5928
|
+
* watcher's current generation. Retiring mid-tick can still let ONE
|
|
5929
|
+
* `serviceInFlightMessage` pass complete first — acceptable, since that
|
|
5930
|
+
* method contains no non-idempotent action.
|
|
4983
5931
|
*/
|
|
4984
|
-
async runWatcherLoop(sessionId, watcher) {
|
|
5932
|
+
async runWatcherLoop(sessionId, watcher, generation) {
|
|
4985
5933
|
try {
|
|
4986
5934
|
while (watcher.inFlight.size > 0) {
|
|
5935
|
+
if (watcher.generation !== generation) return;
|
|
5936
|
+
watcher.lastTickAt = this.now();
|
|
4987
5937
|
await this.sleep(this.pausedPollIntervalMs);
|
|
5938
|
+
if (watcher.generation !== generation) return;
|
|
4988
5939
|
let messages = null;
|
|
4989
5940
|
try {
|
|
4990
5941
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
@@ -5005,6 +5956,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5005
5956
|
}
|
|
5006
5957
|
}
|
|
5007
5958
|
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
5959
|
+
if (watcher.generation !== generation) return;
|
|
5008
5960
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
5009
5961
|
await this.serviceInFlightMessage(
|
|
5010
5962
|
sessionId,
|
|
@@ -6980,7 +7932,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
6980
7932
|
if (trimmed === "") {
|
|
6981
7933
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6982
7934
|
}
|
|
6983
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7935
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join7(homeDir, trimmed.slice(2)) : trimmed;
|
|
6984
7936
|
if (!isAbsolute2(expanded)) {
|
|
6985
7937
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6986
7938
|
}
|
|
@@ -7078,7 +8030,7 @@ function logActivity(state, entry) {
|
|
|
7078
8030
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
7079
8031
|
if (!meetsThreshold(state, level)) return;
|
|
7080
8032
|
forwardRunnerActivity(
|
|
7081
|
-
{ level, message: entry.message, error: entry.error },
|
|
8033
|
+
{ level, message: entry.message, error: entry.error, metadata: entry.metadata },
|
|
7082
8034
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
7083
8035
|
);
|
|
7084
8036
|
const fullEntry = {
|
|
@@ -7098,6 +8050,26 @@ function logActivity(state, entry) {
|
|
|
7098
8050
|
}
|
|
7099
8051
|
}
|
|
7100
8052
|
}
|
|
8053
|
+
function reportSessionDbRecovery(state) {
|
|
8054
|
+
try {
|
|
8055
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir4(), env: process.env });
|
|
8056
|
+
for (const record of report.records) {
|
|
8057
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
8058
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
8059
|
+
logActivity(state, {
|
|
8060
|
+
type: activity.level === "error" ? "error" : "info",
|
|
8061
|
+
level: activity.level,
|
|
8062
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
8063
|
+
metadata: activity.metadata
|
|
8064
|
+
});
|
|
8065
|
+
}
|
|
8066
|
+
acknowledgeSessionDbRecoveryReport(report.path);
|
|
8067
|
+
} catch (error2) {
|
|
8068
|
+
console.error(
|
|
8069
|
+
`[run] could not report session-DB recovery activity: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
8070
|
+
);
|
|
8071
|
+
}
|
|
8072
|
+
}
|
|
7101
8073
|
function displayStatus(state) {
|
|
7102
8074
|
if (!state.interactive) return;
|
|
7103
8075
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
@@ -7187,6 +8159,8 @@ async function driveChannels(state, driver) {
|
|
|
7187
8159
|
let unreachableMs = 0;
|
|
7188
8160
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7189
8161
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
8162
|
+
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
8163
|
+
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
7190
8164
|
while (state.running) {
|
|
7191
8165
|
const cycleStartedAtMs = performance.now();
|
|
7192
8166
|
let idleThisCycle = false;
|
|
@@ -7210,11 +8184,19 @@ async function driveChannels(state, driver) {
|
|
|
7210
8184
|
state.messageCount += processed;
|
|
7211
8185
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
7212
8186
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7213
|
-
const
|
|
8187
|
+
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
8188
|
+
const appliedFiles = fileActivitySnapshot.appliedFiles;
|
|
7214
8189
|
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
7215
8190
|
const fileActivity = carriedOverFileSync || filesApplied;
|
|
7216
8191
|
lastSeenAppliedFiles = appliedFiles;
|
|
7217
|
-
|
|
8192
|
+
const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
|
|
8193
|
+
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
8194
|
+
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
8195
|
+
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
8196
|
+
const opencodeAuthApplies = fileActivitySnapshot.opencodeAuthApplyCount;
|
|
8197
|
+
const opencodeAuthApplied = opencodeAuthApplies !== lastSeenOpencodeAuthApplies;
|
|
8198
|
+
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
8199
|
+
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
7218
8200
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7219
8201
|
idlePolls = 0;
|
|
7220
8202
|
idleMs = 0;
|
|
@@ -7283,7 +8265,7 @@ async function driveChannels(state, driver) {
|
|
|
7283
8265
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7284
8266
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7285
8267
|
function sessionDbPath() {
|
|
7286
|
-
return
|
|
8268
|
+
return join7(homedir4(), ".local", "share", "opencode", "opencode.db");
|
|
7287
8269
|
}
|
|
7288
8270
|
async function runSweep(state, driver, config) {
|
|
7289
8271
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7366,7 +8348,7 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7366
8348
|
for (const warning2 of config.warnings) {
|
|
7367
8349
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
7368
8350
|
}
|
|
7369
|
-
const dbBytes = statSessionDbBytes(
|
|
8351
|
+
const dbBytes = statSessionDbBytes(homedir4());
|
|
7370
8352
|
void (async () => {
|
|
7371
8353
|
const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
7372
8354
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -7394,95 +8376,110 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7394
8376
|
);
|
|
7395
8377
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
7396
8378
|
}
|
|
7397
|
-
function
|
|
7398
|
-
|
|
7399
|
-
}
|
|
7400
|
-
function scheduleClaudeUsageReporting(state, options) {
|
|
7401
|
-
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
7402
|
-
options.claudeUsageReporting,
|
|
7403
|
-
process.env
|
|
7404
|
-
);
|
|
8379
|
+
function scheduleUsageReporting(state, params) {
|
|
8380
|
+
const { mode, warnings } = params.resolved;
|
|
7405
8381
|
for (const warning2 of warnings) {
|
|
7406
8382
|
logActivity(state, {
|
|
7407
8383
|
type: "info",
|
|
7408
8384
|
level: "warn",
|
|
7409
|
-
message:
|
|
8385
|
+
message: `${params.label} usage reporting: ${warning2}`
|
|
7410
8386
|
});
|
|
7411
8387
|
}
|
|
7412
8388
|
if (mode === "off") {
|
|
7413
8389
|
logActivity(state, {
|
|
7414
8390
|
type: "info",
|
|
7415
8391
|
level: "debug",
|
|
7416
|
-
message:
|
|
8392
|
+
message: `${params.label} usage reporting is off (${params.offFlagHint})`
|
|
7417
8393
|
});
|
|
7418
8394
|
return null;
|
|
7419
8395
|
}
|
|
7420
8396
|
let consecutiveFailures = 0;
|
|
7421
|
-
let
|
|
8397
|
+
let phase = "dormant";
|
|
7422
8398
|
let rearmRequested = false;
|
|
8399
|
+
const armProbe = () => {
|
|
8400
|
+
phase = "probe-pending";
|
|
8401
|
+
params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
|
|
8402
|
+
};
|
|
7423
8403
|
const scheduleNextTick = () => {
|
|
7424
|
-
|
|
7425
|
-
|
|
7426
|
-
|
|
8404
|
+
if (rearmRequested) {
|
|
8405
|
+
rearmRequested = false;
|
|
8406
|
+
armProbe();
|
|
8407
|
+
return;
|
|
8408
|
+
}
|
|
8409
|
+
phase = "steady-pending";
|
|
8410
|
+
params.setTimer(setTimeout(() => void tick(false), params.nextDelayMs()));
|
|
7427
8411
|
};
|
|
7428
8412
|
const rearm = () => {
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
8413
|
+
switch (phase) {
|
|
8414
|
+
case "tick-in-flight":
|
|
8415
|
+
rearmRequested = true;
|
|
8416
|
+
return;
|
|
8417
|
+
case "probe-pending":
|
|
8418
|
+
return;
|
|
8419
|
+
case "steady-pending":
|
|
8420
|
+
if (params.getTimer()) {
|
|
8421
|
+
clearTimeout(params.getTimer());
|
|
8422
|
+
params.setTimer(null);
|
|
8423
|
+
}
|
|
8424
|
+
rearmRequested = false;
|
|
8425
|
+
armProbe();
|
|
8426
|
+
return;
|
|
8427
|
+
case "dormant":
|
|
8428
|
+
rearmRequested = false;
|
|
8429
|
+
armProbe();
|
|
8430
|
+
return;
|
|
7432
8431
|
}
|
|
7433
|
-
rearmRequested = false;
|
|
7434
|
-
armed = true;
|
|
7435
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7436
8432
|
};
|
|
7437
8433
|
const tick = async (isProbe) => {
|
|
8434
|
+
phase = "tick-in-flight";
|
|
7438
8435
|
try {
|
|
7439
|
-
const usage = await
|
|
7440
|
-
const result = await
|
|
8436
|
+
const usage = await params.fetchUsage();
|
|
8437
|
+
const result = await params.report(usage);
|
|
7441
8438
|
if (result.ok) {
|
|
7442
8439
|
if (consecutiveFailures > 0) {
|
|
7443
8440
|
logActivity(state, {
|
|
7444
8441
|
type: "info",
|
|
7445
8442
|
level: "info",
|
|
7446
|
-
message:
|
|
8443
|
+
message: `${params.label} usage reporting recovered`
|
|
7447
8444
|
});
|
|
7448
8445
|
}
|
|
7449
8446
|
consecutiveFailures = 0;
|
|
7450
8447
|
logActivity(state, {
|
|
7451
8448
|
type: "info",
|
|
7452
8449
|
level: "debug",
|
|
7453
|
-
message:
|
|
8450
|
+
message: `Reported ${params.label} usage to Evident`
|
|
7454
8451
|
});
|
|
7455
8452
|
} else {
|
|
7456
8453
|
consecutiveFailures++;
|
|
7457
8454
|
logActivity(state, {
|
|
7458
8455
|
type: "info",
|
|
7459
|
-
level:
|
|
7460
|
-
message: `Failed to report
|
|
8456
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
8457
|
+
message: `Failed to report ${params.label} usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
7461
8458
|
});
|
|
7462
8459
|
}
|
|
7463
8460
|
scheduleNextTick();
|
|
7464
8461
|
} catch (error2) {
|
|
7465
|
-
if (
|
|
8462
|
+
if (params.isLocalCredentialProblem(error2)) {
|
|
7466
8463
|
if (mode === "on") {
|
|
7467
8464
|
logActivity(state, {
|
|
7468
8465
|
type: "info",
|
|
7469
8466
|
level: "warn",
|
|
7470
|
-
message:
|
|
8467
|
+
message: `${params.label} usage reporting is forced on but no usable login was found \u2014 ${params.forcedOnHint}; reporting will keep retrying`
|
|
7471
8468
|
});
|
|
7472
8469
|
scheduleNextTick();
|
|
7473
8470
|
} else if (isProbe) {
|
|
7474
8471
|
logActivity(state, {
|
|
7475
8472
|
type: "info",
|
|
7476
8473
|
level: "debug",
|
|
7477
|
-
message:
|
|
8474
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
7478
8475
|
});
|
|
7479
|
-
|
|
8476
|
+
phase = "dormant";
|
|
7480
8477
|
if (rearmRequested) rearm();
|
|
7481
8478
|
} else {
|
|
7482
8479
|
logActivity(state, {
|
|
7483
8480
|
type: "info",
|
|
7484
8481
|
level: "debug",
|
|
7485
|
-
message:
|
|
8482
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
7486
8483
|
});
|
|
7487
8484
|
scheduleNextTick();
|
|
7488
8485
|
}
|
|
@@ -7491,17 +8488,118 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7491
8488
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
7492
8489
|
logActivity(state, {
|
|
7493
8490
|
type: "info",
|
|
7494
|
-
level:
|
|
7495
|
-
message:
|
|
8491
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
8492
|
+
message: `${params.label} usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
7496
8493
|
});
|
|
7497
8494
|
scheduleNextTick();
|
|
7498
8495
|
}
|
|
7499
8496
|
}
|
|
7500
8497
|
};
|
|
7501
|
-
|
|
7502
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
8498
|
+
armProbe();
|
|
7503
8499
|
return rearm;
|
|
7504
8500
|
}
|
|
8501
|
+
function scheduleClaudeUsageReporting(state, options) {
|
|
8502
|
+
return scheduleUsageReporting(state, {
|
|
8503
|
+
label: "Claude",
|
|
8504
|
+
resolved: resolveClaudeUsageReportingMode(options.claudeUsageReporting, process.env),
|
|
8505
|
+
offFlagHint: "--claude-usage-reporting off",
|
|
8506
|
+
getTimer: () => state.claudeUsageTimer,
|
|
8507
|
+
setTimer: (timer) => {
|
|
8508
|
+
state.claudeUsageTimer = timer;
|
|
8509
|
+
},
|
|
8510
|
+
fetchUsage: getClaudeUsage,
|
|
8511
|
+
report: (usage) => reportClaudeUsage(state.agentId, state.authHeader, usage),
|
|
8512
|
+
isLocalCredentialProblem,
|
|
8513
|
+
forcedOnHint: "run `claude` to sign in",
|
|
8514
|
+
firstDelayMs: () => FIRST_REPORT_DELAY_MS,
|
|
8515
|
+
nextDelayMs: nextReportDelayMs,
|
|
8516
|
+
failureLogLevel: claudeUsageFailureLogLevel
|
|
8517
|
+
});
|
|
8518
|
+
}
|
|
8519
|
+
var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
8520
|
+
var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
8521
|
+
var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
8522
|
+
function scheduleResourceUsageReporting(state, options) {
|
|
8523
|
+
const { enabled, warnings } = resolveResourceUsageReportingEnabled(
|
|
8524
|
+
options.resourceUsageReporting,
|
|
8525
|
+
process.env
|
|
8526
|
+
);
|
|
8527
|
+
for (const warning2 of warnings) {
|
|
8528
|
+
logActivity(state, {
|
|
8529
|
+
type: "info",
|
|
8530
|
+
level: "warn",
|
|
8531
|
+
message: `Resource usage reporting: ${warning2}`
|
|
8532
|
+
});
|
|
8533
|
+
}
|
|
8534
|
+
if (!enabled) {
|
|
8535
|
+
logActivity(state, {
|
|
8536
|
+
type: "info",
|
|
8537
|
+
level: "debug",
|
|
8538
|
+
message: "Resource usage reporting is off (--no-resource-usage-reporting)"
|
|
8539
|
+
});
|
|
8540
|
+
return;
|
|
8541
|
+
}
|
|
8542
|
+
const collect = createResourceUsageCollector(homedir4());
|
|
8543
|
+
let consecutiveFailures = 0;
|
|
8544
|
+
const tick = async () => {
|
|
8545
|
+
try {
|
|
8546
|
+
const { usage, warnings: collectWarnings } = await collect();
|
|
8547
|
+
for (const warning2 of collectWarnings) {
|
|
8548
|
+
logActivity(state, {
|
|
8549
|
+
type: "info",
|
|
8550
|
+
level: "debug",
|
|
8551
|
+
message: `Resource usage collection: ${warning2}`
|
|
8552
|
+
});
|
|
8553
|
+
}
|
|
8554
|
+
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
8555
|
+
if (result.ok) {
|
|
8556
|
+
if (consecutiveFailures > 0) {
|
|
8557
|
+
logActivity(state, {
|
|
8558
|
+
type: "info",
|
|
8559
|
+
level: "info",
|
|
8560
|
+
message: "Resource usage reporting recovered"
|
|
8561
|
+
});
|
|
8562
|
+
}
|
|
8563
|
+
consecutiveFailures = 0;
|
|
8564
|
+
logActivity(state, {
|
|
8565
|
+
type: "info",
|
|
8566
|
+
level: "debug",
|
|
8567
|
+
message: "Reported resource usage to Evident"
|
|
8568
|
+
});
|
|
8569
|
+
} else {
|
|
8570
|
+
consecutiveFailures++;
|
|
8571
|
+
logActivity(state, {
|
|
8572
|
+
type: "info",
|
|
8573
|
+
level: reportFailureLogLevel(
|
|
8574
|
+
consecutiveFailures,
|
|
8575
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8576
|
+
),
|
|
8577
|
+
message: `Failed to report resource usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
8578
|
+
});
|
|
8579
|
+
}
|
|
8580
|
+
} catch (error2) {
|
|
8581
|
+
consecutiveFailures++;
|
|
8582
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8583
|
+
logActivity(state, {
|
|
8584
|
+
type: "info",
|
|
8585
|
+
level: reportFailureLogLevel(
|
|
8586
|
+
consecutiveFailures,
|
|
8587
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8588
|
+
),
|
|
8589
|
+
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
8590
|
+
});
|
|
8591
|
+
} finally {
|
|
8592
|
+
state.resourceUsageTimer = setTimeout(
|
|
8593
|
+
() => void tick(),
|
|
8594
|
+
jitteredDelayMs(
|
|
8595
|
+
RESOURCE_USAGE_BASE_REPORT_DELAY_MS,
|
|
8596
|
+
RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION
|
|
8597
|
+
)
|
|
8598
|
+
);
|
|
8599
|
+
}
|
|
8600
|
+
};
|
|
8601
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
8602
|
+
}
|
|
7505
8603
|
async function notifyOffline(state) {
|
|
7506
8604
|
if (!state.agentId || !state.authHeader) return;
|
|
7507
8605
|
if (!state.connected) {
|
|
@@ -7542,6 +8640,15 @@ async function cleanup(state, opts = {}) {
|
|
|
7542
8640
|
state.claudeUsageTimer = null;
|
|
7543
8641
|
}
|
|
7544
8642
|
state.claudeUsageRearm = null;
|
|
8643
|
+
if (state.openaiUsageTimer) {
|
|
8644
|
+
clearTimeout(state.openaiUsageTimer);
|
|
8645
|
+
state.openaiUsageTimer = null;
|
|
8646
|
+
}
|
|
8647
|
+
state.openaiUsageRearm = null;
|
|
8648
|
+
if (state.resourceUsageTimer) {
|
|
8649
|
+
clearTimeout(state.resourceUsageTimer);
|
|
8650
|
+
state.resourceUsageTimer = null;
|
|
8651
|
+
}
|
|
7545
8652
|
if (opts.graceful && state.channelDriver) {
|
|
7546
8653
|
state.channelDriver.stop();
|
|
7547
8654
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -7589,7 +8696,7 @@ async function run(options) {
|
|
|
7589
8696
|
let fileSyncDirectories;
|
|
7590
8697
|
try {
|
|
7591
8698
|
logLevel = resolveLogLevel(options);
|
|
7592
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
8699
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir4());
|
|
7593
8700
|
} catch (error2) {
|
|
7594
8701
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
7595
8702
|
if (options.json) {
|
|
@@ -7624,6 +8731,9 @@ async function run(options) {
|
|
|
7624
8731
|
sessionCleanupTimers: [],
|
|
7625
8732
|
claudeUsageTimer: null,
|
|
7626
8733
|
claudeUsageRearm: null,
|
|
8734
|
+
openaiUsageTimer: null,
|
|
8735
|
+
openaiUsageRearm: null,
|
|
8736
|
+
resourceUsageTimer: null,
|
|
7627
8737
|
authHeader: ""
|
|
7628
8738
|
};
|
|
7629
8739
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
@@ -7818,6 +8928,7 @@ async function run(options) {
|
|
|
7818
8928
|
} else {
|
|
7819
8929
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
7820
8930
|
}
|
|
8931
|
+
reportSessionDbRecovery(state);
|
|
7821
8932
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
7822
8933
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
7823
8934
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
@@ -7885,7 +8996,7 @@ async function run(options) {
|
|
|
7885
8996
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
7886
8997
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
7887
8998
|
fileSyncDirectories,
|
|
7888
|
-
homeDir:
|
|
8999
|
+
homeDir: homedir4(),
|
|
7889
9000
|
maxActiveSessions,
|
|
7890
9001
|
log: (entry) => (
|
|
7891
9002
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -8023,6 +9134,23 @@ async function run(options) {
|
|
|
8023
9134
|
}
|
|
8024
9135
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
8025
9136
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
9137
|
+
state.openaiUsageRearm = scheduleUsageReporting(state, {
|
|
9138
|
+
label: "OpenAI",
|
|
9139
|
+
resolved: resolveOpenAiUsageReportingMode(options.openaiUsageReporting, process.env),
|
|
9140
|
+
offFlagHint: "--openai-usage-reporting off",
|
|
9141
|
+
getTimer: () => state.openaiUsageTimer,
|
|
9142
|
+
setTimer: (timer) => {
|
|
9143
|
+
state.openaiUsageTimer = timer;
|
|
9144
|
+
},
|
|
9145
|
+
fetchUsage: () => getOpenAiUsage(state.port),
|
|
9146
|
+
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
9147
|
+
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
9148
|
+
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
9149
|
+
firstDelayMs: firstReportDelayMs,
|
|
9150
|
+
nextDelayMs: usageReportDelayMs,
|
|
9151
|
+
failureLogLevel: usageReportFailureLogLevel
|
|
9152
|
+
});
|
|
9153
|
+
scheduleResourceUsageReporting(state, options);
|
|
8026
9154
|
if (!interactive || state.json) {
|
|
8027
9155
|
log2(state, "Driving channel messages...");
|
|
8028
9156
|
}
|
|
@@ -8103,6 +9231,12 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8103
9231
|
).option(
|
|
8104
9232
|
"--claude-usage-reporting <mode>",
|
|
8105
9233
|
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
9234
|
+
).option(
|
|
9235
|
+
"--openai-usage-reporting <mode>",
|
|
9236
|
+
"Report OpenAI plan usage to Evident: auto | on | off (default: auto). Env: EVIDENT_OPENAI_USAGE_REPORTING"
|
|
9237
|
+
).option(
|
|
9238
|
+
"--no-resource-usage-reporting",
|
|
9239
|
+
"Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
|
|
8106
9240
|
).option(
|
|
8107
9241
|
"--enable-file-sync-to <dir>",
|
|
8108
9242
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
@@ -8135,6 +9269,10 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8135
9269
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
8136
9270
|
// (resolveClaudeUsageReportingMode).
|
|
8137
9271
|
claudeUsageReporting: options.claudeUsageReporting,
|
|
9272
|
+
openaiUsageReporting: options.openaiUsageReporting,
|
|
9273
|
+
// Raw value — resolution is single-sourced in run.ts's
|
|
9274
|
+
// resolveResourceUsageReportingEnabled.
|
|
9275
|
+
resourceUsageReporting: options.resourceUsageReporting,
|
|
8138
9276
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
8139
9277
|
// resolveFileSyncDirectories.
|
|
8140
9278
|
enableFileSyncTo: options.enableFileSyncTo,
|