@evident-ai/cli 3.3.1-dev.e98fa27 → 3.4.1-dev.11f0c53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -1
- package/dist/index.js +1540 -173
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -722,6 +722,14 @@ function toReportedWindow(window) {
|
|
|
722
722
|
if (!window) return null;
|
|
723
723
|
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
724
724
|
}
|
|
725
|
+
function toReportedOwner(snapshot) {
|
|
726
|
+
if (!snapshot.owner) return null;
|
|
727
|
+
return {
|
|
728
|
+
email: snapshot.owner.email,
|
|
729
|
+
organization_name: snapshot.owner.organizationName,
|
|
730
|
+
rate_limit_tier: snapshot.owner.rateLimitTier
|
|
731
|
+
};
|
|
732
|
+
}
|
|
725
733
|
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
726
734
|
try {
|
|
727
735
|
const apiUrl = getApiUrlConfig();
|
|
@@ -730,7 +738,71 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
730
738
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
731
739
|
body: JSON.stringify({
|
|
732
740
|
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
733
|
-
seven_day: toReportedWindow(snapshot.sevenDay)
|
|
741
|
+
seven_day: toReportedWindow(snapshot.sevenDay),
|
|
742
|
+
owner: toReportedOwner(snapshot)
|
|
743
|
+
}),
|
|
744
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
745
|
+
});
|
|
746
|
+
if (!response.ok) {
|
|
747
|
+
const serverMessage = await readErrorMessage(response);
|
|
748
|
+
return {
|
|
749
|
+
ok: false,
|
|
750
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
return { ok: true };
|
|
754
|
+
} catch (error2) {
|
|
755
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
function toReportedOpenAiWindow(window) {
|
|
759
|
+
if (!window) return null;
|
|
760
|
+
return {
|
|
761
|
+
utilization: window.utilization,
|
|
762
|
+
window_minutes: window.windowMinutes,
|
|
763
|
+
resets_at: window.resetsAt
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
767
|
+
try {
|
|
768
|
+
const apiUrl = getApiUrlConfig();
|
|
769
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/openai-usage`, {
|
|
770
|
+
method: "POST",
|
|
771
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
772
|
+
body: JSON.stringify({
|
|
773
|
+
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
774
|
+
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
775
|
+
has_credits: snapshot.hasCredits,
|
|
776
|
+
credits_unlimited: snapshot.creditsUnlimited
|
|
777
|
+
}),
|
|
778
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
779
|
+
});
|
|
780
|
+
if (!response.ok) {
|
|
781
|
+
const serverMessage = await readErrorMessage(response);
|
|
782
|
+
return {
|
|
783
|
+
ok: false,
|
|
784
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
return { ok: true };
|
|
788
|
+
} catch (error2) {
|
|
789
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
async function reportResourceUsage(agentId, authHeader, usage) {
|
|
793
|
+
try {
|
|
794
|
+
const apiUrl = getApiUrlConfig();
|
|
795
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/resource-usage`, {
|
|
796
|
+
method: "POST",
|
|
797
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
798
|
+
body: JSON.stringify({
|
|
799
|
+
cpu_percent: usage.cpuPercent,
|
|
800
|
+
cpu_count: usage.cpuCount,
|
|
801
|
+
memory_total_bytes: usage.memoryTotalBytes,
|
|
802
|
+
memory_available_bytes: usage.memoryAvailableBytes,
|
|
803
|
+
disk_total_bytes: usage.diskTotalBytes,
|
|
804
|
+
disk_free_bytes: usage.diskFreeBytes,
|
|
805
|
+
opencode_db_bytes: usage.opencodeDbBytes
|
|
734
806
|
}),
|
|
735
807
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
736
808
|
});
|
|
@@ -933,7 +1005,11 @@ import { readFileSync } from "fs";
|
|
|
933
1005
|
import { homedir } from "os";
|
|
934
1006
|
import { join } from "path";
|
|
935
1007
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
1008
|
+
var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
|
|
1009
|
+
var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
|
|
936
1010
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
1011
|
+
var cachedOwner = null;
|
|
1012
|
+
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
937
1013
|
function parseClaudeCliCredentials(raw) {
|
|
938
1014
|
let parsed;
|
|
939
1015
|
try {
|
|
@@ -967,7 +1043,7 @@ function readClaudeCliCredentials() {
|
|
|
967
1043
|
}
|
|
968
1044
|
}
|
|
969
1045
|
try {
|
|
970
|
-
const raw = readFileSync(join(homedir(),
|
|
1046
|
+
const raw = readFileSync(join(homedir(), ...CLAUDE_CREDENTIALS_SEGMENTS), "utf-8");
|
|
971
1047
|
return parseClaudeCliCredentials(raw);
|
|
972
1048
|
} catch (err) {
|
|
973
1049
|
const code = err.code;
|
|
@@ -1006,6 +1082,47 @@ function toWindow(value) {
|
|
|
1006
1082
|
}
|
|
1007
1083
|
return { utilization: window.utilization, resetsAt };
|
|
1008
1084
|
}
|
|
1085
|
+
function ownerLookupFailure(error2) {
|
|
1086
|
+
const name = error2?.name;
|
|
1087
|
+
return name === "TimeoutError" || name === "AbortError" ? "timed out" : "request failed";
|
|
1088
|
+
}
|
|
1089
|
+
async function getClaudeUsageOwner(accessToken) {
|
|
1090
|
+
if (cachedOwner?.accessToken === accessToken) {
|
|
1091
|
+
return { owner: cachedOwner.owner, ownerLookupError: null };
|
|
1092
|
+
}
|
|
1093
|
+
try {
|
|
1094
|
+
const response = await fetch(CLAUDE_PROFILE_URL, {
|
|
1095
|
+
headers: {
|
|
1096
|
+
Authorization: `Bearer ${accessToken}`,
|
|
1097
|
+
"Content-Type": "application/json",
|
|
1098
|
+
"anthropic-version": "2023-06-01"
|
|
1099
|
+
},
|
|
1100
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1101
|
+
});
|
|
1102
|
+
if (!response.ok) {
|
|
1103
|
+
return { owner: null, ownerLookupError: `HTTP ${response.status}` };
|
|
1104
|
+
}
|
|
1105
|
+
let body;
|
|
1106
|
+
try {
|
|
1107
|
+
body = await response.json();
|
|
1108
|
+
} catch (error2) {
|
|
1109
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1110
|
+
}
|
|
1111
|
+
const profile = body;
|
|
1112
|
+
if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
|
|
1113
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1114
|
+
}
|
|
1115
|
+
const owner = {
|
|
1116
|
+
email: profile.account.email,
|
|
1117
|
+
organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
|
|
1118
|
+
rateLimitTier: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
|
|
1119
|
+
};
|
|
1120
|
+
cachedOwner = { accessToken, owner };
|
|
1121
|
+
return { owner, ownerLookupError: null };
|
|
1122
|
+
} catch (error2) {
|
|
1123
|
+
return { owner: null, ownerLookupError: ownerLookupFailure(error2) };
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1009
1126
|
async function getClaudeUsage() {
|
|
1010
1127
|
const credentials2 = readClaudeCliCredentials();
|
|
1011
1128
|
if (!credentials2) {
|
|
@@ -1025,15 +1142,19 @@ async function getClaudeUsage() {
|
|
|
1025
1142
|
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
1026
1143
|
"Content-Type": "application/json",
|
|
1027
1144
|
"anthropic-version": "2023-06-01"
|
|
1028
|
-
}
|
|
1145
|
+
},
|
|
1146
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1029
1147
|
});
|
|
1030
1148
|
if (!res.ok) {
|
|
1031
1149
|
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
1032
1150
|
}
|
|
1033
1151
|
const body = await res.json();
|
|
1152
|
+
const { owner, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
|
|
1034
1153
|
return {
|
|
1035
1154
|
fiveHour: toWindow(body.five_hour),
|
|
1036
|
-
sevenDay: toWindow(body.seven_day)
|
|
1155
|
+
sevenDay: toWindow(body.seven_day),
|
|
1156
|
+
owner,
|
|
1157
|
+
ownerLookupError
|
|
1037
1158
|
};
|
|
1038
1159
|
}
|
|
1039
1160
|
|
|
@@ -1062,8 +1183,8 @@ async function claudeUsage() {
|
|
|
1062
1183
|
}
|
|
1063
1184
|
|
|
1064
1185
|
// src/commands/run.ts
|
|
1065
|
-
import { homedir as
|
|
1066
|
-
import { isAbsolute as isAbsolute2, join as
|
|
1186
|
+
import { homedir as homedir4 } from "os";
|
|
1187
|
+
import { isAbsolute as isAbsolute2, join as join7, parse, resolve as resolvePath } from "path";
|
|
1067
1188
|
import chalk6 from "chalk";
|
|
1068
1189
|
|
|
1069
1190
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1296,6 +1417,8 @@ var SEVERITY_BY_LEVEL = {
|
|
|
1296
1417
|
error: "error"
|
|
1297
1418
|
};
|
|
1298
1419
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1420
|
+
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1421
|
+
var MAX_METADATA_ENTRIES = 20;
|
|
1299
1422
|
var TRUNCATION_MARKER = "\u2026";
|
|
1300
1423
|
function redact(message) {
|
|
1301
1424
|
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
@@ -1304,6 +1427,24 @@ function truncate(message) {
|
|
|
1304
1427
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1305
1428
|
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
1306
1429
|
}
|
|
1430
|
+
function sanitiseMetadata(metadata) {
|
|
1431
|
+
if (!metadata) return {};
|
|
1432
|
+
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1433
|
+
if (Object.keys(metadata).length > entries.length) {
|
|
1434
|
+
console.error(
|
|
1435
|
+
`[runner-activity-telemetry] dropped ${Object.keys(metadata).length - entries.length} metadata entries`
|
|
1436
|
+
);
|
|
1437
|
+
}
|
|
1438
|
+
const sanitised = [];
|
|
1439
|
+
for (const [key, value] of entries) {
|
|
1440
|
+
if (typeof value === "string") {
|
|
1441
|
+
sanitised.push([key, truncate(redact(value).slice(0, MAX_METADATA_VALUE_LENGTH))]);
|
|
1442
|
+
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1443
|
+
sanitised.push([key, value]);
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
return Object.fromEntries(sanitised);
|
|
1447
|
+
}
|
|
1307
1448
|
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1308
1449
|
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1309
1450
|
var windowStartedAt = 0;
|
|
@@ -1342,7 +1483,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1342
1483
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1343
1484
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1344
1485
|
message,
|
|
1345
|
-
metadata: { source: "cli.run" },
|
|
1486
|
+
metadata: { ...sanitiseMetadata(entry.metadata), source: "cli.run" },
|
|
1346
1487
|
agentId: context.agentId
|
|
1347
1488
|
});
|
|
1348
1489
|
} catch (err) {
|
|
@@ -1352,6 +1493,150 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1352
1493
|
}
|
|
1353
1494
|
}
|
|
1354
1495
|
|
|
1496
|
+
// src/lib/opencode/session-db-recovery-report.ts
|
|
1497
|
+
import { readFileSync as readFileSync2, unlinkSync } from "fs";
|
|
1498
|
+
import { join as join2 } from "path";
|
|
1499
|
+
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1500
|
+
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1501
|
+
return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1502
|
+
}
|
|
1503
|
+
function drainSessionDbRecoveryReport({
|
|
1504
|
+
homeDir,
|
|
1505
|
+
env
|
|
1506
|
+
}) {
|
|
1507
|
+
const path = sessionDbRecoveryReportPath(homeDir, env);
|
|
1508
|
+
let content;
|
|
1509
|
+
try {
|
|
1510
|
+
content = readFileSync2(path, "utf8");
|
|
1511
|
+
} catch (error2) {
|
|
1512
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")
|
|
1513
|
+
return { path, records: [], skippedLines: 0, readError: null };
|
|
1514
|
+
const readError = error2 instanceof Error ? error2.message : String(error2);
|
|
1515
|
+
console.error(`[session-db-recovery-report] could not read ${path}: ${readError}`);
|
|
1516
|
+
return { path, records: [], skippedLines: 0, readError };
|
|
1517
|
+
}
|
|
1518
|
+
let skippedLines = 0;
|
|
1519
|
+
const records = content.split("\n").flatMap((line) => {
|
|
1520
|
+
if (!line.trim()) return [];
|
|
1521
|
+
try {
|
|
1522
|
+
const value = JSON.parse(line);
|
|
1523
|
+
if (!isSessionDbRecoveryRecord(value)) {
|
|
1524
|
+
skippedLines++;
|
|
1525
|
+
return [];
|
|
1526
|
+
}
|
|
1527
|
+
return [value];
|
|
1528
|
+
} catch (error2) {
|
|
1529
|
+
skippedLines++;
|
|
1530
|
+
console.error(
|
|
1531
|
+
`[session-db-recovery-report] skipped malformed record in ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1532
|
+
);
|
|
1533
|
+
return [];
|
|
1534
|
+
}
|
|
1535
|
+
});
|
|
1536
|
+
return { path, records, skippedLines, readError: null };
|
|
1537
|
+
}
|
|
1538
|
+
function acknowledgeSessionDbRecoveryReport(path) {
|
|
1539
|
+
try {
|
|
1540
|
+
unlinkSync(path);
|
|
1541
|
+
} catch (error2) {
|
|
1542
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1543
|
+
console.error(
|
|
1544
|
+
`[session-db-recovery-report] could not remove ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1545
|
+
);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
function buildSessionDbRecoveryActivity(record) {
|
|
1549
|
+
const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
|
|
1550
|
+
if (!level) return null;
|
|
1551
|
+
const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
|
|
1552
|
+
switch (record.outcome) {
|
|
1553
|
+
case "fresh_session_db":
|
|
1554
|
+
return {
|
|
1555
|
+
level,
|
|
1556
|
+
metadata: withoutContractFields(record),
|
|
1557
|
+
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.`
|
|
1558
|
+
};
|
|
1559
|
+
case "restore_retried":
|
|
1560
|
+
return {
|
|
1561
|
+
level,
|
|
1562
|
+
metadata: withoutContractFields(record),
|
|
1563
|
+
message: "The initial session database restore failed; the runner will retry it. Monitor the runner backup for another restore failure."
|
|
1564
|
+
};
|
|
1565
|
+
case "replica_recovered":
|
|
1566
|
+
if (record.reason === "quarantine")
|
|
1567
|
+
return {
|
|
1568
|
+
level,
|
|
1569
|
+
metadata: withoutContractFields(record),
|
|
1570
|
+
message: `Session database recovery quarantined ${record.quarantined_objects ?? "unknown"} objects (${record.quarantined_bytes ?? "unknown"} bytes); ${record.quarantine_failed_objects ?? "unknown"} moves failed. Review the preserved backup at ${record.quarantine_destination ?? "an unknown destination"} before deleting it.`
|
|
1571
|
+
};
|
|
1572
|
+
if (record.reason === "prune")
|
|
1573
|
+
return {
|
|
1574
|
+
level,
|
|
1575
|
+
metadata: withoutContractFields(record),
|
|
1576
|
+
message: "Session database recovery discarded a damaged newest backup and retried. Sessions recorded after the previous backup point may be unavailable. Review the runner backup for another restore failure."
|
|
1577
|
+
};
|
|
1578
|
+
if (record.reason === "clear")
|
|
1579
|
+
return {
|
|
1580
|
+
level,
|
|
1581
|
+
metadata: withoutContractFields(record),
|
|
1582
|
+
message: "Session database recovery deleted the damaged backup and prior session history is unavailable. Review the runner backup configuration before relying on restored session history."
|
|
1583
|
+
};
|
|
1584
|
+
return null;
|
|
1585
|
+
case "history_rolled_back":
|
|
1586
|
+
return {
|
|
1587
|
+
level,
|
|
1588
|
+
metadata: withoutContractFields(record),
|
|
1589
|
+
message: `Session history was rolled back to verified restore point ${record.verified_restore_point ?? "unknown"}; everything after it is unavailable. Review the runner backup for another restore failure.`
|
|
1590
|
+
};
|
|
1591
|
+
case "restore_misconfigured":
|
|
1592
|
+
return {
|
|
1593
|
+
level,
|
|
1594
|
+
metadata: withoutContractFields(record),
|
|
1595
|
+
message: "This runner started with a fresh session database because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions."
|
|
1596
|
+
};
|
|
1597
|
+
case "session_db_boot_refused":
|
|
1598
|
+
return {
|
|
1599
|
+
level,
|
|
1600
|
+
metadata: withoutContractFields(record),
|
|
1601
|
+
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1602
|
+
};
|
|
1603
|
+
default:
|
|
1604
|
+
return null;
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
function withoutContractFields(record) {
|
|
1608
|
+
const { v: _v, event: _event, ...metadata } = record;
|
|
1609
|
+
return metadata;
|
|
1610
|
+
}
|
|
1611
|
+
var OUTCOMES = /* @__PURE__ */ new Set([
|
|
1612
|
+
"replica_recovered",
|
|
1613
|
+
"restore_retried",
|
|
1614
|
+
"fresh_session_db",
|
|
1615
|
+
"history_rolled_back",
|
|
1616
|
+
"restore_misconfigured",
|
|
1617
|
+
"session_db_boot_refused"
|
|
1618
|
+
]);
|
|
1619
|
+
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1620
|
+
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
1621
|
+
var NUMBER_FIELDS = [
|
|
1622
|
+
"litestream_exit_code",
|
|
1623
|
+
"attempt",
|
|
1624
|
+
"replica_objects",
|
|
1625
|
+
"replica_bytes",
|
|
1626
|
+
"quarantined_objects",
|
|
1627
|
+
"quarantine_failed_objects",
|
|
1628
|
+
"quarantined_bytes",
|
|
1629
|
+
"restore_points_tried"
|
|
1630
|
+
];
|
|
1631
|
+
var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"];
|
|
1632
|
+
function isSessionDbRecoveryRecord(value) {
|
|
1633
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1634
|
+
const record = value;
|
|
1635
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1636
|
+
(field) => record[field] === null || typeof record[field] === "string"
|
|
1637
|
+
);
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1355
1640
|
// src/lib/opencode/health.ts
|
|
1356
1641
|
async function checkOpenCodeHealth(port) {
|
|
1357
1642
|
try {
|
|
@@ -1396,6 +1681,62 @@ function buildOpenCodeVersionWarning(version2) {
|
|
|
1396
1681
|
|
|
1397
1682
|
// src/lib/opencode/process.ts
|
|
1398
1683
|
import { execSync, spawn } from "child_process";
|
|
1684
|
+
|
|
1685
|
+
// src/lib/process-stop.ts
|
|
1686
|
+
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
1687
|
+
if (!child.pid) {
|
|
1688
|
+
return { outcome: "not-running", code: child.exitCode, signal: child.signalCode };
|
|
1689
|
+
}
|
|
1690
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
1691
|
+
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
1692
|
+
}
|
|
1693
|
+
return new Promise((resolve3, reject) => {
|
|
1694
|
+
let forced = false;
|
|
1695
|
+
let settled = false;
|
|
1696
|
+
const timer = setTimeout(() => {
|
|
1697
|
+
forced = true;
|
|
1698
|
+
try {
|
|
1699
|
+
sendKill();
|
|
1700
|
+
} catch (error2) {
|
|
1701
|
+
if (error2.code === "ESRCH") {
|
|
1702
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
1703
|
+
} else {
|
|
1704
|
+
fail(error2);
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
}, timeoutMs);
|
|
1708
|
+
const finish = (result) => {
|
|
1709
|
+
if (settled) return;
|
|
1710
|
+
settled = true;
|
|
1711
|
+
clearTimeout(timer);
|
|
1712
|
+
child.removeListener("exit", onExit);
|
|
1713
|
+
resolve3(result);
|
|
1714
|
+
};
|
|
1715
|
+
const fail = (error2) => {
|
|
1716
|
+
if (settled) return;
|
|
1717
|
+
settled = true;
|
|
1718
|
+
clearTimeout(timer);
|
|
1719
|
+
child.removeListener("exit", onExit);
|
|
1720
|
+
reject(error2);
|
|
1721
|
+
};
|
|
1722
|
+
const onExit = (code, signal) => {
|
|
1723
|
+
finish({ outcome: forced ? "killed" : "exited", code, signal });
|
|
1724
|
+
};
|
|
1725
|
+
child.once("exit", onExit);
|
|
1726
|
+
try {
|
|
1727
|
+
sendTerm();
|
|
1728
|
+
} catch (error2) {
|
|
1729
|
+
if (error2.code === "ESRCH") {
|
|
1730
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
1731
|
+
} else {
|
|
1732
|
+
fail(error2);
|
|
1733
|
+
}
|
|
1734
|
+
return;
|
|
1735
|
+
}
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1739
|
+
// src/lib/opencode/process.ts
|
|
1399
1740
|
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
1400
1741
|
function getProcessCwd(pid) {
|
|
1401
1742
|
const platform = process.platform;
|
|
@@ -1567,23 +1908,20 @@ async function startOpenCode(port) {
|
|
|
1567
1908
|
});
|
|
1568
1909
|
return child;
|
|
1569
1910
|
}
|
|
1570
|
-
function
|
|
1571
|
-
|
|
1572
|
-
return;
|
|
1573
|
-
}
|
|
1574
|
-
try {
|
|
1911
|
+
function stopOpenCodeAndWait(opencodeProcess, timeoutMs) {
|
|
1912
|
+
const sendSignal = (signal) => {
|
|
1575
1913
|
if (process.platform === "win32") {
|
|
1576
|
-
opencodeProcess.kill(
|
|
1914
|
+
opencodeProcess.kill(signal);
|
|
1577
1915
|
} else {
|
|
1578
|
-
process.kill(-opencodeProcess.pid,
|
|
1916
|
+
process.kill(-opencodeProcess.pid, signal);
|
|
1579
1917
|
}
|
|
1580
|
-
}
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1918
|
+
};
|
|
1919
|
+
return stopProcessAndWait(
|
|
1920
|
+
opencodeProcess,
|
|
1921
|
+
timeoutMs,
|
|
1922
|
+
() => sendSignal("SIGTERM"),
|
|
1923
|
+
() => sendSignal("SIGKILL")
|
|
1924
|
+
);
|
|
1587
1925
|
}
|
|
1588
1926
|
|
|
1589
1927
|
// src/lib/opencode/install.ts
|
|
@@ -1700,13 +2038,22 @@ function buildNoProviderWarning(hasProvider) {
|
|
|
1700
2038
|
return "Warning: opencode has no authenticated model provider configured, so it won't be able to answer prompts. Run `opencode auth login` to set one up (see https://opencode.ai for details).";
|
|
1701
2039
|
}
|
|
1702
2040
|
|
|
2041
|
+
// src/lib/http-timeout.ts
|
|
2042
|
+
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2043
|
+
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
2044
|
+
return ((input, init) => fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) }));
|
|
2045
|
+
}
|
|
2046
|
+
|
|
1703
2047
|
// src/lib/opencode/session.ts
|
|
2048
|
+
function timedFetch(input, init) {
|
|
2049
|
+
return withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(input, init);
|
|
2050
|
+
}
|
|
1704
2051
|
function opencodeBase(port) {
|
|
1705
2052
|
return `http://127.0.0.1:${port}`;
|
|
1706
2053
|
}
|
|
1707
2054
|
async function getOpenCodeDirectory(port) {
|
|
1708
2055
|
try {
|
|
1709
|
-
const res = await
|
|
2056
|
+
const res = await timedFetch(`${opencodeBase(port)}/path`);
|
|
1710
2057
|
if (!res.ok) return null;
|
|
1711
2058
|
const body = await res.json();
|
|
1712
2059
|
const dir = typeof body.directory === "string" && body.directory || typeof body.worktree === "string" && body.worktree || typeof body.path?.cwd === "string" && body.path.cwd || typeof body.path?.directory === "string" && body.path.directory || null;
|
|
@@ -1757,7 +2104,7 @@ function isAssistantInFlight(m) {
|
|
|
1757
2104
|
}
|
|
1758
2105
|
async function getSessionMessages(port, sessionId) {
|
|
1759
2106
|
try {
|
|
1760
|
-
const res = await
|
|
2107
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1761
2108
|
if (!res.ok) return null;
|
|
1762
2109
|
const body = await res.json();
|
|
1763
2110
|
return Array.isArray(body) ? body : null;
|
|
@@ -1787,7 +2134,7 @@ function sessionLastActivityMs(session) {
|
|
|
1787
2134
|
}
|
|
1788
2135
|
async function listSessions(port) {
|
|
1789
2136
|
try {
|
|
1790
|
-
const res = await
|
|
2137
|
+
const res = await timedFetch(`${opencodeBase(port)}/session`);
|
|
1791
2138
|
if (!res.ok) return null;
|
|
1792
2139
|
const body = await res.json();
|
|
1793
2140
|
return Array.isArray(body) ? body : null;
|
|
@@ -1797,7 +2144,7 @@ async function listSessions(port) {
|
|
|
1797
2144
|
}
|
|
1798
2145
|
async function deleteSession(port, id) {
|
|
1799
2146
|
try {
|
|
1800
|
-
const res = await
|
|
2147
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1801
2148
|
return res.status >= 200 && res.status < 300;
|
|
1802
2149
|
} catch {
|
|
1803
2150
|
return false;
|
|
@@ -1805,7 +2152,7 @@ async function deleteSession(port, id) {
|
|
|
1805
2152
|
}
|
|
1806
2153
|
async function sessionExists(port, id) {
|
|
1807
2154
|
try {
|
|
1808
|
-
const res = await
|
|
2155
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${id}`);
|
|
1809
2156
|
if (res.status >= 200 && res.status < 300) return true;
|
|
1810
2157
|
if (res.status === 404) return false;
|
|
1811
2158
|
return null;
|
|
@@ -1815,7 +2162,7 @@ async function sessionExists(port, id) {
|
|
|
1815
2162
|
}
|
|
1816
2163
|
async function getSessionStatuses(port) {
|
|
1817
2164
|
try {
|
|
1818
|
-
const res = await
|
|
2165
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/status`);
|
|
1819
2166
|
if (!res.ok) {
|
|
1820
2167
|
console.error(
|
|
1821
2168
|
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
@@ -1848,7 +2195,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1848
2195
|
if (directory && directory.trim()) {
|
|
1849
2196
|
url.searchParams.set("directory", directory.trim());
|
|
1850
2197
|
}
|
|
1851
|
-
const response = await
|
|
2198
|
+
const response = await timedFetch(url, {
|
|
1852
2199
|
method: "POST",
|
|
1853
2200
|
headers: { "Content-Type": "application/json" },
|
|
1854
2201
|
body: JSON.stringify({})
|
|
@@ -1861,8 +2208,9 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1861
2208
|
return data.id;
|
|
1862
2209
|
}
|
|
1863
2210
|
async function getModelAttachmentCapability(port, model) {
|
|
2211
|
+
const { model: baseModel } = splitModelVariant(model);
|
|
1864
2212
|
try {
|
|
1865
|
-
const res = await
|
|
2213
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1866
2214
|
if (!res.ok) {
|
|
1867
2215
|
console.error(
|
|
1868
2216
|
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -1877,9 +2225,9 @@ async function getModelAttachmentCapability(port, model) {
|
|
|
1877
2225
|
);
|
|
1878
2226
|
return null;
|
|
1879
2227
|
}
|
|
1880
|
-
const slash =
|
|
1881
|
-
const providerId = slash > 0 ?
|
|
1882
|
-
let modelId = slash > 0 ?
|
|
2228
|
+
const slash = baseModel ? baseModel.indexOf("/") : -1;
|
|
2229
|
+
const providerId = slash > 0 ? baseModel.slice(0, slash) : void 0;
|
|
2230
|
+
let modelId = slash > 0 ? baseModel.slice(slash + 1) : void 0;
|
|
1883
2231
|
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
1884
2232
|
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
1885
2233
|
if (!provider && !providerId) {
|
|
@@ -1959,6 +2307,29 @@ async function buildFileParts(attachments, capable) {
|
|
|
1959
2307
|
}
|
|
1960
2308
|
return { parts, outcomes, capabilityUnknown };
|
|
1961
2309
|
}
|
|
2310
|
+
function splitModelVariant(raw) {
|
|
2311
|
+
const value = raw?.trim();
|
|
2312
|
+
if (!value) return {};
|
|
2313
|
+
const hashIndex = value.indexOf("#");
|
|
2314
|
+
if (hashIndex === -1) return { model: value };
|
|
2315
|
+
const model = value.slice(0, hashIndex).trim() || void 0;
|
|
2316
|
+
const variant = value.slice(hashIndex + 1).trim() || void 0;
|
|
2317
|
+
return { model, variant };
|
|
2318
|
+
}
|
|
2319
|
+
function applyModelOptions(body, options) {
|
|
2320
|
+
if (options?.agent) body.agent = options.agent;
|
|
2321
|
+
const { model, variant } = splitModelVariant(options?.model);
|
|
2322
|
+
if (model) {
|
|
2323
|
+
const slashIndex = model.indexOf("/");
|
|
2324
|
+
if (slashIndex !== -1) {
|
|
2325
|
+
body.model = {
|
|
2326
|
+
providerID: model.substring(0, slashIndex),
|
|
2327
|
+
modelID: model.substring(slashIndex + 1)
|
|
2328
|
+
};
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
if (variant) body.variant = variant;
|
|
2332
|
+
}
|
|
1962
2333
|
function messageText(m) {
|
|
1963
2334
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
1964
2335
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
@@ -1983,26 +2354,18 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1983
2354
|
const body = {
|
|
1984
2355
|
parts
|
|
1985
2356
|
};
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
}
|
|
1989
|
-
if (options?.model) {
|
|
1990
|
-
const slashIndex = options.model.indexOf("/");
|
|
1991
|
-
if (slashIndex !== -1) {
|
|
1992
|
-
body.model = {
|
|
1993
|
-
providerID: options.model.substring(0, slashIndex),
|
|
1994
|
-
modelID: options.model.substring(slashIndex + 1)
|
|
1995
|
-
};
|
|
1996
|
-
}
|
|
1997
|
-
}
|
|
1998
|
-
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
2357
|
+
applyModelOptions(body, options);
|
|
2358
|
+
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1999
2359
|
method: "POST",
|
|
2000
2360
|
headers: { "Content-Type": "application/json" },
|
|
2001
2361
|
body: JSON.stringify(body)
|
|
2002
2362
|
});
|
|
2003
2363
|
if (res.status < 200 || res.status >= 300) {
|
|
2004
2364
|
const text = await res.text().catch(() => "");
|
|
2005
|
-
|
|
2365
|
+
const { variant } = splitModelVariant(options?.model);
|
|
2366
|
+
throw new Error(
|
|
2367
|
+
`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}${variant ? ` (variant: ${variant})` : ""}`
|
|
2368
|
+
);
|
|
2006
2369
|
}
|
|
2007
2370
|
const READ_BACK_ATTEMPTS = 5;
|
|
2008
2371
|
const READ_BACK_DELAY_MS = 150;
|
|
@@ -2157,7 +2520,7 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
2157
2520
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
2158
2521
|
}
|
|
2159
2522
|
function isB2AbandonmentConfirmed(params) {
|
|
2160
|
-
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
2523
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false && params.rootOngoing === false;
|
|
2161
2524
|
}
|
|
2162
2525
|
function isAmbiguousTerminalFinish(m) {
|
|
2163
2526
|
if (completedOf(m) == null) return false;
|
|
@@ -2170,7 +2533,7 @@ function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
|
|
|
2170
2533
|
return isAmbiguousTerminalFinish(reply);
|
|
2171
2534
|
}
|
|
2172
2535
|
function isAmbiguousFinishResolved(params) {
|
|
2173
|
-
return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
|
|
2536
|
+
return params.sessionOngoing === false || params.sessionOngoing !== true && params.pinnedForMs >= params.maxPinnedMs;
|
|
2174
2537
|
}
|
|
2175
2538
|
function messageError(messages, userMessageId) {
|
|
2176
2539
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2244,7 +2607,7 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
2244
2607
|
}
|
|
2245
2608
|
async function hasAnyConfiguredProvider(port) {
|
|
2246
2609
|
try {
|
|
2247
|
-
const res = await
|
|
2610
|
+
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2248
2611
|
if (!res.ok) {
|
|
2249
2612
|
console.error(
|
|
2250
2613
|
`[hasAnyConfiguredProvider] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
@@ -2380,10 +2743,10 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2380
2743
|
|
|
2381
2744
|
// src/lib/opencode/session-db-size.ts
|
|
2382
2745
|
import { statSync as statSync2 } from "fs";
|
|
2383
|
-
import { join as
|
|
2746
|
+
import { join as join3 } from "path";
|
|
2384
2747
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2385
2748
|
function statSessionDbBytes(homeDir) {
|
|
2386
|
-
const dbPath =
|
|
2749
|
+
const dbPath = join3(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2387
2750
|
try {
|
|
2388
2751
|
return statSync2(dbPath).size;
|
|
2389
2752
|
} catch (err) {
|
|
@@ -2952,44 +3315,421 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
2952
3315
|
}
|
|
2953
3316
|
}
|
|
2954
3317
|
|
|
3318
|
+
// src/lib/replication.ts
|
|
3319
|
+
import { spawn as spawn2 } from "child_process";
|
|
3320
|
+
function startSessionDbReplication(configPath) {
|
|
3321
|
+
return spawn2("litestream", ["replicate", "-config", configPath], {
|
|
3322
|
+
stdio: "inherit"
|
|
3323
|
+
});
|
|
3324
|
+
}
|
|
3325
|
+
async function stopSessionDbReplication(child, timeoutMs) {
|
|
3326
|
+
return stopProcessAndWait(
|
|
3327
|
+
child,
|
|
3328
|
+
timeoutMs,
|
|
3329
|
+
() => child.kill("SIGTERM"),
|
|
3330
|
+
() => child.kill("SIGKILL")
|
|
3331
|
+
);
|
|
3332
|
+
}
|
|
3333
|
+
|
|
3334
|
+
// src/lib/openai-usage.ts
|
|
3335
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
3336
|
+
import { homedir as homedir2 } from "os";
|
|
3337
|
+
import { join as join4 } from "path";
|
|
3338
|
+
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3339
|
+
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3340
|
+
var OpenAiUsageError = class extends Error {
|
|
3341
|
+
constructor(message, reason) {
|
|
3342
|
+
super(message);
|
|
3343
|
+
this.reason = reason;
|
|
3344
|
+
}
|
|
3345
|
+
};
|
|
3346
|
+
function isLocalCredentialProblem2(err) {
|
|
3347
|
+
return err instanceof OpenAiUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
|
|
3348
|
+
}
|
|
3349
|
+
function readOpenCodeChatGptCredentials() {
|
|
3350
|
+
try {
|
|
3351
|
+
const raw = readFileSync3(join4(homedir2(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3352
|
+
let parsed;
|
|
3353
|
+
try {
|
|
3354
|
+
parsed = JSON.parse(raw);
|
|
3355
|
+
} catch {
|
|
3356
|
+
return null;
|
|
3357
|
+
}
|
|
3358
|
+
const entry = parsed.openai;
|
|
3359
|
+
if (entry?.type !== "oauth" || typeof entry.access !== "string" || !entry.access || typeof entry.expires !== "number") {
|
|
3360
|
+
return null;
|
|
3361
|
+
}
|
|
3362
|
+
return { accessToken: entry.access, expiresAt: entry.expires };
|
|
3363
|
+
} catch (err) {
|
|
3364
|
+
const code = err.code;
|
|
3365
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
3366
|
+
console.warn(
|
|
3367
|
+
`readOpenCodeChatGptCredentials: reading auth.json failed (${code ?? "unknown"})`
|
|
3368
|
+
);
|
|
3369
|
+
}
|
|
3370
|
+
return null;
|
|
3371
|
+
}
|
|
3372
|
+
}
|
|
3373
|
+
function toWindow2(headers, name) {
|
|
3374
|
+
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
3375
|
+
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
3376
|
+
if (utilizationHeader == null || utilizationHeader === "" || windowMinutesHeader == null) {
|
|
3377
|
+
return null;
|
|
3378
|
+
}
|
|
3379
|
+
const utilization = Number(utilizationHeader);
|
|
3380
|
+
const windowMinutes = Number(windowMinutesHeader);
|
|
3381
|
+
if (!Number.isFinite(utilization) || !Number.isFinite(windowMinutes) || windowMinutes <= 0) {
|
|
3382
|
+
return null;
|
|
3383
|
+
}
|
|
3384
|
+
const resetAtHeader = headers.get(`x-codex-${name}-reset-at`);
|
|
3385
|
+
const resetSeconds = resetAtHeader == null || resetAtHeader === "" ? NaN : Number(resetAtHeader);
|
|
3386
|
+
const resetsAt = Number.isFinite(resetSeconds) ? new Date(resetSeconds * 1e3).toISOString() : null;
|
|
3387
|
+
return { utilization: Math.min(100, Math.max(0, utilization)), windowMinutes, resetsAt };
|
|
3388
|
+
}
|
|
3389
|
+
function parseCodexUsageHeaders(headers) {
|
|
3390
|
+
return {
|
|
3391
|
+
primary: toWindow2(headers, "primary"),
|
|
3392
|
+
secondary: toWindow2(headers, "secondary"),
|
|
3393
|
+
hasCredits: headers.has("x-codex-credits-has-credits") ? headers.get("x-codex-credits-has-credits")?.toLowerCase() === "true" : null,
|
|
3394
|
+
creditsUnlimited: headers.has("x-codex-credits-unlimited") ? headers.get("x-codex-credits-unlimited")?.toLowerCase() === "true" : null
|
|
3395
|
+
};
|
|
3396
|
+
}
|
|
3397
|
+
function normalizeProbeModel(model) {
|
|
3398
|
+
return model.endsWith("-fast") ? model.slice(0, -"-fast".length) : model;
|
|
3399
|
+
}
|
|
3400
|
+
async function resolveProbeModels(port) {
|
|
3401
|
+
try {
|
|
3402
|
+
const res = await withRequestTimeout(
|
|
3403
|
+
fetch,
|
|
3404
|
+
REQUEST_TIMEOUT_MS
|
|
3405
|
+
)(`${opencodeBase(port)}/config/providers`);
|
|
3406
|
+
if (!res.ok) {
|
|
3407
|
+
console.error(
|
|
3408
|
+
`[resolveProbeModels] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
3409
|
+
);
|
|
3410
|
+
return [];
|
|
3411
|
+
}
|
|
3412
|
+
const body = await res.json();
|
|
3413
|
+
const provider = body?.providers?.find((candidate) => candidate?.id === "openai");
|
|
3414
|
+
if (!provider || !provider.models || typeof provider.models !== "object") return [];
|
|
3415
|
+
const candidates = [
|
|
3416
|
+
...typeof body?.default?.openai === "string" ? [body.default.openai] : [],
|
|
3417
|
+
...Object.keys(provider.models)
|
|
3418
|
+
].map(normalizeProbeModel);
|
|
3419
|
+
return [...new Set(candidates)].slice(0, 4);
|
|
3420
|
+
} catch (err) {
|
|
3421
|
+
console.error(
|
|
3422
|
+
`[resolveProbeModels] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
3423
|
+
);
|
|
3424
|
+
return [];
|
|
3425
|
+
}
|
|
3426
|
+
}
|
|
3427
|
+
function hasPrimaryHeaders(headers) {
|
|
3428
|
+
return [
|
|
3429
|
+
"x-codex-primary-used-percent",
|
|
3430
|
+
"x-codex-primary-window-minutes",
|
|
3431
|
+
"x-codex-primary-reset-at"
|
|
3432
|
+
].some((name) => headers.has(name));
|
|
3433
|
+
}
|
|
3434
|
+
async function getOpenAiUsage(port) {
|
|
3435
|
+
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3436
|
+
if (!credentials2) {
|
|
3437
|
+
throw new OpenAiUsageError(
|
|
3438
|
+
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3439
|
+
"no_credentials"
|
|
3440
|
+
);
|
|
3441
|
+
}
|
|
3442
|
+
if (credentials2.expiresAt < Date.now()) {
|
|
3443
|
+
throw new OpenAiUsageError(
|
|
3444
|
+
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3445
|
+
"credentials_expired"
|
|
3446
|
+
);
|
|
3447
|
+
}
|
|
3448
|
+
const models = await resolveProbeModels(port);
|
|
3449
|
+
if (models.length === 0) {
|
|
3450
|
+
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
3451
|
+
}
|
|
3452
|
+
let lastStatus;
|
|
3453
|
+
for (const model of models) {
|
|
3454
|
+
let res;
|
|
3455
|
+
try {
|
|
3456
|
+
res = await withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(CODEX_RESPONSES_URL, {
|
|
3457
|
+
method: "POST",
|
|
3458
|
+
headers: {
|
|
3459
|
+
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
3460
|
+
"Content-Type": "application/json"
|
|
3461
|
+
},
|
|
3462
|
+
body: JSON.stringify({ model, store: false, stream: true })
|
|
3463
|
+
});
|
|
3464
|
+
} catch (err) {
|
|
3465
|
+
throw new OpenAiUsageError(
|
|
3466
|
+
`OpenAI usage probe request failed: ${err instanceof Error ? err.message : String(err)}.`,
|
|
3467
|
+
"request_failed"
|
|
3468
|
+
);
|
|
3469
|
+
}
|
|
3470
|
+
try {
|
|
3471
|
+
lastStatus = res.status;
|
|
3472
|
+
if (hasPrimaryHeaders(res.headers)) {
|
|
3473
|
+
const usage = parseCodexUsageHeaders(res.headers);
|
|
3474
|
+
if (!usage.primary && !usage.secondary) {
|
|
3475
|
+
throw new OpenAiUsageError(
|
|
3476
|
+
`OpenAI usage probe returned no usable window (HTTP ${res.status}).`,
|
|
3477
|
+
"no_usable_window"
|
|
3478
|
+
);
|
|
3479
|
+
}
|
|
3480
|
+
return usage;
|
|
3481
|
+
}
|
|
3482
|
+
if (res.status === 401) {
|
|
3483
|
+
throw new OpenAiUsageError(
|
|
3484
|
+
"ChatGPT credentials have expired (HTTP 401).",
|
|
3485
|
+
"credentials_expired"
|
|
3486
|
+
);
|
|
3487
|
+
}
|
|
3488
|
+
if (res.status === 403 || res.status === 429) {
|
|
3489
|
+
throw new OpenAiUsageError(
|
|
3490
|
+
`OpenAI usage probe was blocked (HTTP ${res.status}).`,
|
|
3491
|
+
"probe_blocked"
|
|
3492
|
+
);
|
|
3493
|
+
}
|
|
3494
|
+
} finally {
|
|
3495
|
+
await res.body?.cancel().catch(() => {
|
|
3496
|
+
});
|
|
3497
|
+
}
|
|
3498
|
+
}
|
|
3499
|
+
throw new OpenAiUsageError(
|
|
3500
|
+
`OpenAI usage probe failed: HTTP ${lastStatus ?? "unknown"}.`,
|
|
3501
|
+
"request_failed"
|
|
3502
|
+
);
|
|
3503
|
+
}
|
|
3504
|
+
|
|
3505
|
+
// src/lib/reporting-schedule.ts
|
|
3506
|
+
function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
3507
|
+
const jitterRangeMs = baseMs * jitterFraction;
|
|
3508
|
+
return baseMs - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
3509
|
+
}
|
|
3510
|
+
function firstReportDelayMs(random = Math.random) {
|
|
3511
|
+
return 5e3 + random() * 1e4;
|
|
3512
|
+
}
|
|
3513
|
+
var VALID_USAGE_REPORTING_MODES = ["auto", "on", "off"];
|
|
3514
|
+
function resolveUsageReportingMode(flagValue, env, names) {
|
|
3515
|
+
const raw = flagValue ?? env[names.envVar];
|
|
3516
|
+
if (raw === void 0 || raw === "") return { mode: "auto", warnings: [] };
|
|
3517
|
+
const normalized = raw.trim().toLowerCase();
|
|
3518
|
+
if (VALID_USAGE_REPORTING_MODES.includes(normalized)) {
|
|
3519
|
+
return { mode: normalized, warnings: [] };
|
|
3520
|
+
}
|
|
3521
|
+
const source = flagValue !== void 0 ? names.flagName : names.envVar;
|
|
3522
|
+
return {
|
|
3523
|
+
mode: "auto",
|
|
3524
|
+
warnings: [
|
|
3525
|
+
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_USAGE_REPORTING_MODES.join(", ")}; using auto`
|
|
3526
|
+
]
|
|
3527
|
+
};
|
|
3528
|
+
}
|
|
3529
|
+
var BASE_USAGE_REPORT_DELAY_MS = 10 * 6e4;
|
|
3530
|
+
var USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
3531
|
+
function usageReportDelayMs(random = Math.random) {
|
|
3532
|
+
return jitteredDelayMs(BASE_USAGE_REPORT_DELAY_MS, USAGE_REPORT_DELAY_JITTER_FRACTION, random);
|
|
3533
|
+
}
|
|
3534
|
+
var USAGE_REPORT_FAILURE_REESCALATION_TICKS = 6;
|
|
3535
|
+
function usageReportFailureLogLevel(consecutiveFailures) {
|
|
3536
|
+
return reportFailureLogLevel(consecutiveFailures, USAGE_REPORT_FAILURE_REESCALATION_TICKS);
|
|
3537
|
+
}
|
|
3538
|
+
function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
|
|
3539
|
+
return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
|
|
3540
|
+
}
|
|
3541
|
+
function failureStreakSuffix(consecutiveFailures) {
|
|
3542
|
+
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
3543
|
+
}
|
|
3544
|
+
|
|
2955
3545
|
// src/lib/claude-usage-reporting.ts
|
|
2956
|
-
var VALID_MODES = ["auto", "on", "off"];
|
|
2957
3546
|
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
2958
|
-
|
|
3547
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
3548
|
+
flagName: "--claude-usage-reporting",
|
|
3549
|
+
envVar: "EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
3550
|
+
});
|
|
3551
|
+
}
|
|
3552
|
+
function nextReportDelayMs(random = Math.random) {
|
|
3553
|
+
return usageReportDelayMs(random);
|
|
3554
|
+
}
|
|
3555
|
+
var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
|
|
3556
|
+
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
3557
|
+
return usageReportFailureLogLevel(consecutiveFailures);
|
|
3558
|
+
}
|
|
3559
|
+
|
|
3560
|
+
// src/lib/openai-usage-reporting.ts
|
|
3561
|
+
function resolveOpenAiUsageReportingMode(flagValue, env) {
|
|
3562
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
3563
|
+
flagName: "--openai-usage-reporting",
|
|
3564
|
+
envVar: "EVIDENT_OPENAI_USAGE_REPORTING"
|
|
3565
|
+
});
|
|
3566
|
+
}
|
|
3567
|
+
|
|
3568
|
+
// src/lib/resource-usage-reporting.ts
|
|
3569
|
+
var ENABLED_VALUES = /* @__PURE__ */ new Set(["on", "true", "1"]);
|
|
3570
|
+
var DISABLED_VALUES = /* @__PURE__ */ new Set(["off", "false", "0"]);
|
|
3571
|
+
function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
3572
|
+
if (flagValue === false) {
|
|
3573
|
+
return { enabled: false, warnings: [] };
|
|
3574
|
+
}
|
|
3575
|
+
const raw = env.EVIDENT_RESOURCE_USAGE_REPORTING;
|
|
2959
3576
|
if (raw === void 0 || raw === "") {
|
|
2960
|
-
return {
|
|
3577
|
+
return { enabled: true, warnings: [] };
|
|
2961
3578
|
}
|
|
2962
3579
|
const normalized = raw.trim().toLowerCase();
|
|
2963
|
-
if (
|
|
2964
|
-
return {
|
|
3580
|
+
if (DISABLED_VALUES.has(normalized)) {
|
|
3581
|
+
return { enabled: false, warnings: [] };
|
|
3582
|
+
}
|
|
3583
|
+
if (ENABLED_VALUES.has(normalized)) {
|
|
3584
|
+
return { enabled: true, warnings: [] };
|
|
3585
|
+
}
|
|
3586
|
+
return {
|
|
3587
|
+
enabled: true,
|
|
3588
|
+
warnings: [
|
|
3589
|
+
`Ignoring invalid EVIDENT_RESOURCE_USAGE_REPORTING "${raw}": expected on or off; leaving reporting on`
|
|
3590
|
+
]
|
|
3591
|
+
};
|
|
3592
|
+
}
|
|
3593
|
+
|
|
3594
|
+
// src/lib/resource-usage.ts
|
|
3595
|
+
import { cpus, totalmem, freemem } from "os";
|
|
3596
|
+
import { statfsSync as statfsSync2 } from "fs";
|
|
3597
|
+
|
|
3598
|
+
// src/lib/ecs-task-metadata.ts
|
|
3599
|
+
var ECS_METADATA_TIMEOUT_MS = 2e3;
|
|
3600
|
+
function parseEcsTaskLimits(payload) {
|
|
3601
|
+
if (typeof payload !== "object" || payload === null) return null;
|
|
3602
|
+
const limits = payload.Limits;
|
|
3603
|
+
if (typeof limits !== "object" || limits === null) return null;
|
|
3604
|
+
const cpu = limits.CPU;
|
|
3605
|
+
const memory = limits.Memory;
|
|
3606
|
+
if (typeof cpu !== "number" || !Number.isFinite(cpu) || cpu <= 0) return null;
|
|
3607
|
+
if (typeof memory !== "number" || !Number.isFinite(memory) || memory <= 0) return null;
|
|
3608
|
+
return {
|
|
3609
|
+
cpuCount: Math.max(1, Math.round(cpu)),
|
|
3610
|
+
memoryTotalBytes: memory * 1024 * 1024
|
|
3611
|
+
};
|
|
3612
|
+
}
|
|
3613
|
+
async function readEcsTaskLimits(env) {
|
|
3614
|
+
const uri = env.ECS_CONTAINER_METADATA_URI_V4;
|
|
3615
|
+
if (!uri) {
|
|
3616
|
+
return { limits: null };
|
|
3617
|
+
}
|
|
3618
|
+
const url = `${uri}/task`;
|
|
3619
|
+
try {
|
|
3620
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(ECS_METADATA_TIMEOUT_MS) });
|
|
3621
|
+
if (!response.ok) {
|
|
3622
|
+
return {
|
|
3623
|
+
limits: null,
|
|
3624
|
+
warning: `ECS task metadata fetch (${url}) returned HTTP ${response.status}`
|
|
3625
|
+
};
|
|
3626
|
+
}
|
|
3627
|
+
const payload = await response.json();
|
|
3628
|
+
const limits = parseEcsTaskLimits(payload);
|
|
3629
|
+
if (limits === null) {
|
|
3630
|
+
return {
|
|
3631
|
+
limits: null,
|
|
3632
|
+
warning: `ECS task metadata fetch (${url}) returned an unexpected payload`
|
|
3633
|
+
};
|
|
3634
|
+
}
|
|
3635
|
+
return { limits };
|
|
3636
|
+
} catch (error2) {
|
|
3637
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3638
|
+
return { limits: null, warning: `ECS task metadata fetch (${url}) failed: ${message}` };
|
|
3639
|
+
}
|
|
3640
|
+
}
|
|
3641
|
+
|
|
3642
|
+
// src/lib/resource-usage.ts
|
|
3643
|
+
function readCpuSample() {
|
|
3644
|
+
let busyMs = 0;
|
|
3645
|
+
let idleMs = 0;
|
|
3646
|
+
for (const cpu of cpus()) {
|
|
3647
|
+
busyMs += cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.irq;
|
|
3648
|
+
idleMs += cpu.times.idle;
|
|
2965
3649
|
}
|
|
2966
|
-
|
|
2967
|
-
return {
|
|
2968
|
-
mode: "auto",
|
|
2969
|
-
warnings: [
|
|
2970
|
-
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_MODES.join(", ")}; using auto`
|
|
2971
|
-
]
|
|
2972
|
-
};
|
|
3650
|
+
return { busyMs, idleMs };
|
|
2973
3651
|
}
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
const
|
|
2978
|
-
|
|
3652
|
+
function cpuPercentBetween(previous, current) {
|
|
3653
|
+
const deltaBusy = current.busyMs - previous.busyMs;
|
|
3654
|
+
const deltaIdle = current.idleMs - previous.idleMs;
|
|
3655
|
+
const total = deltaBusy + deltaIdle;
|
|
3656
|
+
if (total === 0) return null;
|
|
3657
|
+
return Math.round((deltaBusy / total * 100 + Number.EPSILON) * 100) / 100;
|
|
2979
3658
|
}
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
3659
|
+
function clamp(value, min, max) {
|
|
3660
|
+
return Math.min(Math.max(value, min), max);
|
|
3661
|
+
}
|
|
3662
|
+
function round2(value) {
|
|
3663
|
+
return Math.round((value + Number.EPSILON) * 100) / 100;
|
|
3664
|
+
}
|
|
3665
|
+
function readDisk(homeDir) {
|
|
3666
|
+
try {
|
|
3667
|
+
const stats = statfsSync2(homeDir);
|
|
3668
|
+
return {
|
|
3669
|
+
totalBytes: stats.bsize * stats.blocks,
|
|
3670
|
+
freeBytes: stats.bsize * stats.bavail
|
|
3671
|
+
};
|
|
3672
|
+
} catch (error2) {
|
|
3673
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3674
|
+
return {
|
|
3675
|
+
totalBytes: null,
|
|
3676
|
+
freeBytes: null,
|
|
3677
|
+
warning: `Could not read disk usage for ${homeDir}: ${message}`
|
|
3678
|
+
};
|
|
3679
|
+
}
|
|
3680
|
+
}
|
|
3681
|
+
function createResourceUsageCollector(homeDir) {
|
|
3682
|
+
let previous = readCpuSample();
|
|
3683
|
+
return async () => {
|
|
3684
|
+
const current = readCpuSample();
|
|
3685
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
3686
|
+
const hostCpuCount = cpus().length;
|
|
3687
|
+
previous = current;
|
|
3688
|
+
const disk = readDisk(homeDir);
|
|
3689
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
3690
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
3691
|
+
const warnings = [];
|
|
3692
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
3693
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
3694
|
+
let cpuPercent = hostCpuPercent;
|
|
3695
|
+
let cpuCount = hostCpuCount;
|
|
3696
|
+
let memoryTotalBytes = totalmem();
|
|
3697
|
+
let memoryAvailableBytes = freemem();
|
|
3698
|
+
if (limits !== null) {
|
|
3699
|
+
cpuCount = limits.cpuCount;
|
|
3700
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3701
|
+
memoryAvailableBytes = clamp(
|
|
3702
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3703
|
+
0,
|
|
3704
|
+
limits.memoryTotalBytes
|
|
3705
|
+
);
|
|
3706
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
3707
|
+
}
|
|
3708
|
+
return {
|
|
3709
|
+
usage: {
|
|
3710
|
+
cpuPercent,
|
|
3711
|
+
cpuCount,
|
|
3712
|
+
memoryTotalBytes,
|
|
3713
|
+
memoryAvailableBytes,
|
|
3714
|
+
diskTotalBytes: disk.totalBytes,
|
|
3715
|
+
diskFreeBytes: disk.freeBytes,
|
|
3716
|
+
opencodeDbBytes
|
|
3717
|
+
},
|
|
3718
|
+
warnings
|
|
3719
|
+
};
|
|
3720
|
+
};
|
|
2984
3721
|
}
|
|
2985
3722
|
|
|
2986
3723
|
// src/lib/channels/driver.ts
|
|
2987
|
-
import { homedir as
|
|
3724
|
+
import { homedir as homedir3 } from "os";
|
|
3725
|
+
|
|
3726
|
+
// src/lib/runner-file-sync.ts
|
|
3727
|
+
import { join as join6 } from "path";
|
|
2988
3728
|
|
|
2989
3729
|
// src/lib/file-push.ts
|
|
2990
3730
|
import { randomUUID } from "crypto";
|
|
2991
3731
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2992
|
-
import { basename, dirname as dirname3, isAbsolute, join as
|
|
3732
|
+
import { basename, dirname as dirname3, isAbsolute, join as join5, relative, resolve as resolve2, sep } from "path";
|
|
2993
3733
|
var FILE_MODE = 384;
|
|
2994
3734
|
var DIRECTORY_MODE = 448;
|
|
2995
3735
|
async function writePushedFile(request) {
|
|
@@ -3022,7 +3762,7 @@ async function writePushedFile(request) {
|
|
|
3022
3762
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3023
3763
|
dirname3(candidate)
|
|
3024
3764
|
);
|
|
3025
|
-
const realTarget =
|
|
3765
|
+
const realTarget = join5(existingAncestor, ...missingSegments, basename(candidate));
|
|
3026
3766
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3027
3767
|
if (allowedDirectory === null) {
|
|
3028
3768
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3058,7 +3798,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3058
3798
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3059
3799
|
return null;
|
|
3060
3800
|
}
|
|
3061
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3801
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join5(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3062
3802
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3063
3803
|
return null;
|
|
3064
3804
|
}
|
|
@@ -3131,13 +3871,13 @@ function contains(realDirectory, realTarget) {
|
|
|
3131
3871
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3132
3872
|
let current = existingAncestor;
|
|
3133
3873
|
for (const segment of missingSegments) {
|
|
3134
|
-
current =
|
|
3874
|
+
current = join5(current, segment);
|
|
3135
3875
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3136
3876
|
await chmod(current, DIRECTORY_MODE);
|
|
3137
3877
|
}
|
|
3138
3878
|
}
|
|
3139
3879
|
async function writeAtomically(realTarget, content) {
|
|
3140
|
-
const temporaryPath =
|
|
3880
|
+
const temporaryPath = join5(dirname3(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3141
3881
|
let handle;
|
|
3142
3882
|
try {
|
|
3143
3883
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -3179,17 +3919,28 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3179
3919
|
for (const id of options.ackFailures.keys()) {
|
|
3180
3920
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3181
3921
|
}
|
|
3182
|
-
if (pending.length === 0)
|
|
3922
|
+
if (pending.length === 0) {
|
|
3923
|
+
return { applied: 0, claudeCredentialApplied: false, opencodeAuthApplied: false };
|
|
3924
|
+
}
|
|
3183
3925
|
options.log({
|
|
3184
3926
|
level: "info",
|
|
3185
3927
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3186
3928
|
});
|
|
3187
3929
|
let applied = 0;
|
|
3930
|
+
let claudeCredentialApplied = false;
|
|
3931
|
+
let opencodeAuthApplied = false;
|
|
3188
3932
|
for (const file of pending) {
|
|
3189
3933
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3190
|
-
|
|
3934
|
+
const outcome = await applyOne(options, file);
|
|
3935
|
+
if (outcome.applied) applied += 1;
|
|
3936
|
+
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
3937
|
+
if (outcome.opencodeAuthApplied) opencodeAuthApplied = true;
|
|
3191
3938
|
}
|
|
3192
|
-
return
|
|
3939
|
+
return {
|
|
3940
|
+
applied,
|
|
3941
|
+
claudeCredentialApplied,
|
|
3942
|
+
opencodeAuthApplied
|
|
3943
|
+
};
|
|
3193
3944
|
}
|
|
3194
3945
|
async function listPendingFiles(options) {
|
|
3195
3946
|
let res;
|
|
@@ -3250,6 +4001,19 @@ function asPendingFile(entry) {
|
|
|
3250
4001
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3251
4002
|
return { id, path, size };
|
|
3252
4003
|
}
|
|
4004
|
+
var NOT_APPLIED = {
|
|
4005
|
+
applied: false,
|
|
4006
|
+
claudeCredentialApplied: false,
|
|
4007
|
+
opencodeAuthApplied: false
|
|
4008
|
+
};
|
|
4009
|
+
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
4010
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4011
|
+
return expanded === join6(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
4012
|
+
}
|
|
4013
|
+
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
4014
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4015
|
+
return expanded === join6(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
4016
|
+
}
|
|
3253
4017
|
async function applyOne(options, file) {
|
|
3254
4018
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
3255
4019
|
if (options.allowedDirectories.length === 0) {
|
|
@@ -3258,7 +4022,7 @@ async function applyOne(options, file) {
|
|
|
3258
4022
|
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
3259
4023
|
});
|
|
3260
4024
|
await ack(options, file, "rejected", "file_sync_disabled");
|
|
3261
|
-
return
|
|
4025
|
+
return NOT_APPLIED;
|
|
3262
4026
|
}
|
|
3263
4027
|
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
3264
4028
|
options.log({
|
|
@@ -3266,12 +4030,12 @@ async function applyOne(options, file) {
|
|
|
3266
4030
|
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
3267
4031
|
});
|
|
3268
4032
|
await ack(options, file, "rejected", "file_too_large");
|
|
3269
|
-
return
|
|
4033
|
+
return NOT_APPLIED;
|
|
3270
4034
|
}
|
|
3271
4035
|
const download = await downloadContent(options, file, label);
|
|
3272
4036
|
if (!download.ok) {
|
|
3273
4037
|
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
3274
|
-
return
|
|
4038
|
+
return NOT_APPLIED;
|
|
3275
4039
|
}
|
|
3276
4040
|
let outcome;
|
|
3277
4041
|
try {
|
|
@@ -3287,7 +4051,7 @@ async function applyOne(options, file) {
|
|
|
3287
4051
|
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
3288
4052
|
});
|
|
3289
4053
|
await ack(options, file, "rejected", "write_failed");
|
|
3290
|
-
return
|
|
4054
|
+
return NOT_APPLIED;
|
|
3291
4055
|
}
|
|
3292
4056
|
if (!outcome.ok) {
|
|
3293
4057
|
options.log({
|
|
@@ -3295,14 +4059,18 @@ async function applyOne(options, file) {
|
|
|
3295
4059
|
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
3296
4060
|
});
|
|
3297
4061
|
await ack(options, file, "rejected", outcome.code);
|
|
3298
|
-
return
|
|
4062
|
+
return NOT_APPLIED;
|
|
3299
4063
|
}
|
|
3300
4064
|
options.log({
|
|
3301
4065
|
level: "info",
|
|
3302
4066
|
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
3303
4067
|
});
|
|
3304
4068
|
await ack(options, file, "applied");
|
|
3305
|
-
return
|
|
4069
|
+
return {
|
|
4070
|
+
applied: true,
|
|
4071
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir),
|
|
4072
|
+
opencodeAuthApplied: isOpenCodeAuthPath(file.path, options.homeDir)
|
|
4073
|
+
};
|
|
3306
4074
|
}
|
|
3307
4075
|
function durableDownloadCode(status2) {
|
|
3308
4076
|
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
@@ -3413,8 +4181,13 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
|
3413
4181
|
var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
|
|
3414
4182
|
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
3415
4183
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
4184
|
+
var WATCHER_STALL_MS = 3 * POLL_MISS_GRACE_MS;
|
|
4185
|
+
var MAX_WATCHER_STALL_RESTARTS = 3;
|
|
4186
|
+
var MAX_RELEASED_OPENCODE_IDS = 256;
|
|
3416
4187
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3417
4188
|
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
4189
|
+
var WEDGE_WARNING_INTERVAL_MS = 5 * 60 * 1e3;
|
|
4190
|
+
var MAX_WEDGED_CONVERSATIONS = 256;
|
|
3418
4191
|
var ChannelAuthError = class extends Error {
|
|
3419
4192
|
constructor(message) {
|
|
3420
4193
|
super(message);
|
|
@@ -3458,6 +4231,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3458
4231
|
fileSyncDirectories;
|
|
3459
4232
|
homeDir;
|
|
3460
4233
|
maxActiveSessions;
|
|
4234
|
+
watcherStallMs;
|
|
4235
|
+
wedgeWarningIntervalMs;
|
|
3461
4236
|
/** Cache of conversationId → opencode sessionId. */
|
|
3462
4237
|
sessions = /* @__PURE__ */ new Map();
|
|
3463
4238
|
/**
|
|
@@ -3488,6 +4263,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3488
4263
|
* bounded cost.
|
|
3489
4264
|
*/
|
|
3490
4265
|
supersededSessions = /* @__PURE__ */ new Map();
|
|
4266
|
+
/**
|
|
4267
|
+
* Local re-drive fence for a message force-released by the stall watchdog
|
|
4268
|
+
* (#1618, `reconcileWatchers`'s `unrecoverable_released` arm — the only writer,
|
|
4269
|
+
* see `recordReleasedOpencodeId`). The row's server-side `opencode_message_id`
|
|
4270
|
+
* is `null` for exactly this shape (its `markProcessing` never landed), so
|
|
4271
|
+
* without a local record of the id the driver last knew, the next drain's
|
|
4272
|
+
* `if (message.opencode_message_id)` re-drive-fence check at
|
|
4273
|
+
* `processConversation` would not engage and it would blind-`prompt_async`
|
|
4274
|
+
* a turn that may still be running in opencode — the one duplicate-turn
|
|
4275
|
+
* hazard this whole design exists to close (§3/D1 of the drain-wedge plan).
|
|
4276
|
+
* `processConversation` reads `message.opencode_message_id ?? this
|
|
4277
|
+
* .releasedOpencodeIds.get(id)?.opencodeMessageId` as the EFFECTIVE id and
|
|
4278
|
+
* threads it into `resolveRedrive`, which asks opencode itself whether the
|
|
4279
|
+
* turn is still ongoing before ever dispatching.
|
|
4280
|
+
*
|
|
4281
|
+
* Bounded FIFO, mirroring `supersededSessions` above (`MAX_RELEASED_OPENCODE_IDS`,
|
|
4282
|
+
* `recordReleasedOpencodeId`). Cleared by `clearRedriveUnresolved` (every
|
|
4283
|
+
* non-`unresolved` `resolveRedrive` outcome fires it, including a fresh
|
|
4284
|
+
* dispatch) and at the top-level fresh-dispatch site, so it does not outlive
|
|
4285
|
+
* the row it was recorded for.
|
|
4286
|
+
*/
|
|
4287
|
+
releasedOpencodeIds = /* @__PURE__ */ new Map();
|
|
4288
|
+
/**
|
|
4289
|
+
* Per-conversation throttle state for the #183 recurrence warning (#1618
|
|
4290
|
+
* WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
|
|
4291
|
+
* `firstWedgedAt` anchors `stuck_for_ms`; `lastWarnedAt` throttles both the
|
|
4292
|
+
* log line and the `dispatch_wedged` signal to at most once per
|
|
4293
|
+
* `wedgeWarningIntervalMs`; `consecutiveTicks` is reported in the log text
|
|
4294
|
+
* so the operator sees magnitude, not repetition. Cleared the moment the
|
|
4295
|
+
* conversation dispatches anything (a fresh wedge, if it recurs, is a new
|
|
4296
|
+
* incident). Bounded FIFO, mirroring `supersededSessions`
|
|
4297
|
+
* (`MAX_WEDGED_CONVERSATIONS`).
|
|
4298
|
+
*/
|
|
4299
|
+
wedgeWarnings = /* @__PURE__ */ new Map();
|
|
3491
4300
|
/**
|
|
3492
4301
|
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
3493
4302
|
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
@@ -3706,6 +4515,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3706
4515
|
* same trick `lastProxiedActivityAt` uses.
|
|
3707
4516
|
*/
|
|
3708
4517
|
appliedFileCount = 0;
|
|
4518
|
+
/**
|
|
4519
|
+
* Generation counter, NOT a tally (#1656): advances by exactly one per sync
|
|
4520
|
+
* batch that applied the Claude CLI credential file, not by how many
|
|
4521
|
+
* credential files were in that batch. `run.ts` only ever tests inequality
|
|
4522
|
+
* against the value it saw last cycle, so magnitude is meaningless — keep it
|
|
4523
|
+
* that way rather than "fixing" it into a count.
|
|
4524
|
+
*/
|
|
4525
|
+
claudeCredentialApplyCount = 0;
|
|
4526
|
+
opencodeAuthApplyCount = 0;
|
|
3709
4527
|
/**
|
|
3710
4528
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3711
4529
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -3730,15 +4548,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3730
4548
|
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3731
4549
|
this.log = config.log ?? (() => {
|
|
3732
4550
|
});
|
|
3733
|
-
this.fetchImpl =
|
|
4551
|
+
this.fetchImpl = withRequestTimeout(
|
|
4552
|
+
config.fetchImpl ?? fetch,
|
|
4553
|
+
config.requestTimeoutMs ?? REQUEST_TIMEOUT_MS
|
|
4554
|
+
);
|
|
3734
4555
|
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3735
4556
|
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3736
4557
|
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
3737
4558
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
3738
4559
|
this.now = config.now ?? (() => Date.now());
|
|
3739
4560
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3740
|
-
this.homeDir = config.homeDir ??
|
|
4561
|
+
this.homeDir = config.homeDir ?? homedir3();
|
|
3741
4562
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4563
|
+
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4564
|
+
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
3742
4565
|
}
|
|
3743
4566
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
3744
4567
|
get opencodeBase() {
|
|
@@ -3752,6 +4575,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3752
4575
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
3753
4576
|
*/
|
|
3754
4577
|
async drainPending() {
|
|
4578
|
+
try {
|
|
4579
|
+
this.reconcileWatchers();
|
|
4580
|
+
} catch (err) {
|
|
4581
|
+
this.log({
|
|
4582
|
+
level: "error",
|
|
4583
|
+
message: `Watchdog: reconcileWatchers threw unexpectedly (drain continues): ${err instanceof Error ? err.message : String(err)}`
|
|
4584
|
+
});
|
|
4585
|
+
}
|
|
3755
4586
|
if (this.stopped) return 0;
|
|
3756
4587
|
if (this.draining) return 0;
|
|
3757
4588
|
this.draining = true;
|
|
@@ -3785,7 +4616,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3785
4616
|
if (this.syncingFiles) return 0;
|
|
3786
4617
|
this.syncingFiles = true;
|
|
3787
4618
|
try {
|
|
3788
|
-
const
|
|
4619
|
+
const result = await syncPendingRunnerFiles({
|
|
3789
4620
|
agentId: this.agentId,
|
|
3790
4621
|
apiUrl: this.apiUrl,
|
|
3791
4622
|
getAuthHeader: this.getAuthHeader,
|
|
@@ -3795,8 +4626,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3795
4626
|
ackFailures: this.fileAckFailures,
|
|
3796
4627
|
log: this.log
|
|
3797
4628
|
});
|
|
3798
|
-
this.appliedFileCount += applied;
|
|
3799
|
-
|
|
4629
|
+
this.appliedFileCount += result.applied;
|
|
4630
|
+
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
4631
|
+
if (result.opencodeAuthApplied) this.opencodeAuthApplyCount += 1;
|
|
4632
|
+
return result.applied;
|
|
3800
4633
|
} catch (err) {
|
|
3801
4634
|
this.log({
|
|
3802
4635
|
level: "error",
|
|
@@ -3887,12 +4720,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3887
4720
|
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3888
4721
|
* idle checks still shows up as an advance.
|
|
3889
4722
|
*
|
|
4723
|
+
* A THIRD signal, `claudeCredentialApplies`, is a separate re-arm trigger
|
|
4724
|
+
* (#1656), not idle accounting: `run.ts` gates re-probing Claude usage
|
|
4725
|
+
* reporting on it advancing, so an unrelated file sync can never disturb a
|
|
4726
|
+
* healthy reporting cadence (#1627) — it never even reaches that trigger, let
|
|
4727
|
+
* alone gets declined by it. Keep this narrower signal OUT of `appliedFiles`,
|
|
4728
|
+
* whose consumer is idle-timeout suppression and must key on ANY file, not
|
|
4729
|
+
* just a Claude credential.
|
|
4730
|
+
*
|
|
3890
4731
|
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3891
4732
|
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3892
4733
|
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3893
4734
|
*/
|
|
3894
4735
|
fileSyncActivity() {
|
|
3895
|
-
return {
|
|
4736
|
+
return {
|
|
4737
|
+
appliedFiles: this.appliedFileCount,
|
|
4738
|
+
inFlight: this.syncingFiles,
|
|
4739
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount,
|
|
4740
|
+
opencodeAuthApplyCount: this.opencodeAuthApplyCount
|
|
4741
|
+
};
|
|
3896
4742
|
}
|
|
3897
4743
|
/**
|
|
3898
4744
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
@@ -4005,8 +4851,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4005
4851
|
skippedAlreadyDispatched += 1;
|
|
4006
4852
|
continue;
|
|
4007
4853
|
}
|
|
4008
|
-
|
|
4009
|
-
|
|
4854
|
+
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
4855
|
+
if (effectiveOpencodeMessageId) {
|
|
4856
|
+
const outcome = await this.resolveRedrive(
|
|
4857
|
+
conv,
|
|
4858
|
+
sessionId,
|
|
4859
|
+
message,
|
|
4860
|
+
sessionCreated,
|
|
4861
|
+
effectiveOpencodeMessageId
|
|
4862
|
+
);
|
|
4010
4863
|
if (outcome === "abandoned") {
|
|
4011
4864
|
continue;
|
|
4012
4865
|
}
|
|
@@ -4117,21 +4970,102 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4117
4970
|
}
|
|
4118
4971
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4119
4972
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
4973
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
4120
4974
|
this.dispatched.add(message.id);
|
|
4121
4975
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
4122
4976
|
dispatched += 1;
|
|
4123
4977
|
void this.postSignal(conv.id, message.id, "dispatched");
|
|
4124
4978
|
}
|
|
4125
4979
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
4126
|
-
this.
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
conversation_id: conv.id
|
|
4130
|
-
});
|
|
4980
|
+
this.reportWedgedConversation(conv, messages);
|
|
4981
|
+
} else if (dispatched > 0) {
|
|
4982
|
+
this.wedgeWarnings.delete(conv.id);
|
|
4131
4983
|
}
|
|
4132
4984
|
this.ensureWatcherRunning(sessionId);
|
|
4133
4985
|
return dispatched;
|
|
4134
4986
|
}
|
|
4987
|
+
/**
|
|
4988
|
+
* The #183 "eyes but nothing sent" recurrence for `conv`, throttled and
|
|
4989
|
+
* escalated (#1618 WI-4). `messages` is the conversation's full pending list
|
|
4990
|
+
* on THIS tick — the caller has already confirmed every one of them is a
|
|
4991
|
+
* skip-because-already-`dispatched`, the exact signature of a message stuck
|
|
4992
|
+
* acknowledged-but-never-worked.
|
|
4993
|
+
*
|
|
4994
|
+
* Unthrottled, this fired every ~6s drain tick for as long as a wedge lasted
|
|
4995
|
+
* (52,843 occurrences observed in one incident) — burning the GLOBAL
|
|
4996
|
+
* 30-events/60s `runner-activity-telemetry.ts` budget that was itself
|
|
4997
|
+
* suppressing the diagnostics needed to debug the wedge. The `warn` log (and
|
|
4998
|
+
* the `dispatch_wedged` signal once the wedge has persisted past the same
|
|
4999
|
+
* interval) fire at most once per `wedgeWarningIntervalMs` per conversation,
|
|
5000
|
+
* naming the consecutive-tick count so the operator sees magnitude rather
|
|
5001
|
+
* than repetition.
|
|
5002
|
+
*
|
|
5003
|
+
* Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
|
|
5004
|
+
* unconditionally on this same tick and is already recovering anything it
|
|
5005
|
+
* can see. This is reporting only — see `countUntrackedIds`'s doc for the
|
|
5006
|
+
* one case it recovers nothing FOR (§3/D5 of the drain-wedge plan).
|
|
5007
|
+
*/
|
|
5008
|
+
reportWedgedConversation(conv, messages) {
|
|
5009
|
+
const now = this.now();
|
|
5010
|
+
const existing = this.wedgeWarnings.get(conv.id);
|
|
5011
|
+
const firstWedgedAt = existing?.firstWedgedAt ?? now;
|
|
5012
|
+
const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
|
|
5013
|
+
const dueForWarn = !existing || now - existing.lastWarnedAt >= this.wedgeWarningIntervalMs;
|
|
5014
|
+
if (!dueForWarn) {
|
|
5015
|
+
this.wedgeWarnings.delete(conv.id);
|
|
5016
|
+
this.wedgeWarnings.set(conv.id, {
|
|
5017
|
+
firstWedgedAt,
|
|
5018
|
+
lastWarnedAt: existing.lastWarnedAt,
|
|
5019
|
+
consecutiveTicks
|
|
5020
|
+
});
|
|
5021
|
+
return;
|
|
5022
|
+
}
|
|
5023
|
+
const stuckForMs = now - firstWedgedAt;
|
|
5024
|
+
const untracked = this.countUntrackedIds(messages);
|
|
5025
|
+
this.log({
|
|
5026
|
+
level: "warn",
|
|
5027
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
5028
|
+
conversation_id: conv.id
|
|
5029
|
+
});
|
|
5030
|
+
this.wedgeWarnings.delete(conv.id);
|
|
5031
|
+
this.wedgeWarnings.set(conv.id, { firstWedgedAt, lastWarnedAt: now, consecutiveTicks });
|
|
5032
|
+
while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
|
|
5033
|
+
const oldest = this.wedgeWarnings.keys().next().value;
|
|
5034
|
+
if (oldest === void 0) break;
|
|
5035
|
+
this.wedgeWarnings.delete(oldest);
|
|
5036
|
+
}
|
|
5037
|
+
if (stuckForMs >= this.wedgeWarningIntervalMs) {
|
|
5038
|
+
void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
|
|
5039
|
+
stuck_for_ms: stuckForMs,
|
|
5040
|
+
untracked
|
|
5041
|
+
});
|
|
5042
|
+
}
|
|
5043
|
+
}
|
|
5044
|
+
/**
|
|
5045
|
+
* How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
|
|
5046
|
+
* WI-4) — the §3/D5 orphan discriminator. `reconcileWatchers` proves the
|
|
5047
|
+
* `dispatched`/`inFlight` pairing invariant holds by construction across
|
|
5048
|
+
* every `dispatched.add` site (see its own doc comment), so `> 0` here means
|
|
5049
|
+
* that invariant has actually been violated for this conversation: there is
|
|
5050
|
+
* no watcher for WI-1's watchdog to restart, so it will NOT self-heal.
|
|
5051
|
+
* `=== 0` means an ordinary stalled/exited watcher, which WI-1 is already
|
|
5052
|
+
* recovering. One pass over `this.watchers`, called only when the throttled
|
|
5053
|
+
* warning above is due to fire — not every tick.
|
|
5054
|
+
*/
|
|
5055
|
+
countUntrackedIds(messages) {
|
|
5056
|
+
let untracked = 0;
|
|
5057
|
+
for (const message of messages) {
|
|
5058
|
+
let tracked = false;
|
|
5059
|
+
for (const watcher of this.watchers.values()) {
|
|
5060
|
+
if (watcher.inFlight.has(message.id)) {
|
|
5061
|
+
tracked = true;
|
|
5062
|
+
break;
|
|
5063
|
+
}
|
|
5064
|
+
}
|
|
5065
|
+
if (!tracked) untracked += 1;
|
|
5066
|
+
}
|
|
5067
|
+
return untracked;
|
|
5068
|
+
}
|
|
4135
5069
|
/**
|
|
4136
5070
|
* Poll a session's message list for the re-drive fence (#965), via the
|
|
4137
5071
|
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
@@ -4200,9 +5134,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4200
5134
|
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
4201
5135
|
* other failure resolves to `unresolved` and is retried whole on the next
|
|
4202
5136
|
* ~2s drain tick.
|
|
5137
|
+
*
|
|
5138
|
+
* `effectiveOpencodeMessageId` (#1618) is the caller-resolved id: the real
|
|
5139
|
+
* server `opencode_message_id` when present, else the stall watchdog's local
|
|
5140
|
+
* `releasedOpencodeIds` fence. Read it here rather than re-deriving it from
|
|
5141
|
+
* `message` so every line below — and the signals this method posts —
|
|
5142
|
+
* keeps reporting the REAL server row; a shadow-copied `message` would
|
|
5143
|
+
* silently diverge from it.
|
|
4203
5144
|
*/
|
|
4204
|
-
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
4205
|
-
const ocId =
|
|
5145
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated, effectiveOpencodeMessageId) {
|
|
5146
|
+
const ocId = effectiveOpencodeMessageId;
|
|
4206
5147
|
if (sessionCreated) {
|
|
4207
5148
|
this.clearRedriveUnresolved(message.id);
|
|
4208
5149
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
@@ -4432,7 +5373,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4432
5373
|
}
|
|
4433
5374
|
return "unresolved";
|
|
4434
5375
|
}
|
|
4435
|
-
/**
|
|
5376
|
+
/**
|
|
5377
|
+
* Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved`
|
|
5378
|
+
* outcome) — including the stall watchdog's local re-drive fence (#1618): once
|
|
5379
|
+
* `resolveRedrive` has resolved to dispatch/reattach/settle, the row either has
|
|
5380
|
+
* a real server-side `opencode_message_id` again or is no longer pending, so
|
|
5381
|
+
* the fence entry is no longer needed.
|
|
5382
|
+
*/
|
|
4436
5383
|
clearRedriveUnresolved(messageId) {
|
|
4437
5384
|
this.redriveUnresolvedSince.delete(messageId);
|
|
4438
5385
|
this.redriveUnresolvedSignalled.delete(messageId);
|
|
@@ -4440,6 +5387,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4440
5387
|
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4441
5388
|
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4442
5389
|
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
5390
|
+
this.releasedOpencodeIds.delete(messageId);
|
|
4443
5391
|
}
|
|
4444
5392
|
/**
|
|
4445
5393
|
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
@@ -4594,6 +5542,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4594
5542
|
this.supersededSessions.delete(oldest);
|
|
4595
5543
|
}
|
|
4596
5544
|
}
|
|
5545
|
+
/**
|
|
5546
|
+
* Record the local re-drive fence for a message force-released without
|
|
5547
|
+
* completing (#1618) — see the `releasedOpencodeIds` field doc. Call BEFORE
|
|
5548
|
+
* `removeInFlight`, which is about to drop the `InFlightMessage` this reads
|
|
5549
|
+
* `opencodeMessageId` from. Hard-capped FIFO, same shape as `supersede`.
|
|
5550
|
+
*/
|
|
5551
|
+
recordReleasedOpencodeId(evidentMessageId, sessionId, opencodeMessageId) {
|
|
5552
|
+
this.releasedOpencodeIds.delete(evidentMessageId);
|
|
5553
|
+
this.releasedOpencodeIds.set(evidentMessageId, { sessionId, opencodeMessageId });
|
|
5554
|
+
while (this.releasedOpencodeIds.size > MAX_RELEASED_OPENCODE_IDS) {
|
|
5555
|
+
const oldest = this.releasedOpencodeIds.keys().next().value;
|
|
5556
|
+
if (oldest === void 0) return;
|
|
5557
|
+
this.releasedOpencodeIds.delete(oldest);
|
|
5558
|
+
}
|
|
5559
|
+
}
|
|
4597
5560
|
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
4598
5561
|
isSuperseded(conversationId, sessionId) {
|
|
4599
5562
|
return this.supersededSessions.get(conversationId) === sessionId;
|
|
@@ -4635,6 +5598,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4635
5598
|
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
4636
5599
|
conversation_id: conv.id
|
|
4637
5600
|
});
|
|
5601
|
+
const watcher = this.watchers.get(bound);
|
|
5602
|
+
if (watcher) {
|
|
5603
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5604
|
+
this.recordReleasedOpencodeId(evidentMessageId, bound, inFlight.opencodeMessageId);
|
|
5605
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5606
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5607
|
+
recovery: "session_gone_released"
|
|
5608
|
+
});
|
|
5609
|
+
}
|
|
5610
|
+
this.watchers.delete(bound);
|
|
5611
|
+
}
|
|
4638
5612
|
this.sessions.delete(conv.id);
|
|
4639
5613
|
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
4640
5614
|
}
|
|
@@ -4825,15 +5799,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4825
5799
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
4826
5800
|
let watcher = this.watchers.get(sessionId);
|
|
4827
5801
|
if (!watcher) {
|
|
4828
|
-
watcher =
|
|
4829
|
-
conv,
|
|
4830
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4831
|
-
loop: null,
|
|
4832
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4833
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4834
|
-
lastGoodPollAt: this.now(),
|
|
4835
|
-
hadUsablePoll: false
|
|
4836
|
-
};
|
|
5802
|
+
watcher = this.newSessionWatcher(conv);
|
|
4837
5803
|
this.watchers.set(sessionId, watcher);
|
|
4838
5804
|
}
|
|
4839
5805
|
const now = this.now();
|
|
@@ -4859,11 +5825,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4859
5825
|
deliveryDeadlineAnchored: false,
|
|
4860
5826
|
b2PinnedSinceMs: 0,
|
|
4861
5827
|
b2LastDescendantCheckMs: 0,
|
|
5828
|
+
b2RootOngoingHeldLogged: false,
|
|
4862
5829
|
b2AbandonedSignalled: false,
|
|
4863
5830
|
ambiguousPinnedSinceMs: 0,
|
|
4864
5831
|
ambiguousResolved: false
|
|
4865
5832
|
});
|
|
4866
5833
|
}
|
|
5834
|
+
/**
|
|
5835
|
+
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
5836
|
+
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
5837
|
+
* misread as stalled by the very first reconciliation that sees it.
|
|
5838
|
+
*/
|
|
5839
|
+
newSessionWatcher(conv) {
|
|
5840
|
+
const now = this.now();
|
|
5841
|
+
return {
|
|
5842
|
+
conv,
|
|
5843
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
5844
|
+
loop: null,
|
|
5845
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
5846
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
5847
|
+
lastGoodPollAt: now,
|
|
5848
|
+
hadUsablePoll: false,
|
|
5849
|
+
generation: 0,
|
|
5850
|
+
lastTickAt: now,
|
|
5851
|
+
lastObservedTickAt: now,
|
|
5852
|
+
consecutiveStallRestarts: 0
|
|
5853
|
+
};
|
|
5854
|
+
}
|
|
4867
5855
|
/**
|
|
4868
5856
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
4869
5857
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
@@ -4893,15 +5881,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4893
5881
|
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
4894
5882
|
let watcher = this.watchers.get(sessionId);
|
|
4895
5883
|
if (!watcher) {
|
|
4896
|
-
watcher =
|
|
4897
|
-
conv,
|
|
4898
|
-
inFlight: /* @__PURE__ */ new Map(),
|
|
4899
|
-
loop: null,
|
|
4900
|
-
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
4901
|
-
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
4902
|
-
lastGoodPollAt: this.now(),
|
|
4903
|
-
hadUsablePoll: false
|
|
4904
|
-
};
|
|
5884
|
+
watcher = this.newSessionWatcher(conv);
|
|
4905
5885
|
this.watchers.set(sessionId, watcher);
|
|
4906
5886
|
}
|
|
4907
5887
|
watcher.inFlight.set(message.id, {
|
|
@@ -4940,17 +5920,116 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4940
5920
|
deliveryDeadlineAnchored: false,
|
|
4941
5921
|
b2PinnedSinceMs: 0,
|
|
4942
5922
|
b2LastDescendantCheckMs: 0,
|
|
5923
|
+
b2RootOngoingHeldLogged: false,
|
|
4943
5924
|
b2AbandonedSignalled: false,
|
|
4944
5925
|
ambiguousPinnedSinceMs: 0,
|
|
4945
5926
|
ambiguousResolved: false
|
|
4946
5927
|
});
|
|
4947
5928
|
}
|
|
5929
|
+
/**
|
|
5930
|
+
* Loop-liveness watchdog (#1618). Runs once per `drainPending()` tick and
|
|
5931
|
+
* restarts any per-session watcher whose loop has exited or stopped ticking
|
|
5932
|
+
* — escalating to a bounded force-release only once
|
|
5933
|
+
* `MAX_WATCHER_STALL_RESTARTS` consecutive restarts have failed to recover
|
|
5934
|
+
* it. Fully synchronous: it only inspects in-memory state and calls the
|
|
5935
|
+
* synchronous `ensureWatcherRunning`/`removeInFlight`, which is what lets it
|
|
5936
|
+
* run from the very top of `drainPending()` — ahead of the un-timed
|
|
5937
|
+
* `getPendingConversations()` await that would otherwise be able to disable
|
|
5938
|
+
* it (`run.ts`'s poll loop is sequential, so a hung fetch there stops
|
|
5939
|
+
* `drainPending()` from being CALLED again at all, not just from finishing).
|
|
5940
|
+
*
|
|
5941
|
+
* Restarts the loop rather than releasing messages directly: a blind release
|
|
5942
|
+
* would let the next drain re-`prompt_async` a turn that may still be
|
|
5943
|
+
* running (ADR-0047; see `releasedOpencodeIds`'s doc). A restarted loop
|
|
5944
|
+
* re-polls with each message's `opencodeMessageId` still in hand and lets
|
|
5945
|
+
* the existing, audited `!activelyRunning` give-up decide, same as it always
|
|
5946
|
+
* has.
|
|
5947
|
+
*
|
|
5948
|
+
* Deliberately does NOT sweep `this.dispatched` for an id no watcher tracks:
|
|
5949
|
+
* that shape has no in-flight entry and therefore no `opencodeMessageId` to
|
|
5950
|
+
* fence a release with, so releasing it here would blind-re-POST a possibly-
|
|
5951
|
+
* running turn — and there is no conversation id in hand to signal with
|
|
5952
|
+
* either. Detection for that shape lives on WI-4's `dispatch_wedged` signal
|
|
5953
|
+
* instead, where a conversation id already exists. If you find yourself
|
|
5954
|
+
* wanting to add a `dispatched` sweep here, don't — read the drain-wedge
|
|
5955
|
+
* plan's §3/D5 first.
|
|
5956
|
+
*/
|
|
5957
|
+
reconcileWatchers() {
|
|
5958
|
+
const now = this.now();
|
|
5959
|
+
for (const [sessionId, watcher] of [...this.watchers]) {
|
|
5960
|
+
if (watcher.lastTickAt !== watcher.lastObservedTickAt) {
|
|
5961
|
+
watcher.consecutiveStallRestarts = 0;
|
|
5962
|
+
}
|
|
5963
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
5964
|
+
if (watcher.inFlight.size === 0 && watcher.loop === null) {
|
|
5965
|
+
this.watchers.delete(sessionId);
|
|
5966
|
+
continue;
|
|
5967
|
+
}
|
|
5968
|
+
if (watcher.loop === null && watcher.inFlight.size > 0) {
|
|
5969
|
+
if (now - watcher.lastTickAt < this.watcherStallMs) continue;
|
|
5970
|
+
this.log({
|
|
5971
|
+
level: "warn",
|
|
5972
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had exited with ${watcher.inFlight.size} message(s) still in flight (idle ${now - watcher.lastTickAt}ms) \u2014 restarting`,
|
|
5973
|
+
conversation_id: watcher.conv.id
|
|
5974
|
+
});
|
|
5975
|
+
this.ensureWatcherRunning(sessionId);
|
|
5976
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
5977
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5978
|
+
recovery: "loop_exited"
|
|
5979
|
+
});
|
|
5980
|
+
}
|
|
5981
|
+
continue;
|
|
5982
|
+
}
|
|
5983
|
+
if (watcher.loop !== null && now - watcher.lastTickAt >= this.watcherStallMs) {
|
|
5984
|
+
const stalledForMs = now - watcher.lastTickAt;
|
|
5985
|
+
watcher.consecutiveStallRestarts += 1;
|
|
5986
|
+
if (watcher.consecutiveStallRestarts > MAX_WATCHER_STALL_RESTARTS) {
|
|
5987
|
+
this.log({
|
|
5988
|
+
level: "error",
|
|
5989
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop stalled through ${watcher.consecutiveStallRestarts} restarts (last stall ${stalledForMs}ms) \u2014 releasing its ${watcher.inFlight.size} in-flight message(s)`,
|
|
5990
|
+
conversation_id: watcher.conv.id
|
|
5991
|
+
});
|
|
5992
|
+
for (const [evidentMessageId, inFlight] of [...watcher.inFlight]) {
|
|
5993
|
+
this.recordReleasedOpencodeId(evidentMessageId, sessionId, inFlight.opencodeMessageId);
|
|
5994
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
5995
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
5996
|
+
recovery: "unrecoverable_released"
|
|
5997
|
+
});
|
|
5998
|
+
}
|
|
5999
|
+
watcher.generation += 1;
|
|
6000
|
+
this.watchers.delete(sessionId);
|
|
6001
|
+
continue;
|
|
6002
|
+
}
|
|
6003
|
+
watcher.generation += 1;
|
|
6004
|
+
watcher.loop = null;
|
|
6005
|
+
watcher.lastGoodPollAt = now;
|
|
6006
|
+
watcher.lastTickAt = now;
|
|
6007
|
+
watcher.lastObservedTickAt = watcher.lastTickAt;
|
|
6008
|
+
this.ensureWatcherRunning(sessionId);
|
|
6009
|
+
this.log({
|
|
6010
|
+
level: "warn",
|
|
6011
|
+
message: `Watchdog: session ${sessionId.slice(0, 8)}'s watcher loop had not ticked in ${stalledForMs}ms \u2014 restarted under generation ${watcher.generation} (${watcher.consecutiveStallRestarts}/${MAX_WATCHER_STALL_RESTARTS})`,
|
|
6012
|
+
conversation_id: watcher.conv.id
|
|
6013
|
+
});
|
|
6014
|
+
for (const evidentMessageId of watcher.inFlight.keys()) {
|
|
6015
|
+
void this.postSignal(watcher.conv.id, evidentMessageId, "watcher_recovered", {
|
|
6016
|
+
recovery: "loop_stalled"
|
|
6017
|
+
});
|
|
6018
|
+
}
|
|
6019
|
+
}
|
|
6020
|
+
}
|
|
6021
|
+
}
|
|
4948
6022
|
/**
|
|
4949
6023
|
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
4950
6024
|
* work and is not already running. Single-flight per session. The loop is
|
|
4951
6025
|
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
4952
6026
|
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
4953
6027
|
* stays as the safety net.
|
|
6028
|
+
*
|
|
6029
|
+
* The generation started here (#1618) is captured in the `.finally` closure
|
|
6030
|
+
* so a RETIRED loop settling late — after `reconcileWatchers` has already
|
|
6031
|
+
* restarted this watcher under a newer generation — can neither null the new
|
|
6032
|
+
* loop's handle nor delete a watcher that still has live work.
|
|
4954
6033
|
*/
|
|
4955
6034
|
ensureWatcherRunning(sessionId) {
|
|
4956
6035
|
const watcher = this.watchers.get(sessionId);
|
|
@@ -4960,7 +6039,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4960
6039
|
this.watchers.delete(sessionId);
|
|
4961
6040
|
return;
|
|
4962
6041
|
}
|
|
4963
|
-
const
|
|
6042
|
+
const generation = watcher.generation;
|
|
6043
|
+
const loop = this.runWatcherLoop(sessionId, watcher, generation).finally(() => {
|
|
6044
|
+
if (watcher.generation !== generation) return;
|
|
4964
6045
|
watcher.loop = null;
|
|
4965
6046
|
if (watcher.inFlight.size === 0) {
|
|
4966
6047
|
this.watchers.delete(sessionId);
|
|
@@ -4980,11 +6061,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4980
6061
|
* `source_message_id`;
|
|
4981
6062
|
* 4. drops messages that completed or timed out from the in-flight set.
|
|
4982
6063
|
* Exits when the in-flight set empties. Never throws.
|
|
6064
|
+
*
|
|
6065
|
+
* `generation` (#1618) is the incarnation this call was started under.
|
|
6066
|
+
* `reconcileWatchers` can restart a stalled loop by bumping
|
|
6067
|
+
* `watcher.generation` and starting a NEW `runWatcherLoop` over the same
|
|
6068
|
+
* `SessionWatcher` object — the stalled promise itself cannot be cancelled,
|
|
6069
|
+
* so this loop instead checks at the top of every iteration, right after
|
|
6070
|
+
* waking from `sleep`, and right before servicing any message, and quietly
|
|
6071
|
+
* retires (returns without touching anything) the moment it is no longer the
|
|
6072
|
+
* watcher's current generation. Retiring mid-tick can still let ONE
|
|
6073
|
+
* `serviceInFlightMessage` pass complete first — acceptable, since that
|
|
6074
|
+
* method contains no non-idempotent action.
|
|
4983
6075
|
*/
|
|
4984
|
-
async runWatcherLoop(sessionId, watcher) {
|
|
6076
|
+
async runWatcherLoop(sessionId, watcher, generation) {
|
|
4985
6077
|
try {
|
|
4986
6078
|
while (watcher.inFlight.size > 0) {
|
|
6079
|
+
if (watcher.generation !== generation) return;
|
|
6080
|
+
watcher.lastTickAt = this.now();
|
|
4987
6081
|
await this.sleep(this.pausedPollIntervalMs);
|
|
6082
|
+
if (watcher.generation !== generation) return;
|
|
4988
6083
|
let messages = null;
|
|
4989
6084
|
try {
|
|
4990
6085
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
@@ -5005,6 +6100,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5005
6100
|
}
|
|
5006
6101
|
}
|
|
5007
6102
|
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
6103
|
+
if (watcher.generation !== generation) return;
|
|
5008
6104
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
5009
6105
|
await this.serviceInFlightMessage(
|
|
5010
6106
|
sessionId,
|
|
@@ -5178,6 +6274,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5178
6274
|
if (snapshotReadable) {
|
|
5179
6275
|
inFlight.b2PinnedSinceMs = 0;
|
|
5180
6276
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
6277
|
+
inFlight.b2RootOngoingHeldLogged = false;
|
|
5181
6278
|
inFlight.b2AbandonedSignalled = false;
|
|
5182
6279
|
}
|
|
5183
6280
|
} else {
|
|
@@ -5189,11 +6286,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5189
6286
|
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
5190
6287
|
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
5191
6288
|
inFlight.b2LastDescendantCheckMs = this.now();
|
|
5192
|
-
const descendantOngoing = await
|
|
6289
|
+
const [descendantOngoing, rootOngoing] = await Promise.all([
|
|
6290
|
+
this.isAnyDescendantSessionOngoing(sessionId),
|
|
6291
|
+
isSessionOngoing(this.port, sessionId)
|
|
6292
|
+
]);
|
|
5193
6293
|
if (isB2AbandonmentConfirmed({
|
|
5194
6294
|
pinnedForMs,
|
|
5195
6295
|
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
5196
|
-
descendantOngoing
|
|
6296
|
+
descendantOngoing,
|
|
6297
|
+
rootOngoing
|
|
5197
6298
|
})) {
|
|
5198
6299
|
inFlight.b2AbandonedSignalled = true;
|
|
5199
6300
|
this.log({
|
|
@@ -5202,12 +6303,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5202
6303
|
conversation_id: conv.id,
|
|
5203
6304
|
message_id: id
|
|
5204
6305
|
});
|
|
6306
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
5205
6307
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
5206
|
-
watched_for_ms: pinnedForMs
|
|
6308
|
+
watched_for_ms: pinnedForMs,
|
|
6309
|
+
finish: reply?.info?.finish ?? reply?.finish,
|
|
6310
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
6311
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
6312
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
5207
6313
|
});
|
|
5208
6314
|
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
5209
6315
|
return;
|
|
5210
6316
|
}
|
|
6317
|
+
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
6318
|
+
inFlight.b2RootOngoingHeldLogged = true;
|
|
6319
|
+
this.log({
|
|
6320
|
+
level: "warn",
|
|
6321
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with root_ongoing=${rootOngoing} and descendant_ongoing=${descendantOngoing} \u2014 holding until OpenCode confirms the root is idle`,
|
|
6322
|
+
conversation_id: conv.id,
|
|
6323
|
+
message_id: id
|
|
6324
|
+
});
|
|
6325
|
+
}
|
|
5211
6326
|
}
|
|
5212
6327
|
}
|
|
5213
6328
|
const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
|
|
@@ -6950,6 +8065,7 @@ var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_
|
|
|
6950
8065
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
6951
8066
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
6952
8067
|
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
8068
|
+
var CHILD_STOP_TIMEOUT_MS = 1e4;
|
|
6953
8069
|
function resolveLogLevel(options) {
|
|
6954
8070
|
const accepted = Object.keys(LOG_LEVELS);
|
|
6955
8071
|
const validate = (value, source) => {
|
|
@@ -6980,7 +8096,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
6980
8096
|
if (trimmed === "") {
|
|
6981
8097
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6982
8098
|
}
|
|
6983
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
8099
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join7(homeDir, trimmed.slice(2)) : trimmed;
|
|
6984
8100
|
if (!isAbsolute2(expanded)) {
|
|
6985
8101
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6986
8102
|
}
|
|
@@ -7078,7 +8194,7 @@ function logActivity(state, entry) {
|
|
|
7078
8194
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
7079
8195
|
if (!meetsThreshold(state, level)) return;
|
|
7080
8196
|
forwardRunnerActivity(
|
|
7081
|
-
{ level, message: entry.message, error: entry.error },
|
|
8197
|
+
{ level, message: entry.message, error: entry.error, metadata: entry.metadata },
|
|
7082
8198
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
7083
8199
|
);
|
|
7084
8200
|
const fullEntry = {
|
|
@@ -7098,6 +8214,26 @@ function logActivity(state, entry) {
|
|
|
7098
8214
|
}
|
|
7099
8215
|
}
|
|
7100
8216
|
}
|
|
8217
|
+
function reportSessionDbRecovery(state) {
|
|
8218
|
+
try {
|
|
8219
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir4(), env: process.env });
|
|
8220
|
+
for (const record of report.records) {
|
|
8221
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
8222
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
8223
|
+
logActivity(state, {
|
|
8224
|
+
type: activity.level === "error" ? "error" : "info",
|
|
8225
|
+
level: activity.level,
|
|
8226
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
8227
|
+
metadata: activity.metadata
|
|
8228
|
+
});
|
|
8229
|
+
}
|
|
8230
|
+
acknowledgeSessionDbRecoveryReport(report.path);
|
|
8231
|
+
} catch (error2) {
|
|
8232
|
+
console.error(
|
|
8233
|
+
`[run] could not report session-DB recovery activity: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
8234
|
+
);
|
|
8235
|
+
}
|
|
8236
|
+
}
|
|
7101
8237
|
function displayStatus(state) {
|
|
7102
8238
|
if (!state.interactive) return;
|
|
7103
8239
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
@@ -7187,6 +8323,8 @@ async function driveChannels(state, driver) {
|
|
|
7187
8323
|
let unreachableMs = 0;
|
|
7188
8324
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7189
8325
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
8326
|
+
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
8327
|
+
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
7190
8328
|
while (state.running) {
|
|
7191
8329
|
const cycleStartedAtMs = performance.now();
|
|
7192
8330
|
let idleThisCycle = false;
|
|
@@ -7210,11 +8348,19 @@ async function driveChannels(state, driver) {
|
|
|
7210
8348
|
state.messageCount += processed;
|
|
7211
8349
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
7212
8350
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7213
|
-
const
|
|
8351
|
+
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
8352
|
+
const appliedFiles = fileActivitySnapshot.appliedFiles;
|
|
7214
8353
|
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
7215
8354
|
const fileActivity = carriedOverFileSync || filesApplied;
|
|
7216
8355
|
lastSeenAppliedFiles = appliedFiles;
|
|
7217
|
-
|
|
8356
|
+
const claudeCredentialApplies = fileActivitySnapshot.claudeCredentialApplies;
|
|
8357
|
+
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
8358
|
+
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
8359
|
+
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
8360
|
+
const opencodeAuthApplies = fileActivitySnapshot.opencodeAuthApplyCount;
|
|
8361
|
+
const opencodeAuthApplied = opencodeAuthApplies !== lastSeenOpencodeAuthApplies;
|
|
8362
|
+
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
8363
|
+
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
7218
8364
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7219
8365
|
idlePolls = 0;
|
|
7220
8366
|
idleMs = 0;
|
|
@@ -7283,7 +8429,7 @@ async function driveChannels(state, driver) {
|
|
|
7283
8429
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7284
8430
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7285
8431
|
function sessionDbPath() {
|
|
7286
|
-
return
|
|
8432
|
+
return join7(homedir4(), ".local", "share", "opencode", "opencode.db");
|
|
7287
8433
|
}
|
|
7288
8434
|
async function runSweep(state, driver, config) {
|
|
7289
8435
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7366,7 +8512,7 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7366
8512
|
for (const warning2 of config.warnings) {
|
|
7367
8513
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
7368
8514
|
}
|
|
7369
|
-
const dbBytes = statSessionDbBytes(
|
|
8515
|
+
const dbBytes = statSessionDbBytes(homedir4());
|
|
7370
8516
|
void (async () => {
|
|
7371
8517
|
const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
7372
8518
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -7394,95 +8540,110 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7394
8540
|
);
|
|
7395
8541
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
7396
8542
|
}
|
|
7397
|
-
function
|
|
7398
|
-
|
|
7399
|
-
}
|
|
7400
|
-
function scheduleClaudeUsageReporting(state, options) {
|
|
7401
|
-
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
7402
|
-
options.claudeUsageReporting,
|
|
7403
|
-
process.env
|
|
7404
|
-
);
|
|
8543
|
+
function scheduleUsageReporting(state, params) {
|
|
8544
|
+
const { mode, warnings } = params.resolved;
|
|
7405
8545
|
for (const warning2 of warnings) {
|
|
7406
8546
|
logActivity(state, {
|
|
7407
8547
|
type: "info",
|
|
7408
8548
|
level: "warn",
|
|
7409
|
-
message:
|
|
8549
|
+
message: `${params.label} usage reporting: ${warning2}`
|
|
7410
8550
|
});
|
|
7411
8551
|
}
|
|
7412
8552
|
if (mode === "off") {
|
|
7413
8553
|
logActivity(state, {
|
|
7414
8554
|
type: "info",
|
|
7415
8555
|
level: "debug",
|
|
7416
|
-
message:
|
|
8556
|
+
message: `${params.label} usage reporting is off (${params.offFlagHint})`
|
|
7417
8557
|
});
|
|
7418
8558
|
return null;
|
|
7419
8559
|
}
|
|
7420
8560
|
let consecutiveFailures = 0;
|
|
7421
|
-
let
|
|
8561
|
+
let phase = "dormant";
|
|
7422
8562
|
let rearmRequested = false;
|
|
8563
|
+
const armProbe = () => {
|
|
8564
|
+
phase = "probe-pending";
|
|
8565
|
+
params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
|
|
8566
|
+
};
|
|
7423
8567
|
const scheduleNextTick = () => {
|
|
7424
|
-
|
|
7425
|
-
|
|
7426
|
-
|
|
8568
|
+
if (rearmRequested) {
|
|
8569
|
+
rearmRequested = false;
|
|
8570
|
+
armProbe();
|
|
8571
|
+
return;
|
|
8572
|
+
}
|
|
8573
|
+
phase = "steady-pending";
|
|
8574
|
+
params.setTimer(setTimeout(() => void tick(false), params.nextDelayMs()));
|
|
7427
8575
|
};
|
|
7428
8576
|
const rearm = () => {
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
8577
|
+
switch (phase) {
|
|
8578
|
+
case "tick-in-flight":
|
|
8579
|
+
rearmRequested = true;
|
|
8580
|
+
return;
|
|
8581
|
+
case "probe-pending":
|
|
8582
|
+
return;
|
|
8583
|
+
case "steady-pending":
|
|
8584
|
+
if (params.getTimer()) {
|
|
8585
|
+
clearTimeout(params.getTimer());
|
|
8586
|
+
params.setTimer(null);
|
|
8587
|
+
}
|
|
8588
|
+
rearmRequested = false;
|
|
8589
|
+
armProbe();
|
|
8590
|
+
return;
|
|
8591
|
+
case "dormant":
|
|
8592
|
+
rearmRequested = false;
|
|
8593
|
+
armProbe();
|
|
8594
|
+
return;
|
|
7432
8595
|
}
|
|
7433
|
-
rearmRequested = false;
|
|
7434
|
-
armed = true;
|
|
7435
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7436
8596
|
};
|
|
7437
8597
|
const tick = async (isProbe) => {
|
|
8598
|
+
phase = "tick-in-flight";
|
|
7438
8599
|
try {
|
|
7439
|
-
const usage = await
|
|
7440
|
-
const result = await
|
|
8600
|
+
const usage = await params.fetchUsage();
|
|
8601
|
+
const result = await params.report(usage);
|
|
7441
8602
|
if (result.ok) {
|
|
7442
8603
|
if (consecutiveFailures > 0) {
|
|
7443
8604
|
logActivity(state, {
|
|
7444
8605
|
type: "info",
|
|
7445
8606
|
level: "info",
|
|
7446
|
-
message:
|
|
8607
|
+
message: `${params.label} usage reporting recovered`
|
|
7447
8608
|
});
|
|
7448
8609
|
}
|
|
7449
8610
|
consecutiveFailures = 0;
|
|
7450
8611
|
logActivity(state, {
|
|
7451
8612
|
type: "info",
|
|
7452
8613
|
level: "debug",
|
|
7453
|
-
message:
|
|
8614
|
+
message: `Reported ${params.label} usage to Evident`
|
|
7454
8615
|
});
|
|
7455
8616
|
} else {
|
|
7456
8617
|
consecutiveFailures++;
|
|
7457
8618
|
logActivity(state, {
|
|
7458
8619
|
type: "info",
|
|
7459
|
-
level:
|
|
7460
|
-
message: `Failed to report
|
|
8620
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
8621
|
+
message: `Failed to report ${params.label} usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
7461
8622
|
});
|
|
7462
8623
|
}
|
|
7463
8624
|
scheduleNextTick();
|
|
7464
8625
|
} catch (error2) {
|
|
7465
|
-
if (
|
|
8626
|
+
if (params.isLocalCredentialProblem(error2)) {
|
|
7466
8627
|
if (mode === "on") {
|
|
7467
8628
|
logActivity(state, {
|
|
7468
8629
|
type: "info",
|
|
7469
8630
|
level: "warn",
|
|
7470
|
-
message:
|
|
8631
|
+
message: `${params.label} usage reporting is forced on but no usable login was found \u2014 ${params.forcedOnHint}; reporting will keep retrying`
|
|
7471
8632
|
});
|
|
7472
8633
|
scheduleNextTick();
|
|
7473
8634
|
} else if (isProbe) {
|
|
7474
8635
|
logActivity(state, {
|
|
7475
8636
|
type: "info",
|
|
7476
8637
|
level: "debug",
|
|
7477
|
-
message:
|
|
8638
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
7478
8639
|
});
|
|
7479
|
-
|
|
8640
|
+
phase = "dormant";
|
|
7480
8641
|
if (rearmRequested) rearm();
|
|
7481
8642
|
} else {
|
|
7482
8643
|
logActivity(state, {
|
|
7483
8644
|
type: "info",
|
|
7484
8645
|
level: "debug",
|
|
7485
|
-
message:
|
|
8646
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
7486
8647
|
});
|
|
7487
8648
|
scheduleNextTick();
|
|
7488
8649
|
}
|
|
@@ -7491,17 +8652,128 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7491
8652
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
7492
8653
|
logActivity(state, {
|
|
7493
8654
|
type: "info",
|
|
7494
|
-
level:
|
|
7495
|
-
message:
|
|
8655
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
8656
|
+
message: `${params.label} usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
7496
8657
|
});
|
|
7497
8658
|
scheduleNextTick();
|
|
7498
8659
|
}
|
|
7499
8660
|
}
|
|
7500
8661
|
};
|
|
7501
|
-
|
|
7502
|
-
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
8662
|
+
armProbe();
|
|
7503
8663
|
return rearm;
|
|
7504
8664
|
}
|
|
8665
|
+
function scheduleClaudeUsageReporting(state, options) {
|
|
8666
|
+
return scheduleUsageReporting(state, {
|
|
8667
|
+
label: "Claude",
|
|
8668
|
+
resolved: resolveClaudeUsageReportingMode(options.claudeUsageReporting, process.env),
|
|
8669
|
+
offFlagHint: "--claude-usage-reporting off",
|
|
8670
|
+
getTimer: () => state.claudeUsageTimer,
|
|
8671
|
+
setTimer: (timer) => {
|
|
8672
|
+
state.claudeUsageTimer = timer;
|
|
8673
|
+
},
|
|
8674
|
+
fetchUsage: async () => {
|
|
8675
|
+
const usage = await getClaudeUsage();
|
|
8676
|
+
if (usage.ownerLookupError) {
|
|
8677
|
+
logActivity(state, {
|
|
8678
|
+
type: "info",
|
|
8679
|
+
level: "debug",
|
|
8680
|
+
message: `Claude usage owner lookup failed: ${usage.ownerLookupError}`
|
|
8681
|
+
});
|
|
8682
|
+
}
|
|
8683
|
+
return usage;
|
|
8684
|
+
},
|
|
8685
|
+
report: (usage) => reportClaudeUsage(state.agentId, state.authHeader, usage),
|
|
8686
|
+
isLocalCredentialProblem,
|
|
8687
|
+
forcedOnHint: "run `claude` to sign in",
|
|
8688
|
+
firstDelayMs: () => FIRST_REPORT_DELAY_MS,
|
|
8689
|
+
nextDelayMs: nextReportDelayMs,
|
|
8690
|
+
failureLogLevel: claudeUsageFailureLogLevel
|
|
8691
|
+
});
|
|
8692
|
+
}
|
|
8693
|
+
var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
8694
|
+
var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
8695
|
+
var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
8696
|
+
function scheduleResourceUsageReporting(state, options) {
|
|
8697
|
+
const { enabled, warnings } = resolveResourceUsageReportingEnabled(
|
|
8698
|
+
options.resourceUsageReporting,
|
|
8699
|
+
process.env
|
|
8700
|
+
);
|
|
8701
|
+
for (const warning2 of warnings) {
|
|
8702
|
+
logActivity(state, {
|
|
8703
|
+
type: "info",
|
|
8704
|
+
level: "warn",
|
|
8705
|
+
message: `Resource usage reporting: ${warning2}`
|
|
8706
|
+
});
|
|
8707
|
+
}
|
|
8708
|
+
if (!enabled) {
|
|
8709
|
+
logActivity(state, {
|
|
8710
|
+
type: "info",
|
|
8711
|
+
level: "debug",
|
|
8712
|
+
message: "Resource usage reporting is off (--no-resource-usage-reporting)"
|
|
8713
|
+
});
|
|
8714
|
+
return;
|
|
8715
|
+
}
|
|
8716
|
+
const collect = createResourceUsageCollector(homedir4());
|
|
8717
|
+
let consecutiveFailures = 0;
|
|
8718
|
+
const tick = async () => {
|
|
8719
|
+
try {
|
|
8720
|
+
const { usage, warnings: collectWarnings } = await collect();
|
|
8721
|
+
for (const warning2 of collectWarnings) {
|
|
8722
|
+
logActivity(state, {
|
|
8723
|
+
type: "info",
|
|
8724
|
+
level: "debug",
|
|
8725
|
+
message: `Resource usage collection: ${warning2}`
|
|
8726
|
+
});
|
|
8727
|
+
}
|
|
8728
|
+
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
8729
|
+
if (result.ok) {
|
|
8730
|
+
if (consecutiveFailures > 0) {
|
|
8731
|
+
logActivity(state, {
|
|
8732
|
+
type: "info",
|
|
8733
|
+
level: "info",
|
|
8734
|
+
message: "Resource usage reporting recovered"
|
|
8735
|
+
});
|
|
8736
|
+
}
|
|
8737
|
+
consecutiveFailures = 0;
|
|
8738
|
+
logActivity(state, {
|
|
8739
|
+
type: "info",
|
|
8740
|
+
level: "debug",
|
|
8741
|
+
message: "Reported resource usage to Evident"
|
|
8742
|
+
});
|
|
8743
|
+
} else {
|
|
8744
|
+
consecutiveFailures++;
|
|
8745
|
+
logActivity(state, {
|
|
8746
|
+
type: "info",
|
|
8747
|
+
level: reportFailureLogLevel(
|
|
8748
|
+
consecutiveFailures,
|
|
8749
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8750
|
+
),
|
|
8751
|
+
message: `Failed to report resource usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
8752
|
+
});
|
|
8753
|
+
}
|
|
8754
|
+
} catch (error2) {
|
|
8755
|
+
consecutiveFailures++;
|
|
8756
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8757
|
+
logActivity(state, {
|
|
8758
|
+
type: "info",
|
|
8759
|
+
level: reportFailureLogLevel(
|
|
8760
|
+
consecutiveFailures,
|
|
8761
|
+
RESOURCE_USAGE_FAILURE_REESCALATION_TICKS
|
|
8762
|
+
),
|
|
8763
|
+
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
8764
|
+
});
|
|
8765
|
+
} finally {
|
|
8766
|
+
state.resourceUsageTimer = setTimeout(
|
|
8767
|
+
() => void tick(),
|
|
8768
|
+
jitteredDelayMs(
|
|
8769
|
+
RESOURCE_USAGE_BASE_REPORT_DELAY_MS,
|
|
8770
|
+
RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION
|
|
8771
|
+
)
|
|
8772
|
+
);
|
|
8773
|
+
}
|
|
8774
|
+
};
|
|
8775
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
8776
|
+
}
|
|
7505
8777
|
async function notifyOffline(state) {
|
|
7506
8778
|
if (!state.agentId || !state.authHeader) return;
|
|
7507
8779
|
if (!state.connected) {
|
|
@@ -7542,6 +8814,15 @@ async function cleanup(state, opts = {}) {
|
|
|
7542
8814
|
state.claudeUsageTimer = null;
|
|
7543
8815
|
}
|
|
7544
8816
|
state.claudeUsageRearm = null;
|
|
8817
|
+
if (state.openaiUsageTimer) {
|
|
8818
|
+
clearTimeout(state.openaiUsageTimer);
|
|
8819
|
+
state.openaiUsageTimer = null;
|
|
8820
|
+
}
|
|
8821
|
+
state.openaiUsageRearm = null;
|
|
8822
|
+
if (state.resourceUsageTimer) {
|
|
8823
|
+
clearTimeout(state.resourceUsageTimer);
|
|
8824
|
+
state.resourceUsageTimer = null;
|
|
8825
|
+
}
|
|
7545
8826
|
if (opts.graceful && state.channelDriver) {
|
|
7546
8827
|
state.channelDriver.stop();
|
|
7547
8828
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -7572,15 +8853,31 @@ async function cleanup(state, opts = {}) {
|
|
|
7572
8853
|
}
|
|
7573
8854
|
if (state.opencodeProcess) {
|
|
7574
8855
|
const opencodeProcess = state.opencodeProcess;
|
|
7575
|
-
|
|
8856
|
+
const result = await timeShutdownPhase(
|
|
8857
|
+
state,
|
|
8858
|
+
durations,
|
|
8859
|
+
"opencode_stop",
|
|
8860
|
+
() => stopOpenCodeAndWait(opencodeProcess, CHILD_STOP_TIMEOUT_MS)
|
|
8861
|
+
);
|
|
7576
8862
|
if (state.interactive) {
|
|
7577
|
-
logActivity(state, { type: "info", message:
|
|
8863
|
+
logActivity(state, { type: "info", message: `Stopped OpenCode process (${result.outcome})` });
|
|
7578
8864
|
displayStatus(state);
|
|
7579
8865
|
} else {
|
|
7580
|
-
log2(state,
|
|
8866
|
+
log2(state, `Stopped OpenCode process (${result.outcome})`);
|
|
7581
8867
|
}
|
|
7582
8868
|
state.opencodeProcess = null;
|
|
7583
8869
|
}
|
|
8870
|
+
if (state.litestreamProcess) {
|
|
8871
|
+
const litestreamProcess = state.litestreamProcess;
|
|
8872
|
+
const result = await timeShutdownPhase(
|
|
8873
|
+
state,
|
|
8874
|
+
durations,
|
|
8875
|
+
"litestream_stop",
|
|
8876
|
+
() => stopSessionDbReplication(litestreamProcess, CHILD_STOP_TIMEOUT_MS)
|
|
8877
|
+
);
|
|
8878
|
+
log2(state, `Stopped litestream replication (${result.outcome})`);
|
|
8879
|
+
state.litestreamProcess = null;
|
|
8880
|
+
}
|
|
7584
8881
|
return durations;
|
|
7585
8882
|
}
|
|
7586
8883
|
async function run(options) {
|
|
@@ -7589,7 +8886,7 @@ async function run(options) {
|
|
|
7589
8886
|
let fileSyncDirectories;
|
|
7590
8887
|
try {
|
|
7591
8888
|
logLevel = resolveLogLevel(options);
|
|
7592
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
8889
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir4());
|
|
7593
8890
|
} catch (error2) {
|
|
7594
8891
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
7595
8892
|
if (options.json) {
|
|
@@ -7614,6 +8911,7 @@ async function run(options) {
|
|
|
7614
8911
|
opencodeConnected: false,
|
|
7615
8912
|
opencodeVersion: null,
|
|
7616
8913
|
opencodeProcess: null,
|
|
8914
|
+
litestreamProcess: null,
|
|
7617
8915
|
connection: null,
|
|
7618
8916
|
channelDriver: null,
|
|
7619
8917
|
running: true,
|
|
@@ -7624,6 +8922,9 @@ async function run(options) {
|
|
|
7624
8922
|
sessionCleanupTimers: [],
|
|
7625
8923
|
claudeUsageTimer: null,
|
|
7626
8924
|
claudeUsageRearm: null,
|
|
8925
|
+
openaiUsageTimer: null,
|
|
8926
|
+
openaiUsageRearm: null,
|
|
8927
|
+
resourceUsageTimer: null,
|
|
7627
8928
|
authHeader: ""
|
|
7628
8929
|
};
|
|
7629
8930
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
@@ -7818,6 +9119,7 @@ async function run(options) {
|
|
|
7818
9119
|
} else {
|
|
7819
9120
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
7820
9121
|
}
|
|
9122
|
+
reportSessionDbRecovery(state);
|
|
7821
9123
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
7822
9124
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
7823
9125
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
@@ -7874,6 +9176,40 @@ async function run(options) {
|
|
|
7874
9176
|
ocSpinner?.fail(error2.message);
|
|
7875
9177
|
throw error2;
|
|
7876
9178
|
}
|
|
9179
|
+
if (options.litestreamConfig) {
|
|
9180
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
9181
|
+
state.litestreamProcess = litestreamProcess;
|
|
9182
|
+
let failureHandled = false;
|
|
9183
|
+
const failRunForReplication = (message) => {
|
|
9184
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
9185
|
+
failureHandled = true;
|
|
9186
|
+
state.shuttingDown = true;
|
|
9187
|
+
logActivity(state, { type: "error", error: message });
|
|
9188
|
+
if (state.interactive) displayStatus(state);
|
|
9189
|
+
void (async () => {
|
|
9190
|
+
try {
|
|
9191
|
+
await cleanup(state);
|
|
9192
|
+
await shutdownTelemetry();
|
|
9193
|
+
} catch (error2) {
|
|
9194
|
+
console.error(
|
|
9195
|
+
`[run] replication failure cleanup failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
9196
|
+
);
|
|
9197
|
+
}
|
|
9198
|
+
process.exit(1);
|
|
9199
|
+
})();
|
|
9200
|
+
};
|
|
9201
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
9202
|
+
failRunForReplication(
|
|
9203
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
9204
|
+
);
|
|
9205
|
+
});
|
|
9206
|
+
litestreamProcess.on("error", (error2) => {
|
|
9207
|
+
failRunForReplication(
|
|
9208
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
9209
|
+
);
|
|
9210
|
+
});
|
|
9211
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
9212
|
+
}
|
|
7877
9213
|
const tunnelSpinner = interactive && !state.json ? ora3("Connecting tunnel...").start() : null;
|
|
7878
9214
|
const channelDriver = new ChannelDriver({
|
|
7879
9215
|
agentId: state.agentId,
|
|
@@ -7885,7 +9221,7 @@ async function run(options) {
|
|
|
7885
9221
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
7886
9222
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
7887
9223
|
fileSyncDirectories,
|
|
7888
|
-
homeDir:
|
|
9224
|
+
homeDir: homedir4(),
|
|
7889
9225
|
maxActiveSessions,
|
|
7890
9226
|
log: (entry) => (
|
|
7891
9227
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -8023,6 +9359,23 @@ async function run(options) {
|
|
|
8023
9359
|
}
|
|
8024
9360
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
8025
9361
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
9362
|
+
state.openaiUsageRearm = scheduleUsageReporting(state, {
|
|
9363
|
+
label: "OpenAI",
|
|
9364
|
+
resolved: resolveOpenAiUsageReportingMode(options.openaiUsageReporting, process.env),
|
|
9365
|
+
offFlagHint: "--openai-usage-reporting off",
|
|
9366
|
+
getTimer: () => state.openaiUsageTimer,
|
|
9367
|
+
setTimer: (timer) => {
|
|
9368
|
+
state.openaiUsageTimer = timer;
|
|
9369
|
+
},
|
|
9370
|
+
fetchUsage: () => getOpenAiUsage(state.port),
|
|
9371
|
+
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
9372
|
+
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
9373
|
+
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
9374
|
+
firstDelayMs: firstReportDelayMs,
|
|
9375
|
+
nextDelayMs: usageReportDelayMs,
|
|
9376
|
+
failureLogLevel: usageReportFailureLogLevel
|
|
9377
|
+
});
|
|
9378
|
+
scheduleResourceUsageReporting(state, options);
|
|
8026
9379
|
if (!interactive || state.json) {
|
|
8027
9380
|
log2(state, "Driving channel messages...");
|
|
8028
9381
|
}
|
|
@@ -8103,6 +9456,12 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8103
9456
|
).option(
|
|
8104
9457
|
"--claude-usage-reporting <mode>",
|
|
8105
9458
|
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
9459
|
+
).option(
|
|
9460
|
+
"--openai-usage-reporting <mode>",
|
|
9461
|
+
"Report OpenAI plan usage to Evident: auto | on | off (default: auto). Env: EVIDENT_OPENAI_USAGE_REPORTING"
|
|
9462
|
+
).option(
|
|
9463
|
+
"--no-resource-usage-reporting",
|
|
9464
|
+
"Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
|
|
8106
9465
|
).option(
|
|
8107
9466
|
"--enable-file-sync-to <dir>",
|
|
8108
9467
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
@@ -8111,6 +9470,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8111
9470
|
).option(
|
|
8112
9471
|
"--tunnel-ready-file <path>",
|
|
8113
9472
|
"Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
|
|
9473
|
+
).option(
|
|
9474
|
+
"--litestream-config <path>",
|
|
9475
|
+
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
8114
9476
|
).action(
|
|
8115
9477
|
(options) => {
|
|
8116
9478
|
run({
|
|
@@ -8135,10 +9497,15 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8135
9497
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
8136
9498
|
// (resolveClaudeUsageReportingMode).
|
|
8137
9499
|
claudeUsageReporting: options.claudeUsageReporting,
|
|
9500
|
+
openaiUsageReporting: options.openaiUsageReporting,
|
|
9501
|
+
// Raw value — resolution is single-sourced in run.ts's
|
|
9502
|
+
// resolveResourceUsageReportingEnabled.
|
|
9503
|
+
resourceUsageReporting: options.resourceUsageReporting,
|
|
8138
9504
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
8139
9505
|
// resolveFileSyncDirectories.
|
|
8140
9506
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
8141
|
-
tunnelReadyFile: options.tunnelReadyFile
|
|
9507
|
+
tunnelReadyFile: options.tunnelReadyFile,
|
|
9508
|
+
litestreamConfig: options.litestreamConfig
|
|
8142
9509
|
});
|
|
8143
9510
|
}
|
|
8144
9511
|
);
|