@evident-ai/cli 3.4.1-dev.8fa4d29 → 3.4.1-dev.9c22b93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -1
- package/dist/index.js +2116 -157
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire } from "module";
|
|
4
|
+
import { createRequire as createRequire2 } from "module";
|
|
5
5
|
import { Command } from "commander";
|
|
6
6
|
|
|
7
7
|
// src/commands/login.ts
|
|
@@ -371,14 +371,14 @@ function blank() {
|
|
|
371
371
|
console.log();
|
|
372
372
|
}
|
|
373
373
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
374
|
-
return new Promise((
|
|
374
|
+
return new Promise((resolve4) => {
|
|
375
375
|
process.stdout.write(chalk.dim(prompt));
|
|
376
376
|
const handler = () => {
|
|
377
377
|
process.stdin.removeListener("data", handler);
|
|
378
378
|
process.stdin.setRawMode?.(false);
|
|
379
379
|
process.stdin.pause();
|
|
380
380
|
console.log();
|
|
381
|
-
|
|
381
|
+
resolve4();
|
|
382
382
|
};
|
|
383
383
|
if (process.stdin.isTTY) {
|
|
384
384
|
process.stdin.setRawMode?.(true);
|
|
@@ -388,7 +388,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
388
388
|
});
|
|
389
389
|
}
|
|
390
390
|
function sleep(ms) {
|
|
391
|
-
return new Promise((
|
|
391
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
392
392
|
}
|
|
393
393
|
|
|
394
394
|
// src/commands/login.ts
|
|
@@ -466,19 +466,19 @@ async function tokenLogin() {
|
|
|
466
466
|
);
|
|
467
467
|
blank();
|
|
468
468
|
process.stdout.write("Paste token: ");
|
|
469
|
-
const token = await new Promise((
|
|
469
|
+
const token = await new Promise((resolve4) => {
|
|
470
470
|
let data = "";
|
|
471
471
|
process.stdin.setEncoding("utf8");
|
|
472
472
|
process.stdin.on("data", (chunk) => {
|
|
473
473
|
data += chunk;
|
|
474
474
|
});
|
|
475
475
|
process.stdin.on("end", () => {
|
|
476
|
-
|
|
476
|
+
resolve4(data.trim());
|
|
477
477
|
});
|
|
478
478
|
if (process.stdin.isTTY) {
|
|
479
479
|
process.stdin.once("data", (chunk) => {
|
|
480
480
|
process.stdin.pause();
|
|
481
|
-
|
|
481
|
+
resolve4(chunk.toString().trim());
|
|
482
482
|
});
|
|
483
483
|
process.stdin.resume();
|
|
484
484
|
}
|
|
@@ -722,6 +722,14 @@ function toReportedWindow(window) {
|
|
|
722
722
|
if (!window) return null;
|
|
723
723
|
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
724
724
|
}
|
|
725
|
+
function toReportedOwner(snapshot) {
|
|
726
|
+
if (!snapshot.owner) return null;
|
|
727
|
+
return {
|
|
728
|
+
email: snapshot.owner.email,
|
|
729
|
+
organization_name: snapshot.owner.organizationName,
|
|
730
|
+
rate_limit_tier: snapshot.owner.rateLimitTier
|
|
731
|
+
};
|
|
732
|
+
}
|
|
725
733
|
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
726
734
|
try {
|
|
727
735
|
const apiUrl = getApiUrlConfig();
|
|
@@ -730,7 +738,42 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
730
738
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
731
739
|
body: JSON.stringify({
|
|
732
740
|
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
733
|
-
seven_day: toReportedWindow(snapshot.sevenDay)
|
|
741
|
+
seven_day: toReportedWindow(snapshot.sevenDay),
|
|
742
|
+
owner: toReportedOwner(snapshot)
|
|
743
|
+
}),
|
|
744
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
745
|
+
});
|
|
746
|
+
if (!response.ok) {
|
|
747
|
+
const serverMessage = await readErrorMessage(response);
|
|
748
|
+
return {
|
|
749
|
+
ok: false,
|
|
750
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
return { ok: true };
|
|
754
|
+
} catch (error2) {
|
|
755
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
function toReportedOpenAiWindow(window) {
|
|
759
|
+
if (!window) return null;
|
|
760
|
+
return {
|
|
761
|
+
utilization: window.utilization,
|
|
762
|
+
window_minutes: window.windowMinutes,
|
|
763
|
+
resets_at: window.resetsAt
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
767
|
+
try {
|
|
768
|
+
const apiUrl = getApiUrlConfig();
|
|
769
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/openai-usage`, {
|
|
770
|
+
method: "POST",
|
|
771
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
772
|
+
body: JSON.stringify({
|
|
773
|
+
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
774
|
+
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
775
|
+
has_credits: snapshot.hasCredits,
|
|
776
|
+
credits_unlimited: snapshot.creditsUnlimited
|
|
734
777
|
}),
|
|
735
778
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
736
779
|
});
|
|
@@ -962,7 +1005,10 @@ import { readFileSync } from "fs";
|
|
|
962
1005
|
import { homedir } from "os";
|
|
963
1006
|
import { join } from "path";
|
|
964
1007
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
1008
|
+
var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
|
|
1009
|
+
var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
|
|
965
1010
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
1011
|
+
var cachedOwner = null;
|
|
966
1012
|
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
967
1013
|
function parseClaudeCliCredentials(raw) {
|
|
968
1014
|
let parsed;
|
|
@@ -1036,6 +1082,47 @@ function toWindow(value) {
|
|
|
1036
1082
|
}
|
|
1037
1083
|
return { utilization: window.utilization, resetsAt };
|
|
1038
1084
|
}
|
|
1085
|
+
function ownerLookupFailure(error2) {
|
|
1086
|
+
const name = error2?.name;
|
|
1087
|
+
return name === "TimeoutError" || name === "AbortError" ? "timed out" : "request failed";
|
|
1088
|
+
}
|
|
1089
|
+
async function getClaudeUsageOwner(accessToken) {
|
|
1090
|
+
if (cachedOwner?.accessToken === accessToken) {
|
|
1091
|
+
return { owner: cachedOwner.owner, ownerLookupError: null };
|
|
1092
|
+
}
|
|
1093
|
+
try {
|
|
1094
|
+
const response = await fetch(CLAUDE_PROFILE_URL, {
|
|
1095
|
+
headers: {
|
|
1096
|
+
Authorization: `Bearer ${accessToken}`,
|
|
1097
|
+
"Content-Type": "application/json",
|
|
1098
|
+
"anthropic-version": "2023-06-01"
|
|
1099
|
+
},
|
|
1100
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1101
|
+
});
|
|
1102
|
+
if (!response.ok) {
|
|
1103
|
+
return { owner: null, ownerLookupError: `HTTP ${response.status}` };
|
|
1104
|
+
}
|
|
1105
|
+
let body;
|
|
1106
|
+
try {
|
|
1107
|
+
body = await response.json();
|
|
1108
|
+
} catch (error2) {
|
|
1109
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1110
|
+
}
|
|
1111
|
+
const profile = body;
|
|
1112
|
+
if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
|
|
1113
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1114
|
+
}
|
|
1115
|
+
const owner = {
|
|
1116
|
+
email: profile.account.email,
|
|
1117
|
+
organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
|
|
1118
|
+
rateLimitTier: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
|
|
1119
|
+
};
|
|
1120
|
+
cachedOwner = { accessToken, owner };
|
|
1121
|
+
return { owner, ownerLookupError: null };
|
|
1122
|
+
} catch (error2) {
|
|
1123
|
+
return { owner: null, ownerLookupError: ownerLookupFailure(error2) };
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1039
1126
|
async function getClaudeUsage() {
|
|
1040
1127
|
const credentials2 = readClaudeCliCredentials();
|
|
1041
1128
|
if (!credentials2) {
|
|
@@ -1055,15 +1142,19 @@ async function getClaudeUsage() {
|
|
|
1055
1142
|
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
1056
1143
|
"Content-Type": "application/json",
|
|
1057
1144
|
"anthropic-version": "2023-06-01"
|
|
1058
|
-
}
|
|
1145
|
+
},
|
|
1146
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1059
1147
|
});
|
|
1060
1148
|
if (!res.ok) {
|
|
1061
1149
|
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
1062
1150
|
}
|
|
1063
1151
|
const body = await res.json();
|
|
1152
|
+
const { owner, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
|
|
1064
1153
|
return {
|
|
1065
1154
|
fiveHour: toWindow(body.five_hour),
|
|
1066
|
-
sevenDay: toWindow(body.seven_day)
|
|
1155
|
+
sevenDay: toWindow(body.seven_day),
|
|
1156
|
+
owner,
|
|
1157
|
+
ownerLookupError
|
|
1067
1158
|
};
|
|
1068
1159
|
}
|
|
1069
1160
|
|
|
@@ -1092,8 +1183,9 @@ async function claudeUsage() {
|
|
|
1092
1183
|
}
|
|
1093
1184
|
|
|
1094
1185
|
// src/commands/run.ts
|
|
1095
|
-
import {
|
|
1096
|
-
import {
|
|
1186
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync5 } from "fs";
|
|
1187
|
+
import { homedir as homedir5 } from "os";
|
|
1188
|
+
import { isAbsolute as isAbsolute3, join as join9, parse, resolve as resolvePath2 } from "path";
|
|
1097
1189
|
import chalk6 from "chalk";
|
|
1098
1190
|
|
|
1099
1191
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1326,6 +1418,8 @@ var SEVERITY_BY_LEVEL = {
|
|
|
1326
1418
|
error: "error"
|
|
1327
1419
|
};
|
|
1328
1420
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1421
|
+
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1422
|
+
var MAX_METADATA_ENTRIES = 20;
|
|
1329
1423
|
var TRUNCATION_MARKER = "\u2026";
|
|
1330
1424
|
function redact(message) {
|
|
1331
1425
|
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
@@ -1334,6 +1428,24 @@ function truncate(message) {
|
|
|
1334
1428
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1335
1429
|
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
1336
1430
|
}
|
|
1431
|
+
function sanitiseMetadata(metadata) {
|
|
1432
|
+
if (!metadata) return {};
|
|
1433
|
+
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1434
|
+
if (Object.keys(metadata).length > entries.length) {
|
|
1435
|
+
console.error(
|
|
1436
|
+
`[runner-activity-telemetry] dropped ${Object.keys(metadata).length - entries.length} metadata entries`
|
|
1437
|
+
);
|
|
1438
|
+
}
|
|
1439
|
+
const sanitised = [];
|
|
1440
|
+
for (const [key, value] of entries) {
|
|
1441
|
+
if (typeof value === "string") {
|
|
1442
|
+
sanitised.push([key, truncate(redact(value).slice(0, MAX_METADATA_VALUE_LENGTH))]);
|
|
1443
|
+
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1444
|
+
sanitised.push([key, value]);
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
return Object.fromEntries(sanitised);
|
|
1448
|
+
}
|
|
1337
1449
|
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1338
1450
|
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1339
1451
|
var windowStartedAt = 0;
|
|
@@ -1372,7 +1484,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1372
1484
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1373
1485
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1374
1486
|
message,
|
|
1375
|
-
metadata: { source: "cli.run" },
|
|
1487
|
+
metadata: { ...sanitiseMetadata(entry.metadata), source: "cli.run" },
|
|
1376
1488
|
agentId: context.agentId
|
|
1377
1489
|
});
|
|
1378
1490
|
} catch (err) {
|
|
@@ -1382,6 +1494,191 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1382
1494
|
}
|
|
1383
1495
|
}
|
|
1384
1496
|
|
|
1497
|
+
// src/lib/opencode/session-db-recovery-report.ts
|
|
1498
|
+
import { readFileSync as readFileSync2, unlinkSync } from "fs";
|
|
1499
|
+
import { join as join2 } from "path";
|
|
1500
|
+
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1501
|
+
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1502
|
+
return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1503
|
+
}
|
|
1504
|
+
function drainSessionDbRecoveryReport({
|
|
1505
|
+
homeDir,
|
|
1506
|
+
env
|
|
1507
|
+
}) {
|
|
1508
|
+
const path = sessionDbRecoveryReportPath(homeDir, env);
|
|
1509
|
+
let content;
|
|
1510
|
+
try {
|
|
1511
|
+
content = readFileSync2(path, "utf8");
|
|
1512
|
+
} catch (error2) {
|
|
1513
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")
|
|
1514
|
+
return { path, records: [], skippedLines: 0, readError: null };
|
|
1515
|
+
const readError = error2 instanceof Error ? error2.message : String(error2);
|
|
1516
|
+
console.error(`[session-db-recovery-report] could not read ${path}: ${readError}`);
|
|
1517
|
+
return { path, records: [], skippedLines: 0, readError };
|
|
1518
|
+
}
|
|
1519
|
+
let skippedLines = 0;
|
|
1520
|
+
const records = content.split("\n").flatMap((line) => {
|
|
1521
|
+
if (!line.trim()) return [];
|
|
1522
|
+
try {
|
|
1523
|
+
const value = JSON.parse(line);
|
|
1524
|
+
if (!isSessionDbRecoveryRecord(value)) {
|
|
1525
|
+
skippedLines++;
|
|
1526
|
+
return [];
|
|
1527
|
+
}
|
|
1528
|
+
return [
|
|
1529
|
+
{
|
|
1530
|
+
...value,
|
|
1531
|
+
provenance_reason: value.provenance_reason ?? null,
|
|
1532
|
+
provenance_migration_delta: value.provenance_migration_delta ?? null,
|
|
1533
|
+
replication_suspended: value.replication_suspended ?? false
|
|
1534
|
+
}
|
|
1535
|
+
];
|
|
1536
|
+
} catch (error2) {
|
|
1537
|
+
skippedLines++;
|
|
1538
|
+
console.error(
|
|
1539
|
+
`[session-db-recovery-report] skipped malformed record in ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1540
|
+
);
|
|
1541
|
+
return [];
|
|
1542
|
+
}
|
|
1543
|
+
});
|
|
1544
|
+
return { path, records, skippedLines, readError: null };
|
|
1545
|
+
}
|
|
1546
|
+
function acknowledgeSessionDbRecoveryReport(path) {
|
|
1547
|
+
try {
|
|
1548
|
+
unlinkSync(path);
|
|
1549
|
+
} catch (error2) {
|
|
1550
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1551
|
+
console.error(
|
|
1552
|
+
`[session-db-recovery-report] could not remove ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1553
|
+
);
|
|
1554
|
+
}
|
|
1555
|
+
}
|
|
1556
|
+
function buildSessionDbRecoveryActivity(record) {
|
|
1557
|
+
const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
|
|
1558
|
+
if (!level) return null;
|
|
1559
|
+
const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
|
|
1560
|
+
const replication = record.replication_suspended ? " This start is not backing up its new session history; restart after fixing the cause." : "";
|
|
1561
|
+
const giveupMessage = (() => {
|
|
1562
|
+
switch (record.reason) {
|
|
1563
|
+
case "restore_deadline_exceeded":
|
|
1564
|
+
return "The restore did not finish before this startup deadline. Earlier sessions are unavailable for this start; collect the SESSION-DB-RESTORE-TRUNCATED boot log before retrying.";
|
|
1565
|
+
case "restore_tool_unusable":
|
|
1566
|
+
case "classification_unrecognised":
|
|
1567
|
+
return "The restore or classification tool produced an unsupported result. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1568
|
+
case "synchroniser_config_unevaluable":
|
|
1569
|
+
case "synchroniser_config_incomplete":
|
|
1570
|
+
return "The installed hook and synchroniser bundle could not agree on configuration. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1571
|
+
case "synchroniser_config_unresolved":
|
|
1572
|
+
return "The synchroniser configuration could not be resolved. Inspect the preceding boot error and repair the identified runner image, bundle, or local environment before restarting.";
|
|
1573
|
+
case "litestream_config_unavailable":
|
|
1574
|
+
return "The local Litestream configuration could not be generated or written. Inspect the preceding boot error and repair the runner image, bundle, local environment, or write permissions before restarting.";
|
|
1575
|
+
case "classification_fatal":
|
|
1576
|
+
return "Session history could not be restored because backup setup could not be established. Inspect the named boot marker or log and repair the identified backup configuration or permissions.";
|
|
1577
|
+
default:
|
|
1578
|
+
return null;
|
|
1579
|
+
}
|
|
1580
|
+
})();
|
|
1581
|
+
if (giveupMessage)
|
|
1582
|
+
return {
|
|
1583
|
+
level,
|
|
1584
|
+
metadata: withoutContractFields(record),
|
|
1585
|
+
message: `${giveupMessage}${replication}`
|
|
1586
|
+
};
|
|
1587
|
+
switch (record.outcome) {
|
|
1588
|
+
case "fresh_session_db":
|
|
1589
|
+
return {
|
|
1590
|
+
level,
|
|
1591
|
+
metadata: withoutContractFields(record),
|
|
1592
|
+
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.${replication}`
|
|
1593
|
+
};
|
|
1594
|
+
case "restore_retried":
|
|
1595
|
+
return {
|
|
1596
|
+
level,
|
|
1597
|
+
metadata: withoutContractFields(record),
|
|
1598
|
+
message: "The initial session database restore failed; the runner will retry it. Monitor the runner backup for another restore failure."
|
|
1599
|
+
};
|
|
1600
|
+
case "replica_recovered":
|
|
1601
|
+
if (record.reason === "quarantine")
|
|
1602
|
+
return {
|
|
1603
|
+
level,
|
|
1604
|
+
metadata: withoutContractFields(record),
|
|
1605
|
+
message: `Session database recovery quarantined ${record.quarantined_objects ?? "unknown"} objects (${record.quarantined_bytes ?? "unknown"} bytes); ${record.quarantine_failed_objects ?? "unknown"} moves failed. Review the preserved backup at ${record.quarantine_destination ?? "an unknown destination"} before deleting it.`
|
|
1606
|
+
};
|
|
1607
|
+
if (record.reason === "prune")
|
|
1608
|
+
return {
|
|
1609
|
+
level,
|
|
1610
|
+
metadata: withoutContractFields(record),
|
|
1611
|
+
message: "Session database recovery discarded a damaged newest backup and retried. Sessions recorded after the previous backup point may be unavailable. Review the runner backup for another restore failure."
|
|
1612
|
+
};
|
|
1613
|
+
if (record.reason === "clear")
|
|
1614
|
+
return {
|
|
1615
|
+
level,
|
|
1616
|
+
metadata: withoutContractFields(record),
|
|
1617
|
+
message: "Session database recovery deleted the damaged backup and prior session history is unavailable. Review the runner backup configuration before relying on restored session history."
|
|
1618
|
+
};
|
|
1619
|
+
return null;
|
|
1620
|
+
case "history_rolled_back":
|
|
1621
|
+
return {
|
|
1622
|
+
level,
|
|
1623
|
+
metadata: withoutContractFields(record),
|
|
1624
|
+
message: `Session history was rolled back to verified restore point ${record.verified_restore_point ?? "unknown"}; everything after it is unavailable. Review the runner backup for another restore failure.`
|
|
1625
|
+
};
|
|
1626
|
+
case "restore_misconfigured":
|
|
1627
|
+
return {
|
|
1628
|
+
level,
|
|
1629
|
+
metadata: withoutContractFields(record),
|
|
1630
|
+
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1631
|
+
};
|
|
1632
|
+
case "session_db_boot_refused":
|
|
1633
|
+
return {
|
|
1634
|
+
level,
|
|
1635
|
+
metadata: withoutContractFields(record),
|
|
1636
|
+
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1637
|
+
};
|
|
1638
|
+
case "schema_provenance_mismatch":
|
|
1639
|
+
return {
|
|
1640
|
+
level,
|
|
1641
|
+
metadata: withoutContractFields(record),
|
|
1642
|
+
message: `Session database schema provenance mismatch for ${record.dbPath ?? record.db_path ?? "unknown"}: recorded version=${record.recorded_version ?? "unknown"}, current version=${record.current_version ?? "unknown"}, reason=${record.provenance_reason ?? "unknown"}, migration delta=${record.provenance_migration_delta ?? "unknown"}. Inspect the session database and runner backup before continuing.`
|
|
1643
|
+
};
|
|
1644
|
+
default:
|
|
1645
|
+
return null;
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
function withoutContractFields(record) {
|
|
1649
|
+
const { v: _v, event: _event, ...metadata } = record;
|
|
1650
|
+
return metadata;
|
|
1651
|
+
}
|
|
1652
|
+
var OUTCOMES = /* @__PURE__ */ new Set([
|
|
1653
|
+
"replica_recovered",
|
|
1654
|
+
"restore_retried",
|
|
1655
|
+
"fresh_session_db",
|
|
1656
|
+
"history_rolled_back",
|
|
1657
|
+
"restore_misconfigured",
|
|
1658
|
+
"session_db_boot_refused",
|
|
1659
|
+
"schema_provenance_mismatch"
|
|
1660
|
+
]);
|
|
1661
|
+
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1662
|
+
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
1663
|
+
var NUMBER_FIELDS = [
|
|
1664
|
+
"litestream_exit_code",
|
|
1665
|
+
"attempt",
|
|
1666
|
+
"replica_objects",
|
|
1667
|
+
"replica_bytes",
|
|
1668
|
+
"quarantined_objects",
|
|
1669
|
+
"quarantine_failed_objects",
|
|
1670
|
+
"quarantined_bytes",
|
|
1671
|
+
"restore_points_tried"
|
|
1672
|
+
];
|
|
1673
|
+
var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"];
|
|
1674
|
+
function isSessionDbRecoveryRecord(value) {
|
|
1675
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1676
|
+
const record = value;
|
|
1677
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && (record.provenance_reason === void 0 || record.provenance_reason === null || typeof record.provenance_reason === "string") && (record.provenance_migration_delta === void 0 || record.provenance_migration_delta === null || Number.isInteger(record.provenance_migration_delta)) && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1678
|
+
(field) => record[field] === null || typeof record[field] === "string"
|
|
1679
|
+
);
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1385
1682
|
// src/lib/opencode/health.ts
|
|
1386
1683
|
async function checkOpenCodeHealth(port) {
|
|
1387
1684
|
try {
|
|
@@ -1406,11 +1703,644 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1406
1703
|
if (health.healthy) {
|
|
1407
1704
|
return health;
|
|
1408
1705
|
}
|
|
1409
|
-
await new Promise((
|
|
1706
|
+
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
1410
1707
|
}
|
|
1411
1708
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1412
1709
|
}
|
|
1413
1710
|
|
|
1711
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1712
|
+
import { spawn as spawn2 } from "child_process";
|
|
1713
|
+
import { mkdirSync, statSync as statSync2, unlinkSync as unlinkSync2, writeFileSync } from "fs";
|
|
1714
|
+
import { homedir as homedir2 } from "os";
|
|
1715
|
+
import { dirname as dirname2, resolve as resolvePath } from "path";
|
|
1716
|
+
|
|
1717
|
+
// src/lib/runner-synchroniser.ts
|
|
1718
|
+
import { spawn } from "child_process";
|
|
1719
|
+
function appendError(stderr, error2) {
|
|
1720
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1721
|
+
return stderr === "" ? message : `${stderr}
|
|
1722
|
+
${message}`;
|
|
1723
|
+
}
|
|
1724
|
+
function runSynchroniser(args, opts) {
|
|
1725
|
+
return new Promise((resolve4) => {
|
|
1726
|
+
let child;
|
|
1727
|
+
let stdout = "";
|
|
1728
|
+
let stderr = "";
|
|
1729
|
+
let settled = false;
|
|
1730
|
+
const timer = {};
|
|
1731
|
+
const finish = (result) => {
|
|
1732
|
+
if (settled) return;
|
|
1733
|
+
settled = true;
|
|
1734
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
1735
|
+
resolve4(result);
|
|
1736
|
+
};
|
|
1737
|
+
try {
|
|
1738
|
+
child = spawn("runner-synchroniser", args, {
|
|
1739
|
+
env: opts.env ?? process.env,
|
|
1740
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1741
|
+
});
|
|
1742
|
+
} catch (error2) {
|
|
1743
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1744
|
+
return;
|
|
1745
|
+
}
|
|
1746
|
+
child.stdout?.setEncoding("utf8");
|
|
1747
|
+
child.stdout?.on("data", (chunk) => {
|
|
1748
|
+
stdout += chunk;
|
|
1749
|
+
});
|
|
1750
|
+
child.stderr?.setEncoding("utf8");
|
|
1751
|
+
child.stderr?.on("data", (chunk) => {
|
|
1752
|
+
stderr += chunk;
|
|
1753
|
+
});
|
|
1754
|
+
child.once("error", (error2) => {
|
|
1755
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1756
|
+
});
|
|
1757
|
+
child.once("close", (code) => {
|
|
1758
|
+
finish({ code, stdout, stderr, timedOut: false });
|
|
1759
|
+
});
|
|
1760
|
+
timer.handle = setTimeout(
|
|
1761
|
+
() => {
|
|
1762
|
+
child.kill("SIGKILL");
|
|
1763
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1764
|
+
},
|
|
1765
|
+
Math.max(0, opts.timeoutMs)
|
|
1766
|
+
);
|
|
1767
|
+
});
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1771
|
+
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
1772
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
|
|
1773
|
+
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
1774
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
|
|
1775
|
+
function commandError(result) {
|
|
1776
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
1777
|
+
}
|
|
1778
|
+
function reportRecord(stage, outcome, reason, litestreamExitCode, options) {
|
|
1779
|
+
options.reportRecovery({
|
|
1780
|
+
v: 1,
|
|
1781
|
+
event: "session_db_recovery",
|
|
1782
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1783
|
+
stage,
|
|
1784
|
+
outcome,
|
|
1785
|
+
severity: "error",
|
|
1786
|
+
reason,
|
|
1787
|
+
litestream_exit_code: litestreamExitCode,
|
|
1788
|
+
attempt: null,
|
|
1789
|
+
replica_objects: null,
|
|
1790
|
+
replica_bytes: null,
|
|
1791
|
+
quarantine_destination: null,
|
|
1792
|
+
quarantined_objects: null,
|
|
1793
|
+
quarantine_failed_objects: null,
|
|
1794
|
+
quarantined_bytes: null,
|
|
1795
|
+
verified_restore_point: null,
|
|
1796
|
+
restore_points_tried: null,
|
|
1797
|
+
provenance_reason: null,
|
|
1798
|
+
provenance_migration_delta: null,
|
|
1799
|
+
replication_suspended: stage === "restore"
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
function clearMarker(options) {
|
|
1803
|
+
if (!options.noReplicateMarker) return;
|
|
1804
|
+
try {
|
|
1805
|
+
unlinkSync2(options.noReplicateMarker);
|
|
1806
|
+
} catch (error2) {
|
|
1807
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1808
|
+
options.log(
|
|
1809
|
+
`Could not clear the previous session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1810
|
+
"warn"
|
|
1811
|
+
);
|
|
1812
|
+
}
|
|
1813
|
+
}
|
|
1814
|
+
function markNoReplicate(options, message) {
|
|
1815
|
+
if (options.noReplicateMarker) {
|
|
1816
|
+
try {
|
|
1817
|
+
mkdirSync(dirname2(options.noReplicateMarker), { recursive: true });
|
|
1818
|
+
writeFileSync(options.noReplicateMarker, "");
|
|
1819
|
+
} catch (error2) {
|
|
1820
|
+
options.log(
|
|
1821
|
+
`Could not write session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1822
|
+
"error"
|
|
1823
|
+
);
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
options.log(`SESSION-DB-NO-REPLICATE: ${message}`, "warn");
|
|
1827
|
+
}
|
|
1828
|
+
function discardSessionDbDebris(options) {
|
|
1829
|
+
for (const path of [options.dbPath, `${options.dbPath}-wal`, `${options.dbPath}-shm`]) {
|
|
1830
|
+
try {
|
|
1831
|
+
unlinkSync2(path);
|
|
1832
|
+
} catch (error2) {
|
|
1833
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") continue;
|
|
1834
|
+
options.log(
|
|
1835
|
+
`Could not remove session-DB debris ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1836
|
+
"warn"
|
|
1837
|
+
);
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
function splitDiagnostics(text) {
|
|
1842
|
+
return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
1843
|
+
}
|
|
1844
|
+
function logSynchroniserDiagnostics(result, options) {
|
|
1845
|
+
for (const line of splitDiagnostics(result.stderr)) options.log(line, "warn");
|
|
1846
|
+
}
|
|
1847
|
+
function parseSingleQuotedAssignment(line) {
|
|
1848
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
1849
|
+
if (!match || !match[2].startsWith("'")) return null;
|
|
1850
|
+
const valueSource = match[2];
|
|
1851
|
+
let value = "";
|
|
1852
|
+
for (let index = 1; index < valueSource.length; index++) {
|
|
1853
|
+
const character = valueSource[index];
|
|
1854
|
+
if (character !== "'") {
|
|
1855
|
+
value += character;
|
|
1856
|
+
continue;
|
|
1857
|
+
}
|
|
1858
|
+
if (index === valueSource.length - 1) return [match[1], value];
|
|
1859
|
+
if (valueSource.slice(index + 1, index + 4) !== "\\''") return null;
|
|
1860
|
+
value += "'";
|
|
1861
|
+
index += 3;
|
|
1862
|
+
}
|
|
1863
|
+
return null;
|
|
1864
|
+
}
|
|
1865
|
+
function parseSynchroniserEnv(stdout) {
|
|
1866
|
+
const values = {};
|
|
1867
|
+
for (const line of stdout.split("\n")) {
|
|
1868
|
+
if (line.trim() === "") continue;
|
|
1869
|
+
const assignment = parseSingleQuotedAssignment(line);
|
|
1870
|
+
if (!assignment) return null;
|
|
1871
|
+
values[assignment[0]] = assignment[1];
|
|
1872
|
+
}
|
|
1873
|
+
return values;
|
|
1874
|
+
}
|
|
1875
|
+
function runCommand(command, args, options) {
|
|
1876
|
+
return new Promise((resolve4) => {
|
|
1877
|
+
let child;
|
|
1878
|
+
let stdout = "";
|
|
1879
|
+
let stderr = "";
|
|
1880
|
+
let settled = false;
|
|
1881
|
+
const finish = (result) => {
|
|
1882
|
+
if (settled) return;
|
|
1883
|
+
settled = true;
|
|
1884
|
+
if (timer) clearTimeout(timer);
|
|
1885
|
+
resolve4(result);
|
|
1886
|
+
};
|
|
1887
|
+
try {
|
|
1888
|
+
child = spawn2(command, args, {
|
|
1889
|
+
env: options.env,
|
|
1890
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1891
|
+
});
|
|
1892
|
+
} catch (error2) {
|
|
1893
|
+
resolve4({
|
|
1894
|
+
code: null,
|
|
1895
|
+
stdout,
|
|
1896
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
1897
|
+
timedOut: false
|
|
1898
|
+
});
|
|
1899
|
+
return;
|
|
1900
|
+
}
|
|
1901
|
+
child.stdout?.setEncoding("utf8");
|
|
1902
|
+
child.stdout?.on("data", (chunk) => {
|
|
1903
|
+
stdout += chunk;
|
|
1904
|
+
});
|
|
1905
|
+
child.stderr?.setEncoding("utf8");
|
|
1906
|
+
child.stderr?.on("data", (chunk) => {
|
|
1907
|
+
stderr += chunk;
|
|
1908
|
+
});
|
|
1909
|
+
child.once("error", (error2) => {
|
|
1910
|
+
finish({
|
|
1911
|
+
code: null,
|
|
1912
|
+
stdout,
|
|
1913
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
1914
|
+
${error2.message}`,
|
|
1915
|
+
timedOut: false
|
|
1916
|
+
});
|
|
1917
|
+
});
|
|
1918
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
1919
|
+
const timer = setTimeout(
|
|
1920
|
+
() => {
|
|
1921
|
+
child.kill("SIGKILL");
|
|
1922
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1923
|
+
},
|
|
1924
|
+
Math.max(0, options.timeoutMs)
|
|
1925
|
+
);
|
|
1926
|
+
});
|
|
1927
|
+
}
|
|
1928
|
+
async function ensureLitestreamConfig(options, env) {
|
|
1929
|
+
const configPath = options.litestreamConfig;
|
|
1930
|
+
if (!configPath) {
|
|
1931
|
+
markNoReplicate(options, "no Litestream configuration path was provided");
|
|
1932
|
+
reportRecord(
|
|
1933
|
+
"restore",
|
|
1934
|
+
"restore_misconfigured",
|
|
1935
|
+
"litestream_config_unavailable",
|
|
1936
|
+
null,
|
|
1937
|
+
options
|
|
1938
|
+
);
|
|
1939
|
+
return null;
|
|
1940
|
+
}
|
|
1941
|
+
try {
|
|
1942
|
+
if (statSync2(configPath).size > 0) return configPath;
|
|
1943
|
+
} catch (error2) {
|
|
1944
|
+
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
1945
|
+
options.log(
|
|
1946
|
+
`Could not inspect ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1947
|
+
"warn"
|
|
1948
|
+
);
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1951
|
+
const rendered = await runSynchroniser(["litestream-config"], {
|
|
1952
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
1953
|
+
env
|
|
1954
|
+
});
|
|
1955
|
+
logSynchroniserDiagnostics(rendered, options);
|
|
1956
|
+
if (rendered.timedOut || rendered.code !== 0) {
|
|
1957
|
+
options.log(
|
|
1958
|
+
`Could not generate ${configPath}: runner-synchroniser litestream-config failed (${commandError(rendered)})`,
|
|
1959
|
+
"error"
|
|
1960
|
+
);
|
|
1961
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
1962
|
+
reportRecord(
|
|
1963
|
+
"restore",
|
|
1964
|
+
"restore_misconfigured",
|
|
1965
|
+
"litestream_config_unavailable",
|
|
1966
|
+
null,
|
|
1967
|
+
options
|
|
1968
|
+
);
|
|
1969
|
+
return null;
|
|
1970
|
+
}
|
|
1971
|
+
try {
|
|
1972
|
+
mkdirSync(dirname2(configPath), { recursive: true });
|
|
1973
|
+
writeFileSync(configPath, rendered.stdout);
|
|
1974
|
+
} catch (error2) {
|
|
1975
|
+
options.log(
|
|
1976
|
+
`Could not write ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1977
|
+
"error"
|
|
1978
|
+
);
|
|
1979
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
1980
|
+
reportRecord(
|
|
1981
|
+
"restore",
|
|
1982
|
+
"restore_misconfigured",
|
|
1983
|
+
"litestream_config_unavailable",
|
|
1984
|
+
null,
|
|
1985
|
+
options
|
|
1986
|
+
);
|
|
1987
|
+
return null;
|
|
1988
|
+
}
|
|
1989
|
+
const version2 = await runCommand("litestream", ["version"], {
|
|
1990
|
+
env,
|
|
1991
|
+
timeoutMs: 1e4
|
|
1992
|
+
});
|
|
1993
|
+
const litestreamVersion = version2.code === 0 ? version2.stdout.trim() || "unknown" : "unknown";
|
|
1994
|
+
const regionEmpty = !/^\s*region:\s*\S+/m.test(rendered.stdout);
|
|
1995
|
+
options.log(
|
|
1996
|
+
`litestream ${litestreamVersion}; AWS_REGION=${env.AWS_REGION ?? "<unset>"} AWS_DEFAULT_REGION=${env.AWS_DEFAULT_REGION ?? "<unset>"}; rendered litestream.yml region empty: ${regionEmpty ? "yes" : "no"}`
|
|
1997
|
+
);
|
|
1998
|
+
return configPath;
|
|
1999
|
+
}
|
|
2000
|
+
function restoreGiveUp(options, reason, message, litestreamExitCode, outcome = "fresh_session_db") {
|
|
2001
|
+
discardSessionDbDebris(options);
|
|
2002
|
+
markNoReplicate(options, message);
|
|
2003
|
+
reportRecord("restore", outcome, reason, litestreamExitCode, options);
|
|
2004
|
+
}
|
|
2005
|
+
async function restoreSessionDb(options, configPath, env) {
|
|
2006
|
+
const restored = await runCommand(
|
|
2007
|
+
"litestream",
|
|
2008
|
+
["restore", "-config", configPath, "-if-db-not-exists", "-if-replica-exists", options.dbPath],
|
|
2009
|
+
{ env, timeoutMs: SESSION_DB_RESTORE_TIMEOUT_MS }
|
|
2010
|
+
);
|
|
2011
|
+
for (const line of splitDiagnostics(restored.stderr)) options.log(line, "warn");
|
|
2012
|
+
if (restored.timedOut || restored.code === 124 || restored.code === 137) {
|
|
2013
|
+
restoreGiveUp(
|
|
2014
|
+
options,
|
|
2015
|
+
"restore_deadline_exceeded",
|
|
2016
|
+
`SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within ${SESSION_DB_RESTORE_TIMEOUT_MS}ms; opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2017
|
+
restored.code ?? 124
|
|
2018
|
+
);
|
|
2019
|
+
return;
|
|
2020
|
+
}
|
|
2021
|
+
if (restored.code === null || restored.code === 125 || restored.code === 126 || restored.code === 127) {
|
|
2022
|
+
options.log(`litestream restore could not run (${commandError(restored)})`, "error");
|
|
2023
|
+
restoreGiveUp(
|
|
2024
|
+
options,
|
|
2025
|
+
"restore_tool_unusable",
|
|
2026
|
+
`restore tool is broken (${commandError(restored)}); opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2027
|
+
restored.code
|
|
2028
|
+
);
|
|
2029
|
+
return;
|
|
2030
|
+
}
|
|
2031
|
+
const classified = await runSynchroniser(
|
|
2032
|
+
[
|
|
2033
|
+
"session-db-classify",
|
|
2034
|
+
String(restored.code ?? 1),
|
|
2035
|
+
"1",
|
|
2036
|
+
"--on-unusable-replica=leave",
|
|
2037
|
+
"--fresh-db-fallback"
|
|
2038
|
+
],
|
|
2039
|
+
{ timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS, env }
|
|
2040
|
+
);
|
|
2041
|
+
logSynchroniserDiagnostics(classified, options);
|
|
2042
|
+
const classifyCode = classified.code;
|
|
2043
|
+
switch (classifyCode) {
|
|
2044
|
+
case 0:
|
|
2045
|
+
return;
|
|
2046
|
+
case 31:
|
|
2047
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2048
|
+
options.log(
|
|
2049
|
+
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
2050
|
+
"warn"
|
|
2051
|
+
);
|
|
2052
|
+
return;
|
|
2053
|
+
case 32:
|
|
2054
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2055
|
+
discardSessionDbDebris(options);
|
|
2056
|
+
markNoReplicate(
|
|
2057
|
+
options,
|
|
2058
|
+
"session-db-classify asked for another restore attempt (32), but this boot has budget for only one; treating it as a give-up rather than retrying"
|
|
2059
|
+
);
|
|
2060
|
+
return;
|
|
2061
|
+
case 30:
|
|
2062
|
+
restoreGiveUp(
|
|
2063
|
+
options,
|
|
2064
|
+
"classification_fatal",
|
|
2065
|
+
"session-db-classify returned fatal (30); see the FATAL message above",
|
|
2066
|
+
restored.code,
|
|
2067
|
+
"restore_misconfigured"
|
|
2068
|
+
);
|
|
2069
|
+
return;
|
|
2070
|
+
default:
|
|
2071
|
+
restoreGiveUp(
|
|
2072
|
+
options,
|
|
2073
|
+
"classification_unrecognised",
|
|
2074
|
+
`session-db-classify exited ${classifyCode ?? "null"}, which is none of its documented answers`,
|
|
2075
|
+
restored.code
|
|
2076
|
+
);
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
async function verifySessionDb(options, configPath, env) {
|
|
2080
|
+
if (!options.noReplicateMarker || !fileExists(options.noReplicateMarker)) {
|
|
2081
|
+
const result = await runSynchroniser(["session-db-verify", configPath], {
|
|
2082
|
+
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
2083
|
+
env: {
|
|
2084
|
+
...env,
|
|
2085
|
+
// The synchroniser reads this value in SECONDS. Keep this at 120, not
|
|
2086
|
+
// 120_000, so the walkback gives up before the outer process bound.
|
|
2087
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
2088
|
+
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
2089
|
+
)
|
|
2090
|
+
}
|
|
2091
|
+
});
|
|
2092
|
+
logSynchroniserDiagnostics(result, options);
|
|
2093
|
+
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
2094
|
+
options.log(
|
|
2095
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
2096
|
+
"warn"
|
|
2097
|
+
);
|
|
2098
|
+
return false;
|
|
2099
|
+
}
|
|
2100
|
+
if (result.code === 34) {
|
|
2101
|
+
reportRecord(
|
|
2102
|
+
"verify",
|
|
2103
|
+
"session_db_boot_refused",
|
|
2104
|
+
result.stderr.includes("SESSION-DB-LOCAL-DISCARD-FAILED") ? "local_discard_failed" : "replica_separation_unproven",
|
|
2105
|
+
null,
|
|
2106
|
+
options
|
|
2107
|
+
);
|
|
2108
|
+
return true;
|
|
2109
|
+
}
|
|
2110
|
+
if (result.code === 33) {
|
|
2111
|
+
options.log(
|
|
2112
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
2113
|
+
"warn"
|
|
2114
|
+
);
|
|
2115
|
+
return false;
|
|
2116
|
+
}
|
|
2117
|
+
if (result.code !== 0) {
|
|
2118
|
+
options.log(
|
|
2119
|
+
`SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${result.code ?? "null"}, which is none of its documented answers; continuing with the restored opencode.db as-is`,
|
|
2120
|
+
"warn"
|
|
2121
|
+
);
|
|
2122
|
+
}
|
|
2123
|
+
return false;
|
|
2124
|
+
}
|
|
2125
|
+
options.log(
|
|
2126
|
+
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
2127
|
+
"debug"
|
|
2128
|
+
);
|
|
2129
|
+
return false;
|
|
2130
|
+
}
|
|
2131
|
+
function fileExists(path) {
|
|
2132
|
+
try {
|
|
2133
|
+
statSync2(path);
|
|
2134
|
+
return true;
|
|
2135
|
+
} catch (error2) {
|
|
2136
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
2137
|
+
return true;
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2140
|
+
async function restoreAndVerifySessionDb(options) {
|
|
2141
|
+
const env = options.env ?? process.env;
|
|
2142
|
+
clearMarker(options);
|
|
2143
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2144
|
+
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
2145
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2146
|
+
env
|
|
2147
|
+
});
|
|
2148
|
+
logSynchroniserDiagnostics(synchroniserEnv, options);
|
|
2149
|
+
if (synchroniserEnv.timedOut || synchroniserEnv.code !== 0) {
|
|
2150
|
+
options.log(
|
|
2151
|
+
`runner-synchroniser env could not resolve the session-DB configuration (${commandError(synchroniserEnv)})`,
|
|
2152
|
+
"error"
|
|
2153
|
+
);
|
|
2154
|
+
markNoReplicate(
|
|
2155
|
+
options,
|
|
2156
|
+
"could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
2157
|
+
);
|
|
2158
|
+
reportRecord(
|
|
2159
|
+
"restore",
|
|
2160
|
+
"restore_misconfigured",
|
|
2161
|
+
"synchroniser_config_unresolved",
|
|
2162
|
+
null,
|
|
2163
|
+
options
|
|
2164
|
+
);
|
|
2165
|
+
return { verifyFatal: false };
|
|
2166
|
+
}
|
|
2167
|
+
const values = parseSynchroniserEnv(synchroniserEnv.stdout);
|
|
2168
|
+
if (!values) {
|
|
2169
|
+
markNoReplicate(
|
|
2170
|
+
options,
|
|
2171
|
+
"the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
2172
|
+
);
|
|
2173
|
+
reportRecord(
|
|
2174
|
+
"restore",
|
|
2175
|
+
"restore_misconfigured",
|
|
2176
|
+
"synchroniser_config_unevaluable",
|
|
2177
|
+
null,
|
|
2178
|
+
options
|
|
2179
|
+
);
|
|
2180
|
+
return { verifyFatal: false };
|
|
2181
|
+
}
|
|
2182
|
+
const synchroniserDbPath = values.OPENCODE_DB_PATH;
|
|
2183
|
+
if (!synchroniserDbPath) {
|
|
2184
|
+
markNoReplicate(
|
|
2185
|
+
options,
|
|
2186
|
+
"run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
2187
|
+
);
|
|
2188
|
+
reportRecord(
|
|
2189
|
+
"restore",
|
|
2190
|
+
"restore_misconfigured",
|
|
2191
|
+
"synchroniser_config_incomplete",
|
|
2192
|
+
null,
|
|
2193
|
+
options
|
|
2194
|
+
);
|
|
2195
|
+
return { verifyFatal: false };
|
|
2196
|
+
}
|
|
2197
|
+
if (resolvePath(synchroniserDbPath) !== resolvePath(options.dbPath)) {
|
|
2198
|
+
options.log(
|
|
2199
|
+
`runner-synchroniser reported OPENCODE_DB_PATH=${synchroniserDbPath}, but OpenCode uses ${options.dbPath}; continuing with OpenCode's session-DB path`,
|
|
2200
|
+
"warn"
|
|
2201
|
+
);
|
|
2202
|
+
}
|
|
2203
|
+
if (!values.PERSISTENCE_BUCKET) {
|
|
2204
|
+
options.log(
|
|
2205
|
+
"SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated.",
|
|
2206
|
+
"warn"
|
|
2207
|
+
);
|
|
2208
|
+
return { verifyFatal: false };
|
|
2209
|
+
}
|
|
2210
|
+
const configPath = await ensureLitestreamConfig(options, env);
|
|
2211
|
+
if (!configPath) return { verifyFatal: false };
|
|
2212
|
+
await restoreSessionDb(options, configPath, env);
|
|
2213
|
+
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
2214
|
+
return { verifyFatal: false };
|
|
2215
|
+
}
|
|
2216
|
+
return { verifyFatal: await verifySessionDb(options, configPath, env) };
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
// src/lib/opencode/session-db-provenance.ts
|
|
2220
|
+
import { createRequire } from "module";
|
|
2221
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
2222
|
+
import { dirname as dirname3, join as join3 } from "path";
|
|
2223
|
+
var require2 = createRequire(import.meta.url);
|
|
2224
|
+
function readSessionDbMigrationIds(dbPath) {
|
|
2225
|
+
let db;
|
|
2226
|
+
try {
|
|
2227
|
+
const { DatabaseSync } = require2("node:sqlite");
|
|
2228
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
2229
|
+
const columns = db.prepare("PRAGMA table_info(migration)").all();
|
|
2230
|
+
const hasExpectedShape = columns.length === 2 && columns.some(
|
|
2231
|
+
(column) => column.name === "id" && typeof column.type === "string" && column.type.toUpperCase() === "TEXT" && column.pk === 1
|
|
2232
|
+
) && columns.some(
|
|
2233
|
+
(column) => column.name === "time_completed" && typeof column.type === "string" && column.type.toUpperCase() === "INTEGER" && column.notnull === 1 && column.pk === 0
|
|
2234
|
+
);
|
|
2235
|
+
if (!hasExpectedShape) {
|
|
2236
|
+
console.warn(`[readSessionDbMigrationIds] unsupported migration table shape in ${dbPath}`);
|
|
2237
|
+
return null;
|
|
2238
|
+
}
|
|
2239
|
+
const rows = db.prepare("SELECT id FROM migration ORDER BY id").all();
|
|
2240
|
+
if (rows.some((row) => typeof row.id !== "string")) return null;
|
|
2241
|
+
return rows.map((row) => row.id);
|
|
2242
|
+
} catch (error2) {
|
|
2243
|
+
console.warn(
|
|
2244
|
+
`[readSessionDbMigrationIds] could not read migration history from ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2245
|
+
);
|
|
2246
|
+
return null;
|
|
2247
|
+
} finally {
|
|
2248
|
+
try {
|
|
2249
|
+
db?.close();
|
|
2250
|
+
} catch (error2) {
|
|
2251
|
+
console.warn(
|
|
2252
|
+
`[readSessionDbMigrationIds] could not close ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2253
|
+
);
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
}
|
|
2257
|
+
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
2258
|
+
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
2259
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
2260
|
+
}
|
|
2261
|
+
function loadSessionDbProvenanceState(path) {
|
|
2262
|
+
let value;
|
|
2263
|
+
try {
|
|
2264
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
2265
|
+
} catch (error2) {
|
|
2266
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return {};
|
|
2267
|
+
console.error(
|
|
2268
|
+
`[session-db-provenance] could not read ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2269
|
+
);
|
|
2270
|
+
return {};
|
|
2271
|
+
}
|
|
2272
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2273
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2274
|
+
return {};
|
|
2275
|
+
}
|
|
2276
|
+
const state = {};
|
|
2277
|
+
for (const [dbPath, record] of Object.entries(value)) {
|
|
2278
|
+
if (!isSessionDbProvenanceRecord(record)) {
|
|
2279
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2280
|
+
return {};
|
|
2281
|
+
}
|
|
2282
|
+
state[dbPath] = record;
|
|
2283
|
+
}
|
|
2284
|
+
return state;
|
|
2285
|
+
}
|
|
2286
|
+
function saveSessionDbProvenanceState(path, state) {
|
|
2287
|
+
try {
|
|
2288
|
+
mkdirSync2(dirname3(path), { recursive: true });
|
|
2289
|
+
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
2290
|
+
`, "utf8");
|
|
2291
|
+
} catch (error2) {
|
|
2292
|
+
console.error(
|
|
2293
|
+
`[session-db-provenance] could not write ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2294
|
+
);
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
function evaluateSessionDbProvenance(input) {
|
|
2298
|
+
const { currentVersion, currentIds, previous } = input;
|
|
2299
|
+
if (!previous) return { anomaly: false, reason: null };
|
|
2300
|
+
const current = new Set(currentIds);
|
|
2301
|
+
const prior = new Set(previous.migrationIds);
|
|
2302
|
+
for (const id of prior) {
|
|
2303
|
+
if (!current.has(id)) return { anomaly: true, reason: "migration-history-regressed" };
|
|
2304
|
+
}
|
|
2305
|
+
if (current.size > prior.size && previous.opencodeVersion === currentVersion) {
|
|
2306
|
+
return { anomaly: true, reason: "foreign-version-migrations" };
|
|
2307
|
+
}
|
|
2308
|
+
return { anomaly: false, reason: null };
|
|
2309
|
+
}
|
|
2310
|
+
function checkSessionDbProvenance(input) {
|
|
2311
|
+
const { dbPath, currentVersion, homeDir, env } = input;
|
|
2312
|
+
const path = sessionDbProvenanceStatePath(homeDir, env);
|
|
2313
|
+
const state = loadSessionDbProvenanceState(path);
|
|
2314
|
+
const previous = state[dbPath];
|
|
2315
|
+
const currentIds = readSessionDbMigrationIds(dbPath);
|
|
2316
|
+
if (currentIds === null) {
|
|
2317
|
+
return {
|
|
2318
|
+
anomaly: false,
|
|
2319
|
+
reason: null,
|
|
2320
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2321
|
+
migrationDelta: null
|
|
2322
|
+
};
|
|
2323
|
+
}
|
|
2324
|
+
const decision = evaluateSessionDbProvenance({ currentVersion, currentIds, previous });
|
|
2325
|
+
const migrationDelta = previous ? new Set(currentIds).size - new Set(previous.migrationIds).size : null;
|
|
2326
|
+
state[dbPath] = {
|
|
2327
|
+
opencodeVersion: currentVersion,
|
|
2328
|
+
migrationIds: currentIds,
|
|
2329
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2330
|
+
};
|
|
2331
|
+
saveSessionDbProvenanceState(path, state);
|
|
2332
|
+
return {
|
|
2333
|
+
...decision,
|
|
2334
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2335
|
+
migrationDelta
|
|
2336
|
+
};
|
|
2337
|
+
}
|
|
2338
|
+
function isSessionDbProvenanceRecord(value) {
|
|
2339
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2340
|
+
const record = value;
|
|
2341
|
+
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
2342
|
+
}
|
|
2343
|
+
|
|
1414
2344
|
// src/lib/opencode/opencode-version-gate.ts
|
|
1415
2345
|
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
1416
2346
|
function isQueueValidatedVersion(version2) {
|
|
@@ -1425,7 +2355,63 @@ function buildOpenCodeVersionWarning(version2) {
|
|
|
1425
2355
|
}
|
|
1426
2356
|
|
|
1427
2357
|
// src/lib/opencode/process.ts
|
|
1428
|
-
import { execSync, spawn } from "child_process";
|
|
2358
|
+
import { execSync, spawn as spawn3 } from "child_process";
|
|
2359
|
+
|
|
2360
|
+
// src/lib/process-stop.ts
|
|
2361
|
+
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
2362
|
+
if (!child.pid) {
|
|
2363
|
+
return { outcome: "not-running", code: child.exitCode, signal: child.signalCode };
|
|
2364
|
+
}
|
|
2365
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
2366
|
+
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
2367
|
+
}
|
|
2368
|
+
return new Promise((resolve4, reject) => {
|
|
2369
|
+
let forced = false;
|
|
2370
|
+
let settled = false;
|
|
2371
|
+
const timer = setTimeout(() => {
|
|
2372
|
+
forced = true;
|
|
2373
|
+
try {
|
|
2374
|
+
sendKill();
|
|
2375
|
+
} catch (error2) {
|
|
2376
|
+
if (error2.code === "ESRCH") {
|
|
2377
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2378
|
+
} else {
|
|
2379
|
+
fail(error2);
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2382
|
+
}, timeoutMs);
|
|
2383
|
+
const finish = (result) => {
|
|
2384
|
+
if (settled) return;
|
|
2385
|
+
settled = true;
|
|
2386
|
+
clearTimeout(timer);
|
|
2387
|
+
child.removeListener("exit", onExit);
|
|
2388
|
+
resolve4(result);
|
|
2389
|
+
};
|
|
2390
|
+
const fail = (error2) => {
|
|
2391
|
+
if (settled) return;
|
|
2392
|
+
settled = true;
|
|
2393
|
+
clearTimeout(timer);
|
|
2394
|
+
child.removeListener("exit", onExit);
|
|
2395
|
+
reject(error2);
|
|
2396
|
+
};
|
|
2397
|
+
const onExit = (code, signal) => {
|
|
2398
|
+
finish({ outcome: forced ? "killed" : "exited", code, signal });
|
|
2399
|
+
};
|
|
2400
|
+
child.once("exit", onExit);
|
|
2401
|
+
try {
|
|
2402
|
+
sendTerm();
|
|
2403
|
+
} catch (error2) {
|
|
2404
|
+
if (error2.code === "ESRCH") {
|
|
2405
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2406
|
+
} else {
|
|
2407
|
+
fail(error2);
|
|
2408
|
+
}
|
|
2409
|
+
return;
|
|
2410
|
+
}
|
|
2411
|
+
});
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
// src/lib/opencode/process.ts
|
|
1429
2415
|
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
1430
2416
|
function getProcessCwd(pid) {
|
|
1431
2417
|
const platform = process.platform;
|
|
@@ -1581,39 +2567,45 @@ async function findHealthyOpenCodeInstances() {
|
|
|
1581
2567
|
}
|
|
1582
2568
|
return healthy;
|
|
1583
2569
|
}
|
|
1584
|
-
async function startOpenCode(port) {
|
|
2570
|
+
async function startOpenCode(port, options = {}) {
|
|
1585
2571
|
let command = "opencode";
|
|
1586
|
-
|
|
2572
|
+
const printLogs = options.inheritStdio ? ["--print-logs"] : [];
|
|
2573
|
+
let args = ["serve", "--port", port.toString(), "--hostname", "127.0.0.1", ...printLogs];
|
|
1587
2574
|
try {
|
|
1588
2575
|
execSync("which opencode", { stdio: "ignore" });
|
|
1589
2576
|
} catch {
|
|
1590
2577
|
command = "npx";
|
|
1591
|
-
args = [
|
|
1592
|
-
|
|
1593
|
-
|
|
2578
|
+
args = [
|
|
2579
|
+
"opencode",
|
|
2580
|
+
"serve",
|
|
2581
|
+
"--port",
|
|
2582
|
+
port.toString(),
|
|
2583
|
+
"--hostname",
|
|
2584
|
+
"127.0.0.1",
|
|
2585
|
+
...printLogs
|
|
2586
|
+
];
|
|
2587
|
+
}
|
|
2588
|
+
const child = spawn3(command, args, {
|
|
1594
2589
|
detached: true,
|
|
1595
|
-
stdio: "ignore",
|
|
2590
|
+
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
1596
2591
|
cwd: process.cwd()
|
|
1597
2592
|
});
|
|
1598
2593
|
return child;
|
|
1599
2594
|
}
|
|
1600
|
-
function
|
|
1601
|
-
|
|
1602
|
-
return;
|
|
1603
|
-
}
|
|
1604
|
-
try {
|
|
2595
|
+
function stopOpenCodeAndWait(opencodeProcess, timeoutMs) {
|
|
2596
|
+
const sendSignal = (signal) => {
|
|
1605
2597
|
if (process.platform === "win32") {
|
|
1606
|
-
opencodeProcess.kill(
|
|
2598
|
+
opencodeProcess.kill(signal);
|
|
1607
2599
|
} else {
|
|
1608
|
-
process.kill(-opencodeProcess.pid,
|
|
2600
|
+
process.kill(-opencodeProcess.pid, signal);
|
|
1609
2601
|
}
|
|
1610
|
-
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
2602
|
+
};
|
|
2603
|
+
return stopProcessAndWait(
|
|
2604
|
+
opencodeProcess,
|
|
2605
|
+
timeoutMs,
|
|
2606
|
+
() => sendSignal("SIGTERM"),
|
|
2607
|
+
() => sendSignal("SIGKILL")
|
|
2608
|
+
);
|
|
1617
2609
|
}
|
|
1618
2610
|
|
|
1619
2611
|
// src/lib/opencode/install.ts
|
|
@@ -1900,6 +2892,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1900
2892
|
return data.id;
|
|
1901
2893
|
}
|
|
1902
2894
|
async function getModelAttachmentCapability(port, model) {
|
|
2895
|
+
const { model: baseModel } = splitModelVariant(model);
|
|
1903
2896
|
try {
|
|
1904
2897
|
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1905
2898
|
if (!res.ok) {
|
|
@@ -1916,9 +2909,9 @@ async function getModelAttachmentCapability(port, model) {
|
|
|
1916
2909
|
);
|
|
1917
2910
|
return null;
|
|
1918
2911
|
}
|
|
1919
|
-
const slash =
|
|
1920
|
-
const providerId = slash > 0 ?
|
|
1921
|
-
let modelId = slash > 0 ?
|
|
2912
|
+
const slash = baseModel ? baseModel.indexOf("/") : -1;
|
|
2913
|
+
const providerId = slash > 0 ? baseModel.slice(0, slash) : void 0;
|
|
2914
|
+
let modelId = slash > 0 ? baseModel.slice(slash + 1) : void 0;
|
|
1922
2915
|
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
1923
2916
|
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
1924
2917
|
if (!provider && !providerId) {
|
|
@@ -1998,6 +2991,29 @@ async function buildFileParts(attachments, capable) {
|
|
|
1998
2991
|
}
|
|
1999
2992
|
return { parts, outcomes, capabilityUnknown };
|
|
2000
2993
|
}
|
|
2994
|
+
function splitModelVariant(raw) {
|
|
2995
|
+
const value = raw?.trim();
|
|
2996
|
+
if (!value) return {};
|
|
2997
|
+
const hashIndex = value.indexOf("#");
|
|
2998
|
+
if (hashIndex === -1) return { model: value };
|
|
2999
|
+
const model = value.slice(0, hashIndex).trim() || void 0;
|
|
3000
|
+
const variant = value.slice(hashIndex + 1).trim() || void 0;
|
|
3001
|
+
return { model, variant };
|
|
3002
|
+
}
|
|
3003
|
+
function applyModelOptions(body, options) {
|
|
3004
|
+
if (options?.agent) body.agent = options.agent;
|
|
3005
|
+
const { model, variant } = splitModelVariant(options?.model);
|
|
3006
|
+
if (model) {
|
|
3007
|
+
const slashIndex = model.indexOf("/");
|
|
3008
|
+
if (slashIndex !== -1) {
|
|
3009
|
+
body.model = {
|
|
3010
|
+
providerID: model.substring(0, slashIndex),
|
|
3011
|
+
modelID: model.substring(slashIndex + 1)
|
|
3012
|
+
};
|
|
3013
|
+
}
|
|
3014
|
+
}
|
|
3015
|
+
if (variant) body.variant = variant;
|
|
3016
|
+
}
|
|
2001
3017
|
function messageText(m) {
|
|
2002
3018
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2003
3019
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
@@ -2022,18 +3038,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2022
3038
|
const body = {
|
|
2023
3039
|
parts
|
|
2024
3040
|
};
|
|
2025
|
-
|
|
2026
|
-
body.agent = options.agent;
|
|
2027
|
-
}
|
|
2028
|
-
if (options?.model) {
|
|
2029
|
-
const slashIndex = options.model.indexOf("/");
|
|
2030
|
-
if (slashIndex !== -1) {
|
|
2031
|
-
body.model = {
|
|
2032
|
-
providerID: options.model.substring(0, slashIndex),
|
|
2033
|
-
modelID: options.model.substring(slashIndex + 1)
|
|
2034
|
-
};
|
|
2035
|
-
}
|
|
2036
|
-
}
|
|
3041
|
+
applyModelOptions(body, options);
|
|
2037
3042
|
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
2038
3043
|
method: "POST",
|
|
2039
3044
|
headers: { "Content-Type": "application/json" },
|
|
@@ -2041,7 +3046,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2041
3046
|
});
|
|
2042
3047
|
if (res.status < 200 || res.status >= 300) {
|
|
2043
3048
|
const text = await res.text().catch(() => "");
|
|
2044
|
-
|
|
3049
|
+
const { variant } = splitModelVariant(options?.model);
|
|
3050
|
+
throw new Error(
|
|
3051
|
+
`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}${variant ? ` (variant: ${variant})` : ""}`
|
|
3052
|
+
);
|
|
2045
3053
|
}
|
|
2046
3054
|
const READ_BACK_ATTEMPTS = 5;
|
|
2047
3055
|
const READ_BACK_DELAY_MS = 150;
|
|
@@ -2065,7 +3073,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2065
3073
|
}
|
|
2066
3074
|
}
|
|
2067
3075
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2068
|
-
await new Promise((
|
|
3076
|
+
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
2069
3077
|
}
|
|
2070
3078
|
}
|
|
2071
3079
|
return null;
|
|
@@ -2196,7 +3204,7 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
2196
3204
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
2197
3205
|
}
|
|
2198
3206
|
function isB2AbandonmentConfirmed(params) {
|
|
2199
|
-
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
3207
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false && params.rootOngoing === false;
|
|
2200
3208
|
}
|
|
2201
3209
|
function isAmbiguousTerminalFinish(m) {
|
|
2202
3210
|
if (completedOf(m) == null) return false;
|
|
@@ -2209,7 +3217,7 @@ function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
|
|
|
2209
3217
|
return isAmbiguousTerminalFinish(reply);
|
|
2210
3218
|
}
|
|
2211
3219
|
function isAmbiguousFinishResolved(params) {
|
|
2212
|
-
return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
|
|
3220
|
+
return params.sessionOngoing === false || params.sessionOngoing !== true && params.pinnedForMs >= params.maxPinnedMs;
|
|
2213
3221
|
}
|
|
2214
3222
|
function messageError(messages, userMessageId) {
|
|
2215
3223
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2418,13 +3426,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2418
3426
|
}
|
|
2419
3427
|
|
|
2420
3428
|
// src/lib/opencode/session-db-size.ts
|
|
2421
|
-
import { statSync as
|
|
2422
|
-
import { join as
|
|
3429
|
+
import { statSync as statSync3 } from "fs";
|
|
3430
|
+
import { join as join4 } from "path";
|
|
2423
3431
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2424
3432
|
function statSessionDbBytes(homeDir) {
|
|
2425
|
-
const dbPath =
|
|
3433
|
+
const dbPath = join4(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2426
3434
|
try {
|
|
2427
|
-
return
|
|
3435
|
+
return statSync3(dbPath).size;
|
|
2428
3436
|
} catch (err) {
|
|
2429
3437
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2430
3438
|
if (!isMissingFile) {
|
|
@@ -2450,11 +3458,11 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
2450
3458
|
}
|
|
2451
3459
|
|
|
2452
3460
|
// src/lib/opencode/session-db-reclaim.ts
|
|
2453
|
-
import { statSync as
|
|
2454
|
-
import { dirname as
|
|
3461
|
+
import { statSync as statSync4, statfsSync } from "fs";
|
|
3462
|
+
import { dirname as dirname4 } from "path";
|
|
2455
3463
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2456
3464
|
try {
|
|
2457
|
-
const fsStats = statfsSync(
|
|
3465
|
+
const fsStats = statfsSync(dirname4(dbPath));
|
|
2458
3466
|
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2459
3467
|
if (availableBytes < requiredBytes) {
|
|
2460
3468
|
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
@@ -2523,7 +3531,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2523
3531
|
);
|
|
2524
3532
|
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2525
3533
|
}
|
|
2526
|
-
const fileBytesForGuard =
|
|
3534
|
+
const fileBytesForGuard = statSync4(dbPath).size;
|
|
2527
3535
|
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2528
3536
|
if (skipReason !== null) {
|
|
2529
3537
|
console.warn(
|
|
@@ -2651,12 +3659,12 @@ var StreamForwarder = class {
|
|
|
2651
3659
|
let endBody;
|
|
2652
3660
|
if (has_body) {
|
|
2653
3661
|
const chunks = [];
|
|
2654
|
-
bodyPromise = new Promise((
|
|
3662
|
+
bodyPromise = new Promise((resolve4) => {
|
|
2655
3663
|
pushBody = (buf) => {
|
|
2656
3664
|
chunks.push(buf);
|
|
2657
3665
|
};
|
|
2658
3666
|
endBody = () => {
|
|
2659
|
-
|
|
3667
|
+
resolve4(Buffer.concat(chunks));
|
|
2660
3668
|
};
|
|
2661
3669
|
});
|
|
2662
3670
|
}
|
|
@@ -2785,7 +3793,7 @@ function connectTunnel(options) {
|
|
|
2785
3793
|
} = options;
|
|
2786
3794
|
const tunnelUrl = getTunnelUrlConfig();
|
|
2787
3795
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
2788
|
-
return new Promise((
|
|
3796
|
+
return new Promise((resolve4, reject) => {
|
|
2789
3797
|
const ws = new WebSocket2(url, {
|
|
2790
3798
|
headers: {
|
|
2791
3799
|
Authorization: authHeader
|
|
@@ -2849,7 +3857,7 @@ function connectTunnel(options) {
|
|
|
2849
3857
|
clearTimeout(connectionTimeout);
|
|
2850
3858
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
2851
3859
|
onConnected?.(connectedAgentId);
|
|
2852
|
-
|
|
3860
|
+
resolve4({
|
|
2853
3861
|
ws,
|
|
2854
3862
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
2855
3863
|
});
|
|
@@ -2980,10 +3988,10 @@ var RunnerConnection = class {
|
|
|
2980
3988
|
};
|
|
2981
3989
|
|
|
2982
3990
|
// src/lib/tunnel/ready-marker.ts
|
|
2983
|
-
import { writeFileSync } from "fs";
|
|
3991
|
+
import { writeFileSync as writeFileSync3 } from "fs";
|
|
2984
3992
|
function writeTunnelReadyMarker(path, agentId) {
|
|
2985
3993
|
try {
|
|
2986
|
-
|
|
3994
|
+
writeFileSync3(path, `${agentId}
|
|
2987
3995
|
`);
|
|
2988
3996
|
return { ok: true };
|
|
2989
3997
|
} catch (error2) {
|
|
@@ -2991,6 +3999,219 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
2991
3999
|
}
|
|
2992
4000
|
}
|
|
2993
4001
|
|
|
4002
|
+
// src/lib/replication.ts
|
|
4003
|
+
import { spawn as spawn4 } from "child_process";
|
|
4004
|
+
function startSessionDbReplication(configPath) {
|
|
4005
|
+
return spawn4("litestream", ["replicate", "-config", configPath], {
|
|
4006
|
+
stdio: "inherit"
|
|
4007
|
+
});
|
|
4008
|
+
}
|
|
4009
|
+
async function stopSessionDbReplication(child, timeoutMs) {
|
|
4010
|
+
return stopProcessAndWait(
|
|
4011
|
+
child,
|
|
4012
|
+
timeoutMs,
|
|
4013
|
+
() => child.kill("SIGTERM"),
|
|
4014
|
+
() => child.kill("SIGKILL")
|
|
4015
|
+
);
|
|
4016
|
+
}
|
|
4017
|
+
|
|
4018
|
+
// src/lib/process-liveness.ts
|
|
4019
|
+
import { readFileSync as readFileSync4 } from "fs";
|
|
4020
|
+
function isProcessAlive(pid) {
|
|
4021
|
+
try {
|
|
4022
|
+
process.kill(pid, 0);
|
|
4023
|
+
} catch (error2) {
|
|
4024
|
+
const code = error2.code;
|
|
4025
|
+
if (code === "ESRCH") return false;
|
|
4026
|
+
if (code === "EPERM") return true;
|
|
4027
|
+
console.error(
|
|
4028
|
+
`[process-liveness] could not probe process ${pid}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4029
|
+
);
|
|
4030
|
+
return false;
|
|
4031
|
+
}
|
|
4032
|
+
if (process.platform !== "linux") return true;
|
|
4033
|
+
try {
|
|
4034
|
+
const status2 = readFileSync4(`/proc/${pid}/status`, "utf8");
|
|
4035
|
+
return !/^State:\s+Z(?:\s|$)/m.test(status2);
|
|
4036
|
+
} catch (error2) {
|
|
4037
|
+
console.error(
|
|
4038
|
+
`[process-liveness] could not inspect /proc/${pid}/status: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4039
|
+
);
|
|
4040
|
+
return true;
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
|
|
4044
|
+
// src/lib/openai-usage.ts
|
|
4045
|
+
import { readFileSync as readFileSync5 } from "fs";
|
|
4046
|
+
import { homedir as homedir3 } from "os";
|
|
4047
|
+
import { join as join5 } from "path";
|
|
4048
|
+
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
4049
|
+
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
4050
|
+
var OpenAiUsageError = class extends Error {
|
|
4051
|
+
constructor(message, reason) {
|
|
4052
|
+
super(message);
|
|
4053
|
+
this.reason = reason;
|
|
4054
|
+
}
|
|
4055
|
+
};
|
|
4056
|
+
function isLocalCredentialProblem2(err) {
|
|
4057
|
+
return err instanceof OpenAiUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
|
|
4058
|
+
}
|
|
4059
|
+
function readOpenCodeChatGptCredentials() {
|
|
4060
|
+
try {
|
|
4061
|
+
const raw = readFileSync5(join5(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
4062
|
+
let parsed;
|
|
4063
|
+
try {
|
|
4064
|
+
parsed = JSON.parse(raw);
|
|
4065
|
+
} catch {
|
|
4066
|
+
return null;
|
|
4067
|
+
}
|
|
4068
|
+
const entry = parsed.openai;
|
|
4069
|
+
if (entry?.type !== "oauth" || typeof entry.access !== "string" || !entry.access || typeof entry.expires !== "number") {
|
|
4070
|
+
return null;
|
|
4071
|
+
}
|
|
4072
|
+
return { accessToken: entry.access, expiresAt: entry.expires };
|
|
4073
|
+
} catch (err) {
|
|
4074
|
+
const code = err.code;
|
|
4075
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
4076
|
+
console.warn(
|
|
4077
|
+
`readOpenCodeChatGptCredentials: reading auth.json failed (${code ?? "unknown"})`
|
|
4078
|
+
);
|
|
4079
|
+
}
|
|
4080
|
+
return null;
|
|
4081
|
+
}
|
|
4082
|
+
}
|
|
4083
|
+
function toWindow2(headers, name) {
|
|
4084
|
+
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
4085
|
+
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
4086
|
+
if (utilizationHeader == null || utilizationHeader === "" || windowMinutesHeader == null) {
|
|
4087
|
+
return null;
|
|
4088
|
+
}
|
|
4089
|
+
const utilization = Number(utilizationHeader);
|
|
4090
|
+
const windowMinutes = Number(windowMinutesHeader);
|
|
4091
|
+
if (!Number.isFinite(utilization) || !Number.isFinite(windowMinutes) || windowMinutes <= 0) {
|
|
4092
|
+
return null;
|
|
4093
|
+
}
|
|
4094
|
+
const resetAtHeader = headers.get(`x-codex-${name}-reset-at`);
|
|
4095
|
+
const resetSeconds = resetAtHeader == null || resetAtHeader === "" ? NaN : Number(resetAtHeader);
|
|
4096
|
+
const resetsAt = Number.isFinite(resetSeconds) ? new Date(resetSeconds * 1e3).toISOString() : null;
|
|
4097
|
+
return { utilization: Math.min(100, Math.max(0, utilization)), windowMinutes, resetsAt };
|
|
4098
|
+
}
|
|
4099
|
+
function parseCodexUsageHeaders(headers) {
|
|
4100
|
+
return {
|
|
4101
|
+
primary: toWindow2(headers, "primary"),
|
|
4102
|
+
secondary: toWindow2(headers, "secondary"),
|
|
4103
|
+
hasCredits: headers.has("x-codex-credits-has-credits") ? headers.get("x-codex-credits-has-credits")?.toLowerCase() === "true" : null,
|
|
4104
|
+
creditsUnlimited: headers.has("x-codex-credits-unlimited") ? headers.get("x-codex-credits-unlimited")?.toLowerCase() === "true" : null
|
|
4105
|
+
};
|
|
4106
|
+
}
|
|
4107
|
+
function normalizeProbeModel(model) {
|
|
4108
|
+
return model.endsWith("-fast") ? model.slice(0, -"-fast".length) : model;
|
|
4109
|
+
}
|
|
4110
|
+
async function resolveProbeModels(port) {
|
|
4111
|
+
try {
|
|
4112
|
+
const res = await withRequestTimeout(
|
|
4113
|
+
fetch,
|
|
4114
|
+
REQUEST_TIMEOUT_MS
|
|
4115
|
+
)(`${opencodeBase(port)}/config/providers`);
|
|
4116
|
+
if (!res.ok) {
|
|
4117
|
+
console.error(
|
|
4118
|
+
`[resolveProbeModels] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
4119
|
+
);
|
|
4120
|
+
return [];
|
|
4121
|
+
}
|
|
4122
|
+
const body = await res.json();
|
|
4123
|
+
const provider = body?.providers?.find((candidate) => candidate?.id === "openai");
|
|
4124
|
+
if (!provider || !provider.models || typeof provider.models !== "object") return [];
|
|
4125
|
+
const candidates = [
|
|
4126
|
+
...typeof body?.default?.openai === "string" ? [body.default.openai] : [],
|
|
4127
|
+
...Object.keys(provider.models)
|
|
4128
|
+
].map(normalizeProbeModel);
|
|
4129
|
+
return [...new Set(candidates)].slice(0, 4);
|
|
4130
|
+
} catch (err) {
|
|
4131
|
+
console.error(
|
|
4132
|
+
`[resolveProbeModels] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
4133
|
+
);
|
|
4134
|
+
return [];
|
|
4135
|
+
}
|
|
4136
|
+
}
|
|
4137
|
+
function hasPrimaryHeaders(headers) {
|
|
4138
|
+
return [
|
|
4139
|
+
"x-codex-primary-used-percent",
|
|
4140
|
+
"x-codex-primary-window-minutes",
|
|
4141
|
+
"x-codex-primary-reset-at"
|
|
4142
|
+
].some((name) => headers.has(name));
|
|
4143
|
+
}
|
|
4144
|
+
async function getOpenAiUsage(port) {
|
|
4145
|
+
const credentials2 = readOpenCodeChatGptCredentials();
|
|
4146
|
+
if (!credentials2) {
|
|
4147
|
+
throw new OpenAiUsageError(
|
|
4148
|
+
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
4149
|
+
"no_credentials"
|
|
4150
|
+
);
|
|
4151
|
+
}
|
|
4152
|
+
if (credentials2.expiresAt < Date.now()) {
|
|
4153
|
+
throw new OpenAiUsageError(
|
|
4154
|
+
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
4155
|
+
"credentials_expired"
|
|
4156
|
+
);
|
|
4157
|
+
}
|
|
4158
|
+
const models = await resolveProbeModels(port);
|
|
4159
|
+
if (models.length === 0) {
|
|
4160
|
+
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
4161
|
+
}
|
|
4162
|
+
let lastStatus;
|
|
4163
|
+
for (const model of models) {
|
|
4164
|
+
let res;
|
|
4165
|
+
try {
|
|
4166
|
+
res = await withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(CODEX_RESPONSES_URL, {
|
|
4167
|
+
method: "POST",
|
|
4168
|
+
headers: {
|
|
4169
|
+
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
4170
|
+
"Content-Type": "application/json"
|
|
4171
|
+
},
|
|
4172
|
+
body: JSON.stringify({ model, store: false, stream: true })
|
|
4173
|
+
});
|
|
4174
|
+
} catch (err) {
|
|
4175
|
+
throw new OpenAiUsageError(
|
|
4176
|
+
`OpenAI usage probe request failed: ${err instanceof Error ? err.message : String(err)}.`,
|
|
4177
|
+
"request_failed"
|
|
4178
|
+
);
|
|
4179
|
+
}
|
|
4180
|
+
try {
|
|
4181
|
+
lastStatus = res.status;
|
|
4182
|
+
if (hasPrimaryHeaders(res.headers)) {
|
|
4183
|
+
const usage = parseCodexUsageHeaders(res.headers);
|
|
4184
|
+
if (!usage.primary && !usage.secondary) {
|
|
4185
|
+
throw new OpenAiUsageError(
|
|
4186
|
+
`OpenAI usage probe returned no usable window (HTTP ${res.status}).`,
|
|
4187
|
+
"no_usable_window"
|
|
4188
|
+
);
|
|
4189
|
+
}
|
|
4190
|
+
return usage;
|
|
4191
|
+
}
|
|
4192
|
+
if (res.status === 401) {
|
|
4193
|
+
throw new OpenAiUsageError(
|
|
4194
|
+
"ChatGPT credentials have expired (HTTP 401).",
|
|
4195
|
+
"credentials_expired"
|
|
4196
|
+
);
|
|
4197
|
+
}
|
|
4198
|
+
if (res.status === 403 || res.status === 429) {
|
|
4199
|
+
throw new OpenAiUsageError(
|
|
4200
|
+
`OpenAI usage probe was blocked (HTTP ${res.status}).`,
|
|
4201
|
+
"probe_blocked"
|
|
4202
|
+
);
|
|
4203
|
+
}
|
|
4204
|
+
} finally {
|
|
4205
|
+
await res.body?.cancel().catch(() => {
|
|
4206
|
+
});
|
|
4207
|
+
}
|
|
4208
|
+
}
|
|
4209
|
+
throw new OpenAiUsageError(
|
|
4210
|
+
`OpenAI usage probe failed: HTTP ${lastStatus ?? "unknown"}.`,
|
|
4211
|
+
"request_failed"
|
|
4212
|
+
);
|
|
4213
|
+
}
|
|
4214
|
+
|
|
2994
4215
|
// src/lib/reporting-schedule.ts
|
|
2995
4216
|
function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
2996
4217
|
const jitterRangeMs = baseMs * jitterFraction;
|
|
@@ -2999,6 +4220,31 @@ function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
|
2999
4220
|
function firstReportDelayMs(random = Math.random) {
|
|
3000
4221
|
return 5e3 + random() * 1e4;
|
|
3001
4222
|
}
|
|
4223
|
+
var VALID_USAGE_REPORTING_MODES = ["auto", "on", "off"];
|
|
4224
|
+
function resolveUsageReportingMode(flagValue, env, names) {
|
|
4225
|
+
const raw = flagValue ?? env[names.envVar];
|
|
4226
|
+
if (raw === void 0 || raw === "") return { mode: "auto", warnings: [] };
|
|
4227
|
+
const normalized = raw.trim().toLowerCase();
|
|
4228
|
+
if (VALID_USAGE_REPORTING_MODES.includes(normalized)) {
|
|
4229
|
+
return { mode: normalized, warnings: [] };
|
|
4230
|
+
}
|
|
4231
|
+
const source = flagValue !== void 0 ? names.flagName : names.envVar;
|
|
4232
|
+
return {
|
|
4233
|
+
mode: "auto",
|
|
4234
|
+
warnings: [
|
|
4235
|
+
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_USAGE_REPORTING_MODES.join(", ")}; using auto`
|
|
4236
|
+
]
|
|
4237
|
+
};
|
|
4238
|
+
}
|
|
4239
|
+
var BASE_USAGE_REPORT_DELAY_MS = 10 * 6e4;
|
|
4240
|
+
var USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
4241
|
+
function usageReportDelayMs(random = Math.random) {
|
|
4242
|
+
return jitteredDelayMs(BASE_USAGE_REPORT_DELAY_MS, USAGE_REPORT_DELAY_JITTER_FRACTION, random);
|
|
4243
|
+
}
|
|
4244
|
+
var USAGE_REPORT_FAILURE_REESCALATION_TICKS = 6;
|
|
4245
|
+
function usageReportFailureLogLevel(consecutiveFailures) {
|
|
4246
|
+
return reportFailureLogLevel(consecutiveFailures, USAGE_REPORT_FAILURE_REESCALATION_TICKS);
|
|
4247
|
+
}
|
|
3002
4248
|
function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
|
|
3003
4249
|
return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
|
|
3004
4250
|
}
|
|
@@ -3007,33 +4253,26 @@ function failureStreakSuffix(consecutiveFailures) {
|
|
|
3007
4253
|
}
|
|
3008
4254
|
|
|
3009
4255
|
// src/lib/claude-usage-reporting.ts
|
|
3010
|
-
var VALID_MODES = ["auto", "on", "off"];
|
|
3011
4256
|
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
}
|
|
3016
|
-
const normalized = raw.trim().toLowerCase();
|
|
3017
|
-
if (VALID_MODES.includes(normalized)) {
|
|
3018
|
-
return { mode: normalized, warnings: [] };
|
|
3019
|
-
}
|
|
3020
|
-
const source = flagValue !== void 0 ? "--claude-usage-reporting" : "EVIDENT_CLAUDE_USAGE_REPORTING";
|
|
3021
|
-
return {
|
|
3022
|
-
mode: "auto",
|
|
3023
|
-
warnings: [
|
|
3024
|
-
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_MODES.join(", ")}; using auto`
|
|
3025
|
-
]
|
|
3026
|
-
};
|
|
4257
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
4258
|
+
flagName: "--claude-usage-reporting",
|
|
4259
|
+
envVar: "EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
4260
|
+
});
|
|
3027
4261
|
}
|
|
3028
|
-
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
3029
|
-
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
3030
4262
|
function nextReportDelayMs(random = Math.random) {
|
|
3031
|
-
return
|
|
4263
|
+
return usageReportDelayMs(random);
|
|
3032
4264
|
}
|
|
3033
4265
|
var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
|
|
3034
|
-
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
3035
4266
|
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
3036
|
-
return
|
|
4267
|
+
return usageReportFailureLogLevel(consecutiveFailures);
|
|
4268
|
+
}
|
|
4269
|
+
|
|
4270
|
+
// src/lib/openai-usage-reporting.ts
|
|
4271
|
+
function resolveOpenAiUsageReportingMode(flagValue, env) {
|
|
4272
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
4273
|
+
flagName: "--openai-usage-reporting",
|
|
4274
|
+
envVar: "EVIDENT_OPENAI_USAGE_REPORTING"
|
|
4275
|
+
});
|
|
3037
4276
|
}
|
|
3038
4277
|
|
|
3039
4278
|
// src/lib/resource-usage-reporting.ts
|
|
@@ -3192,15 +4431,15 @@ function createResourceUsageCollector(homeDir) {
|
|
|
3192
4431
|
}
|
|
3193
4432
|
|
|
3194
4433
|
// src/lib/channels/driver.ts
|
|
3195
|
-
import { homedir as
|
|
4434
|
+
import { homedir as homedir4 } from "os";
|
|
3196
4435
|
|
|
3197
4436
|
// src/lib/runner-file-sync.ts
|
|
3198
|
-
import { join as
|
|
4437
|
+
import { join as join7 } from "path";
|
|
3199
4438
|
|
|
3200
4439
|
// src/lib/file-push.ts
|
|
3201
4440
|
import { randomUUID } from "crypto";
|
|
3202
4441
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
3203
|
-
import { basename, dirname as
|
|
4442
|
+
import { basename, dirname as dirname5, isAbsolute, join as join6, relative, resolve as resolve2, sep } from "path";
|
|
3204
4443
|
var FILE_MODE = 384;
|
|
3205
4444
|
var DIRECTORY_MODE = 448;
|
|
3206
4445
|
async function writePushedFile(request) {
|
|
@@ -3231,9 +4470,9 @@ async function writePushedFile(request) {
|
|
|
3231
4470
|
}
|
|
3232
4471
|
try {
|
|
3233
4472
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3234
|
-
|
|
4473
|
+
dirname5(candidate)
|
|
3235
4474
|
);
|
|
3236
|
-
const realTarget =
|
|
4475
|
+
const realTarget = join6(existingAncestor, ...missingSegments, basename(candidate));
|
|
3237
4476
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3238
4477
|
if (allowedDirectory === null) {
|
|
3239
4478
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3243,8 +4482,8 @@ async function writePushedFile(request) {
|
|
|
3243
4482
|
}
|
|
3244
4483
|
if (missingSegments.length > 0) {
|
|
3245
4484
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
3246
|
-
const realParent = await realpath(
|
|
3247
|
-
if (realParent !==
|
|
4485
|
+
const realParent = await realpath(dirname5(realTarget));
|
|
4486
|
+
if (realParent !== dirname5(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
3248
4487
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
3249
4488
|
path: realTarget,
|
|
3250
4489
|
bytes,
|
|
@@ -3269,7 +4508,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3269
4508
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3270
4509
|
return null;
|
|
3271
4510
|
}
|
|
3272
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4511
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3273
4512
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3274
4513
|
return null;
|
|
3275
4514
|
}
|
|
@@ -3287,7 +4526,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
3287
4526
|
try {
|
|
3288
4527
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
3289
4528
|
} catch (err) {
|
|
3290
|
-
const parent =
|
|
4529
|
+
const parent = dirname5(current);
|
|
3291
4530
|
if (err.code !== "ENOENT" || parent === current) {
|
|
3292
4531
|
throw err;
|
|
3293
4532
|
}
|
|
@@ -3342,13 +4581,13 @@ function contains(realDirectory, realTarget) {
|
|
|
3342
4581
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3343
4582
|
let current = existingAncestor;
|
|
3344
4583
|
for (const segment of missingSegments) {
|
|
3345
|
-
current =
|
|
4584
|
+
current = join6(current, segment);
|
|
3346
4585
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3347
4586
|
await chmod(current, DIRECTORY_MODE);
|
|
3348
4587
|
}
|
|
3349
4588
|
}
|
|
3350
4589
|
async function writeAtomically(realTarget, content) {
|
|
3351
|
-
const temporaryPath =
|
|
4590
|
+
const temporaryPath = join6(dirname5(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3352
4591
|
let handle;
|
|
3353
4592
|
try {
|
|
3354
4593
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -3390,20 +4629,28 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3390
4629
|
for (const id of options.ackFailures.keys()) {
|
|
3391
4630
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3392
4631
|
}
|
|
3393
|
-
if (pending.length === 0)
|
|
4632
|
+
if (pending.length === 0) {
|
|
4633
|
+
return { applied: 0, claudeCredentialApplied: false, opencodeAuthApplied: false };
|
|
4634
|
+
}
|
|
3394
4635
|
options.log({
|
|
3395
4636
|
level: "info",
|
|
3396
4637
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3397
4638
|
});
|
|
3398
4639
|
let applied = 0;
|
|
3399
4640
|
let claudeCredentialApplied = false;
|
|
4641
|
+
let opencodeAuthApplied = false;
|
|
3400
4642
|
for (const file of pending) {
|
|
3401
4643
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3402
4644
|
const outcome = await applyOne(options, file);
|
|
3403
4645
|
if (outcome.applied) applied += 1;
|
|
3404
4646
|
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
4647
|
+
if (outcome.opencodeAuthApplied) opencodeAuthApplied = true;
|
|
3405
4648
|
}
|
|
3406
|
-
return {
|
|
4649
|
+
return {
|
|
4650
|
+
applied,
|
|
4651
|
+
claudeCredentialApplied,
|
|
4652
|
+
opencodeAuthApplied
|
|
4653
|
+
};
|
|
3407
4654
|
}
|
|
3408
4655
|
async function listPendingFiles(options) {
|
|
3409
4656
|
let res;
|
|
@@ -3464,10 +4711,18 @@ function asPendingFile(entry) {
|
|
|
3464
4711
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3465
4712
|
return { id, path, size };
|
|
3466
4713
|
}
|
|
3467
|
-
var NOT_APPLIED = {
|
|
4714
|
+
var NOT_APPLIED = {
|
|
4715
|
+
applied: false,
|
|
4716
|
+
claudeCredentialApplied: false,
|
|
4717
|
+
opencodeAuthApplied: false
|
|
4718
|
+
};
|
|
3468
4719
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3469
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3470
|
-
return expanded ===
|
|
4720
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4721
|
+
return expanded === join7(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
4722
|
+
}
|
|
4723
|
+
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
4724
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4725
|
+
return expanded === join7(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
3471
4726
|
}
|
|
3472
4727
|
async function applyOne(options, file) {
|
|
3473
4728
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -3523,7 +4778,8 @@ async function applyOne(options, file) {
|
|
|
3523
4778
|
await ack(options, file, "applied");
|
|
3524
4779
|
return {
|
|
3525
4780
|
applied: true,
|
|
3526
|
-
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir)
|
|
4781
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir),
|
|
4782
|
+
opencodeAuthApplied: isOpenCodeAuthPath(file.path, options.homeDir)
|
|
3527
4783
|
};
|
|
3528
4784
|
}
|
|
3529
4785
|
function durableDownloadCode(status2) {
|
|
@@ -3977,6 +5233,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3977
5233
|
* that way rather than "fixing" it into a count.
|
|
3978
5234
|
*/
|
|
3979
5235
|
claudeCredentialApplyCount = 0;
|
|
5236
|
+
opencodeAuthApplyCount = 0;
|
|
3980
5237
|
/**
|
|
3981
5238
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3982
5239
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -4011,7 +5268,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4011
5268
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
4012
5269
|
this.now = config.now ?? (() => Date.now());
|
|
4013
5270
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
4014
|
-
this.homeDir = config.homeDir ??
|
|
5271
|
+
this.homeDir = config.homeDir ?? homedir4();
|
|
4015
5272
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4016
5273
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4017
5274
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
@@ -4081,6 +5338,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4081
5338
|
});
|
|
4082
5339
|
this.appliedFileCount += result.applied;
|
|
4083
5340
|
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
5341
|
+
if (result.opencodeAuthApplied) this.opencodeAuthApplyCount += 1;
|
|
4084
5342
|
return result.applied;
|
|
4085
5343
|
} catch (err) {
|
|
4086
5344
|
this.log({
|
|
@@ -4188,7 +5446,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4188
5446
|
return {
|
|
4189
5447
|
appliedFiles: this.appliedFileCount,
|
|
4190
5448
|
inFlight: this.syncingFiles,
|
|
4191
|
-
claudeCredentialApplies: this.claudeCredentialApplyCount
|
|
5449
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount,
|
|
5450
|
+
opencodeAuthApplyCount: this.opencodeAuthApplyCount
|
|
4192
5451
|
};
|
|
4193
5452
|
}
|
|
4194
5453
|
/**
|
|
@@ -5276,6 +6535,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5276
6535
|
deliveryDeadlineAnchored: false,
|
|
5277
6536
|
b2PinnedSinceMs: 0,
|
|
5278
6537
|
b2LastDescendantCheckMs: 0,
|
|
6538
|
+
b2RootOngoingHeldLogged: false,
|
|
5279
6539
|
b2AbandonedSignalled: false,
|
|
5280
6540
|
ambiguousPinnedSinceMs: 0,
|
|
5281
6541
|
ambiguousResolved: false
|
|
@@ -5370,6 +6630,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5370
6630
|
deliveryDeadlineAnchored: false,
|
|
5371
6631
|
b2PinnedSinceMs: 0,
|
|
5372
6632
|
b2LastDescendantCheckMs: 0,
|
|
6633
|
+
b2RootOngoingHeldLogged: false,
|
|
5373
6634
|
b2AbandonedSignalled: false,
|
|
5374
6635
|
ambiguousPinnedSinceMs: 0,
|
|
5375
6636
|
ambiguousResolved: false
|
|
@@ -5723,6 +6984,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5723
6984
|
if (snapshotReadable) {
|
|
5724
6985
|
inFlight.b2PinnedSinceMs = 0;
|
|
5725
6986
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
6987
|
+
inFlight.b2RootOngoingHeldLogged = false;
|
|
5726
6988
|
inFlight.b2AbandonedSignalled = false;
|
|
5727
6989
|
}
|
|
5728
6990
|
} else {
|
|
@@ -5734,11 +6996,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5734
6996
|
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
5735
6997
|
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
5736
6998
|
inFlight.b2LastDescendantCheckMs = this.now();
|
|
5737
|
-
const descendantOngoing = await
|
|
6999
|
+
const [descendantOngoing, rootOngoing] = await Promise.all([
|
|
7000
|
+
this.isAnyDescendantSessionOngoing(sessionId),
|
|
7001
|
+
isSessionOngoing(this.port, sessionId)
|
|
7002
|
+
]);
|
|
5738
7003
|
if (isB2AbandonmentConfirmed({
|
|
5739
7004
|
pinnedForMs,
|
|
5740
7005
|
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
5741
|
-
descendantOngoing
|
|
7006
|
+
descendantOngoing,
|
|
7007
|
+
rootOngoing
|
|
5742
7008
|
})) {
|
|
5743
7009
|
inFlight.b2AbandonedSignalled = true;
|
|
5744
7010
|
this.log({
|
|
@@ -5747,12 +7013,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5747
7013
|
conversation_id: conv.id,
|
|
5748
7014
|
message_id: id
|
|
5749
7015
|
});
|
|
7016
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
5750
7017
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
5751
|
-
watched_for_ms: pinnedForMs
|
|
7018
|
+
watched_for_ms: pinnedForMs,
|
|
7019
|
+
finish: reply?.info?.finish ?? reply?.finish,
|
|
7020
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
7021
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
7022
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
5752
7023
|
});
|
|
5753
7024
|
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
5754
7025
|
return;
|
|
5755
7026
|
}
|
|
7027
|
+
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
7028
|
+
inFlight.b2RootOngoingHeldLogged = true;
|
|
7029
|
+
this.log({
|
|
7030
|
+
level: "warn",
|
|
7031
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with root_ongoing=${rootOngoing} and descendant_ongoing=${descendantOngoing} \u2014 holding until OpenCode confirms the root is idle`,
|
|
7032
|
+
conversation_id: conv.id,
|
|
7033
|
+
message_id: id
|
|
7034
|
+
});
|
|
7035
|
+
}
|
|
5756
7036
|
}
|
|
5757
7037
|
}
|
|
5758
7038
|
const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
|
|
@@ -7409,7 +8689,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
7409
8689
|
}
|
|
7410
8690
|
if (!ctx.interactive) {
|
|
7411
8691
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
7412
|
-
const proc = await startOpenCode(ctx.port);
|
|
8692
|
+
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
7413
8693
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
7414
8694
|
if (!health.healthy) {
|
|
7415
8695
|
return {
|
|
@@ -7477,7 +8757,7 @@ Port ${port} is already in use.`));
|
|
|
7477
8757
|
}
|
|
7478
8758
|
if (action === "start") {
|
|
7479
8759
|
const spinner = ora2("Starting OpenCode...").start();
|
|
7480
|
-
const proc = await startOpenCode(port);
|
|
8760
|
+
const proc = await startOpenCode(port, { inheritStdio: ctx.inheritStdio });
|
|
7481
8761
|
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
7482
8762
|
if (!health.healthy) {
|
|
7483
8763
|
spinner.fail("Failed to start OpenCode");
|
|
@@ -7489,12 +8769,323 @@ Port ${port} is already in use.`));
|
|
|
7489
8769
|
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
7490
8770
|
}
|
|
7491
8771
|
|
|
8772
|
+
// src/lib/runner-credentials.ts
|
|
8773
|
+
import { chmodSync as chmodSync2, writeFileSync as writeFileSync4 } from "fs";
|
|
8774
|
+
import { spawn as spawn5 } from "child_process";
|
|
8775
|
+
var RUNNER_SECRET_FETCH_TIMEOUT_MS = 6e4;
|
|
8776
|
+
var CREDENTIAL_RESTORE_TIMEOUT_MS = 6e4;
|
|
8777
|
+
var GITHUB_PROBE_TIMEOUT_MS = 1e4;
|
|
8778
|
+
var GIT_CONFIG_GLOBAL = "/tmp/gitconfig";
|
|
8779
|
+
var GIT_CREDENTIAL_HELPER = "/tmp/git-credential-helper.sh";
|
|
8780
|
+
function commandError2(result) {
|
|
8781
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
8782
|
+
}
|
|
8783
|
+
var runCommand2 = (command, args, opts) => {
|
|
8784
|
+
return new Promise((resolve4) => {
|
|
8785
|
+
let child;
|
|
8786
|
+
let stdout = "";
|
|
8787
|
+
let stderr = "";
|
|
8788
|
+
let settled = false;
|
|
8789
|
+
const timer = {};
|
|
8790
|
+
const finish = (result) => {
|
|
8791
|
+
if (settled) return;
|
|
8792
|
+
settled = true;
|
|
8793
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
8794
|
+
resolve4(result);
|
|
8795
|
+
};
|
|
8796
|
+
try {
|
|
8797
|
+
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
8798
|
+
} catch (error2) {
|
|
8799
|
+
finish({
|
|
8800
|
+
code: null,
|
|
8801
|
+
stdout,
|
|
8802
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
8803
|
+
timedOut: false
|
|
8804
|
+
});
|
|
8805
|
+
return;
|
|
8806
|
+
}
|
|
8807
|
+
child.stdout?.setEncoding("utf8");
|
|
8808
|
+
child.stdout?.on("data", (chunk) => {
|
|
8809
|
+
stdout += chunk;
|
|
8810
|
+
});
|
|
8811
|
+
child.stderr?.setEncoding("utf8");
|
|
8812
|
+
child.stderr?.on("data", (chunk) => {
|
|
8813
|
+
stderr += chunk;
|
|
8814
|
+
});
|
|
8815
|
+
child.once("error", (error2) => {
|
|
8816
|
+
finish({
|
|
8817
|
+
code: null,
|
|
8818
|
+
stdout,
|
|
8819
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
8820
|
+
${error2.message}`,
|
|
8821
|
+
timedOut: false
|
|
8822
|
+
});
|
|
8823
|
+
});
|
|
8824
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
8825
|
+
timer.handle = setTimeout(
|
|
8826
|
+
() => {
|
|
8827
|
+
child.kill("SIGKILL");
|
|
8828
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
8829
|
+
},
|
|
8830
|
+
Math.max(0, opts.timeoutMs)
|
|
8831
|
+
);
|
|
8832
|
+
});
|
|
8833
|
+
};
|
|
8834
|
+
function isEnvironmentObject(value) {
|
|
8835
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
8836
|
+
}
|
|
8837
|
+
function secretFailure(marker, detail, log3) {
|
|
8838
|
+
const message = `${marker}: ${detail}`;
|
|
8839
|
+
log3(message, "error");
|
|
8840
|
+
return new Error(message);
|
|
8841
|
+
}
|
|
8842
|
+
async function installRunnerSecret({
|
|
8843
|
+
env,
|
|
8844
|
+
log: log3,
|
|
8845
|
+
commandRunner
|
|
8846
|
+
}) {
|
|
8847
|
+
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
8848
|
+
if (!arn) {
|
|
8849
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
8850
|
+
return false;
|
|
8851
|
+
}
|
|
8852
|
+
const result = await (commandRunner ?? runCommand2)(
|
|
8853
|
+
"aws",
|
|
8854
|
+
[
|
|
8855
|
+
"secretsmanager",
|
|
8856
|
+
"get-secret-value",
|
|
8857
|
+
"--secret-id",
|
|
8858
|
+
arn,
|
|
8859
|
+
"--query",
|
|
8860
|
+
"SecretString",
|
|
8861
|
+
"--output",
|
|
8862
|
+
"text"
|
|
8863
|
+
],
|
|
8864
|
+
{ env, timeoutMs: RUNNER_SECRET_FETCH_TIMEOUT_MS }
|
|
8865
|
+
);
|
|
8866
|
+
if (result.timedOut) {
|
|
8867
|
+
throw secretFailure(
|
|
8868
|
+
"CREDENTIAL-RESTORE-TIMEOUT",
|
|
8869
|
+
`runner-secret did not finish within ${RUNNER_SECRET_FETCH_TIMEOUT_MS}ms`,
|
|
8870
|
+
log3
|
|
8871
|
+
);
|
|
8872
|
+
}
|
|
8873
|
+
if (result.code !== 0) {
|
|
8874
|
+
throw secretFailure("RUNNER-SECRET-UNREADABLE", commandError2(result), log3);
|
|
8875
|
+
}
|
|
8876
|
+
let payload;
|
|
8877
|
+
try {
|
|
8878
|
+
payload = JSON.parse(result.stdout);
|
|
8879
|
+
} catch (error2) {
|
|
8880
|
+
log3(
|
|
8881
|
+
`RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
8882
|
+
"warn"
|
|
8883
|
+
);
|
|
8884
|
+
return false;
|
|
8885
|
+
}
|
|
8886
|
+
if (!isEnvironmentObject(payload)) {
|
|
8887
|
+
log3("RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object", "warn");
|
|
8888
|
+
return false;
|
|
8889
|
+
}
|
|
8890
|
+
let populated = 0;
|
|
8891
|
+
let skipped = 0;
|
|
8892
|
+
let githubTokenPopulated = false;
|
|
8893
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
8894
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
8895
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
8896
|
+
log3(
|
|
8897
|
+
`RUNNER-SECRET-KEY-SKIPPED: ${JSON.stringify(key)} is not a valid environment variable name`,
|
|
8898
|
+
"warn"
|
|
8899
|
+
);
|
|
8900
|
+
skipped += 1;
|
|
8901
|
+
continue;
|
|
8902
|
+
}
|
|
8903
|
+
env[key] = value;
|
|
8904
|
+
populated += 1;
|
|
8905
|
+
if (key === "GH_TOKEN") githubTokenPopulated = true;
|
|
8906
|
+
}
|
|
8907
|
+
if (populated === 0) {
|
|
8908
|
+
log3(
|
|
8909
|
+
"RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md",
|
|
8910
|
+
"warn"
|
|
8911
|
+
);
|
|
8912
|
+
} else {
|
|
8913
|
+
log3(
|
|
8914
|
+
`RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names`
|
|
8915
|
+
);
|
|
8916
|
+
}
|
|
8917
|
+
return githubTokenPopulated;
|
|
8918
|
+
}
|
|
8919
|
+
function restoreFailure(operation, result, log3) {
|
|
8920
|
+
const message = `CREDENTIAL-RESTORE-FAILED: ${operation} failed: ${commandError2(result)}`;
|
|
8921
|
+
log3(message, "error");
|
|
8922
|
+
return new Error(message);
|
|
8923
|
+
}
|
|
8924
|
+
async function runCredentialRestore(args, operation, log3, synchroniserRunner) {
|
|
8925
|
+
const result = await synchroniserRunner(args, { timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS });
|
|
8926
|
+
if (result.timedOut) {
|
|
8927
|
+
log3(
|
|
8928
|
+
`CREDENTIAL-RESTORE-TIMEOUT: ${operation} did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
8929
|
+
"warn"
|
|
8930
|
+
);
|
|
8931
|
+
return result;
|
|
8932
|
+
}
|
|
8933
|
+
if (result.code !== 0) throw restoreFailure(operation, result, log3);
|
|
8934
|
+
return result;
|
|
8935
|
+
}
|
|
8936
|
+
async function restoreCredentialStores({
|
|
8937
|
+
env,
|
|
8938
|
+
log: log3,
|
|
8939
|
+
synchroniserRunner = runSynchroniser
|
|
8940
|
+
}) {
|
|
8941
|
+
await runCredentialRestore(["restore", "claude"], "restore-claude", log3, synchroniserRunner);
|
|
8942
|
+
await runCredentialRestore(["restore", "opencode"], "restore-opencode", log3, synchroniserRunner);
|
|
8943
|
+
const result = await synchroniserRunner(["model-auth-ready"], {
|
|
8944
|
+
timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS
|
|
8945
|
+
});
|
|
8946
|
+
if (result.timedOut) {
|
|
8947
|
+
log3(
|
|
8948
|
+
`CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
8949
|
+
"warn"
|
|
8950
|
+
);
|
|
8951
|
+
return;
|
|
8952
|
+
}
|
|
8953
|
+
switch (result.code) {
|
|
8954
|
+
case 0:
|
|
8955
|
+
return;
|
|
8956
|
+
case 10:
|
|
8957
|
+
log3(
|
|
8958
|
+
`no model credentials under s3://${env.LITESTREAM_BUCKET ?? ""}/${env.LITESTREAM_PREFIX ?? ""}/ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs a model provider fails until one is connected. See 'Seeding a credential store' in infrastructure/evident-runner/MICROVM.md.`,
|
|
8959
|
+
"warn"
|
|
8960
|
+
);
|
|
8961
|
+
return;
|
|
8962
|
+
default:
|
|
8963
|
+
log3("could not determine whether this VM has model credentials", "warn");
|
|
8964
|
+
}
|
|
8965
|
+
}
|
|
8966
|
+
var GIT_CREDENTIAL_HELPER_CONTENT = [
|
|
8967
|
+
"#!/usr/bin/env bash",
|
|
8968
|
+
'[ "$1" = get ] || exit 0',
|
|
8969
|
+
"echo username=x-access-token",
|
|
8970
|
+
'echo "password=${GH_TOKEN}"',
|
|
8971
|
+
""
|
|
8972
|
+
].join("\n");
|
|
8973
|
+
async function probeGitHubAccess({ env, log: log3 }) {
|
|
8974
|
+
const auth = await runCommand2("gh", ["api", "user", "--jq", ".login"], {
|
|
8975
|
+
env,
|
|
8976
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
8977
|
+
});
|
|
8978
|
+
if (auth.timedOut) {
|
|
8979
|
+
log3(`GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`, "warn");
|
|
8980
|
+
return;
|
|
8981
|
+
}
|
|
8982
|
+
if (auth.code !== 0) {
|
|
8983
|
+
log3(`GITHUB-AUTH-REJECTED: ${commandError2(auth)}`, "warn");
|
|
8984
|
+
return;
|
|
8985
|
+
}
|
|
8986
|
+
log3(`GITHUB-AUTH-OK: ${auth.stdout.trim()}`, "debug");
|
|
8987
|
+
const remote = await runCommand2("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
|
8988
|
+
env,
|
|
8989
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
8990
|
+
});
|
|
8991
|
+
if (remote.code !== 0 || remote.timedOut) return;
|
|
8992
|
+
const repo = remote.stdout.trim().replace(/^(?:https:\/\/github\.com\/|git@github\.com:)/, "").replace(/\.git$/, "");
|
|
8993
|
+
if (!repo) return;
|
|
8994
|
+
const repository = await runCommand2("gh", ["api", `repos/${repo}`, "--jq", ".full_name"], {
|
|
8995
|
+
env,
|
|
8996
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
8997
|
+
});
|
|
8998
|
+
if (repository.timedOut) {
|
|
8999
|
+
log3(
|
|
9000
|
+
`GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`,
|
|
9001
|
+
"warn"
|
|
9002
|
+
);
|
|
9003
|
+
} else if (repository.code !== 0) {
|
|
9004
|
+
log3(`GITHUB-REPO-INACCESSIBLE: ${repo}: ${commandError2(repository)}`, "warn");
|
|
9005
|
+
}
|
|
9006
|
+
}
|
|
9007
|
+
async function configureGitHubAccess({ env, log: log3 }) {
|
|
9008
|
+
if (!env.GH_TOKEN) {
|
|
9009
|
+
log3("GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above", "warn");
|
|
9010
|
+
return;
|
|
9011
|
+
}
|
|
9012
|
+
try {
|
|
9013
|
+
env.GIT_CONFIG_GLOBAL = GIT_CONFIG_GLOBAL;
|
|
9014
|
+
writeFileSync4(GIT_CONFIG_GLOBAL, "");
|
|
9015
|
+
writeFileSync4(GIT_CREDENTIAL_HELPER, GIT_CREDENTIAL_HELPER_CONTENT, { mode: 448 });
|
|
9016
|
+
chmodSync2(GIT_CREDENTIAL_HELPER, 448);
|
|
9017
|
+
const config = [
|
|
9018
|
+
["user.name", env.GIT_USER_NAME ?? "evident-bot"],
|
|
9019
|
+
["user.email", env.GIT_USER_EMAIL ?? "evident-bot@users.noreply.github.com"],
|
|
9020
|
+
["init.defaultBranch", "main"],
|
|
9021
|
+
["credential.https://github.com.helper", GIT_CREDENTIAL_HELPER]
|
|
9022
|
+
];
|
|
9023
|
+
for (const [key, value] of config) {
|
|
9024
|
+
const result = await runCommand2("git", ["config", "--global", key, value], {
|
|
9025
|
+
env,
|
|
9026
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9027
|
+
});
|
|
9028
|
+
if (result.timedOut || result.code !== 0) throw new Error(commandError2(result));
|
|
9029
|
+
}
|
|
9030
|
+
} catch (error2) {
|
|
9031
|
+
log3(
|
|
9032
|
+
`GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9033
|
+
"warn"
|
|
9034
|
+
);
|
|
9035
|
+
return;
|
|
9036
|
+
}
|
|
9037
|
+
void probeGitHubAccess({ env, log: log3 }).catch((error2) => {
|
|
9038
|
+
log3(
|
|
9039
|
+
`GITHUB-SETUP-FAILED: GitHub access probe failed: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9040
|
+
"warn"
|
|
9041
|
+
);
|
|
9042
|
+
});
|
|
9043
|
+
}
|
|
9044
|
+
|
|
9045
|
+
// src/lib/opencode/config-overlay.ts
|
|
9046
|
+
import { execFileSync as execFileSync2 } from "child_process";
|
|
9047
|
+
import { copyFileSync, existsSync as existsSync2, statSync as statSync5 } from "fs";
|
|
9048
|
+
import { isAbsolute as isAbsolute2, join as join8, resolve as resolve3 } from "path";
|
|
9049
|
+
function isFile(filePath) {
|
|
9050
|
+
return existsSync2(filePath) && statSync5(filePath).isFile();
|
|
9051
|
+
}
|
|
9052
|
+
function applyRunnerOpenCodeConfig({
|
|
9053
|
+
overlayPath,
|
|
9054
|
+
cwd = process.cwd(),
|
|
9055
|
+
log: log3
|
|
9056
|
+
}) {
|
|
9057
|
+
if (!overlayPath) {
|
|
9058
|
+
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
9059
|
+
return;
|
|
9060
|
+
}
|
|
9061
|
+
const source = isAbsolute2(overlayPath) ? overlayPath : resolve3(cwd, overlayPath);
|
|
9062
|
+
const target = isFile(join8(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
9063
|
+
if (!isFile(source)) {
|
|
9064
|
+
log3(
|
|
9065
|
+
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
9066
|
+
"error"
|
|
9067
|
+
);
|
|
9068
|
+
return;
|
|
9069
|
+
}
|
|
9070
|
+
copyFileSync(source, join8(cwd, target));
|
|
9071
|
+
try {
|
|
9072
|
+
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
9073
|
+
stdio: "ignore"
|
|
9074
|
+
});
|
|
9075
|
+
} catch (error2) {
|
|
9076
|
+
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
9077
|
+
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
9078
|
+
}
|
|
9079
|
+
log3(`Applied runner OpenCode config ${source} to ${join8(cwd, target)}`);
|
|
9080
|
+
}
|
|
9081
|
+
|
|
7492
9082
|
// src/commands/run.ts
|
|
7493
9083
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
7494
9084
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
7495
9085
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
7496
9086
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
7497
9087
|
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
9088
|
+
var CHILD_STOP_TIMEOUT_MS = 1e4;
|
|
7498
9089
|
function resolveLogLevel(options) {
|
|
7499
9090
|
const accepted = Object.keys(LOG_LEVELS);
|
|
7500
9091
|
const validate = (value, source) => {
|
|
@@ -7525,11 +9116,11 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
7525
9116
|
if (trimmed === "") {
|
|
7526
9117
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
7527
9118
|
}
|
|
7528
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7529
|
-
if (!
|
|
9119
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join9(homeDir, trimmed.slice(2)) : trimmed;
|
|
9120
|
+
if (!isAbsolute3(expanded)) {
|
|
7530
9121
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
7531
9122
|
}
|
|
7532
|
-
const normalized =
|
|
9123
|
+
const normalized = resolvePath2(expanded);
|
|
7533
9124
|
if (parse(normalized).root === normalized) {
|
|
7534
9125
|
throw new Error(
|
|
7535
9126
|
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
@@ -7623,7 +9214,7 @@ function logActivity(state, entry) {
|
|
|
7623
9214
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
7624
9215
|
if (!meetsThreshold(state, level)) return;
|
|
7625
9216
|
forwardRunnerActivity(
|
|
7626
|
-
{ level, message: entry.message, error: entry.error },
|
|
9217
|
+
{ level, message: entry.message, error: entry.error, metadata: entry.metadata },
|
|
7627
9218
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
7628
9219
|
);
|
|
7629
9220
|
const fullEntry = {
|
|
@@ -7643,6 +9234,36 @@ function logActivity(state, entry) {
|
|
|
7643
9234
|
}
|
|
7644
9235
|
}
|
|
7645
9236
|
}
|
|
9237
|
+
function reportSessionDbRecovery(state) {
|
|
9238
|
+
try {
|
|
9239
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir5(), env: process.env });
|
|
9240
|
+
for (const record of report.records) {
|
|
9241
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9242
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9243
|
+
logActivity(state, {
|
|
9244
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9245
|
+
level: activity.level,
|
|
9246
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9247
|
+
metadata: activity.metadata
|
|
9248
|
+
});
|
|
9249
|
+
}
|
|
9250
|
+
acknowledgeSessionDbRecoveryReport(report.path);
|
|
9251
|
+
} catch (error2) {
|
|
9252
|
+
console.error(
|
|
9253
|
+
`[run] could not report session-DB recovery activity: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
9254
|
+
);
|
|
9255
|
+
}
|
|
9256
|
+
}
|
|
9257
|
+
function reportSessionDbRecoveryRecord(state, record) {
|
|
9258
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9259
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9260
|
+
logActivity(state, {
|
|
9261
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9262
|
+
level: activity.level,
|
|
9263
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9264
|
+
metadata: activity.metadata
|
|
9265
|
+
});
|
|
9266
|
+
}
|
|
7646
9267
|
function displayStatus(state) {
|
|
7647
9268
|
if (!state.interactive) return;
|
|
7648
9269
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
@@ -7733,6 +9354,7 @@ async function driveChannels(state, driver) {
|
|
|
7733
9354
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7734
9355
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
7735
9356
|
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
9357
|
+
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
7736
9358
|
while (state.running) {
|
|
7737
9359
|
const cycleStartedAtMs = performance.now();
|
|
7738
9360
|
let idleThisCycle = false;
|
|
@@ -7765,6 +9387,10 @@ async function driveChannels(state, driver) {
|
|
|
7765
9387
|
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
7766
9388
|
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
7767
9389
|
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
9390
|
+
const opencodeAuthApplies = fileActivitySnapshot.opencodeAuthApplyCount;
|
|
9391
|
+
const opencodeAuthApplied = opencodeAuthApplies !== lastSeenOpencodeAuthApplies;
|
|
9392
|
+
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
9393
|
+
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
7768
9394
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7769
9395
|
idlePolls = 0;
|
|
7770
9396
|
idleMs = 0;
|
|
@@ -7810,7 +9436,7 @@ async function driveChannels(state, driver) {
|
|
|
7810
9436
|
}
|
|
7811
9437
|
}
|
|
7812
9438
|
}
|
|
7813
|
-
await new Promise((
|
|
9439
|
+
await new Promise((resolve4) => setTimeout(resolve4, CHANNEL_POLL_INTERVAL_MS));
|
|
7814
9440
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
7815
9441
|
if (idleThisCycle) idleMs += cycleMs;
|
|
7816
9442
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -7833,7 +9459,43 @@ async function driveChannels(state, driver) {
|
|
|
7833
9459
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7834
9460
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7835
9461
|
function sessionDbPath() {
|
|
7836
|
-
return
|
|
9462
|
+
return join9(homedir5(), ".local", "share", "opencode", "opencode.db");
|
|
9463
|
+
}
|
|
9464
|
+
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
9465
|
+
const record = {
|
|
9466
|
+
v: 1,
|
|
9467
|
+
event: "session_db_recovery",
|
|
9468
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9469
|
+
stage: "verify",
|
|
9470
|
+
outcome: "schema_provenance_mismatch",
|
|
9471
|
+
severity: "error",
|
|
9472
|
+
reason: provenance.reason ?? "schema-provenance-mismatch",
|
|
9473
|
+
litestream_exit_code: null,
|
|
9474
|
+
attempt: null,
|
|
9475
|
+
replica_objects: null,
|
|
9476
|
+
replica_bytes: null,
|
|
9477
|
+
quarantine_destination: null,
|
|
9478
|
+
quarantined_objects: null,
|
|
9479
|
+
quarantine_failed_objects: null,
|
|
9480
|
+
quarantined_bytes: null,
|
|
9481
|
+
verified_restore_point: null,
|
|
9482
|
+
restore_points_tried: null,
|
|
9483
|
+
provenance_reason: provenance.reason,
|
|
9484
|
+
provenance_migration_delta: provenance.migrationDelta,
|
|
9485
|
+
replication_suspended: false,
|
|
9486
|
+
dbPath: sessionDbPath(),
|
|
9487
|
+
recorded_version: provenance.recordedVersion,
|
|
9488
|
+
current_version: currentVersion,
|
|
9489
|
+
provenance_pre_boot_migration_count: preBootMigrationCount
|
|
9490
|
+
};
|
|
9491
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9492
|
+
if (!activity) throw new Error("could not map session-DB provenance activity");
|
|
9493
|
+
logActivity(state, {
|
|
9494
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9495
|
+
level: activity.level,
|
|
9496
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9497
|
+
metadata: activity.metadata
|
|
9498
|
+
});
|
|
7837
9499
|
}
|
|
7838
9500
|
async function runSweep(state, driver, config) {
|
|
7839
9501
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7880,7 +9542,7 @@ async function runSweep(state, driver, config) {
|
|
|
7880
9542
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
7881
9543
|
dbPath: sessionDbPath(),
|
|
7882
9544
|
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
7883
|
-
allowFullVacuum: protectedNow.size === 0
|
|
9545
|
+
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
7884
9546
|
});
|
|
7885
9547
|
if (reclaimResult.ok) {
|
|
7886
9548
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
@@ -7916,7 +9578,7 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7916
9578
|
for (const warning2 of config.warnings) {
|
|
7917
9579
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
7918
9580
|
}
|
|
7919
|
-
const dbBytes = statSessionDbBytes(
|
|
9581
|
+
const dbBytes = statSessionDbBytes(homedir5());
|
|
7920
9582
|
void (async () => {
|
|
7921
9583
|
const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
7922
9584
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -7944,23 +9606,20 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7944
9606
|
);
|
|
7945
9607
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
7946
9608
|
}
|
|
7947
|
-
function
|
|
7948
|
-
const { mode, warnings } =
|
|
7949
|
-
options.claudeUsageReporting,
|
|
7950
|
-
process.env
|
|
7951
|
-
);
|
|
9609
|
+
function scheduleUsageReporting(state, params) {
|
|
9610
|
+
const { mode, warnings } = params.resolved;
|
|
7952
9611
|
for (const warning2 of warnings) {
|
|
7953
9612
|
logActivity(state, {
|
|
7954
9613
|
type: "info",
|
|
7955
9614
|
level: "warn",
|
|
7956
|
-
message:
|
|
9615
|
+
message: `${params.label} usage reporting: ${warning2}`
|
|
7957
9616
|
});
|
|
7958
9617
|
}
|
|
7959
9618
|
if (mode === "off") {
|
|
7960
9619
|
logActivity(state, {
|
|
7961
9620
|
type: "info",
|
|
7962
9621
|
level: "debug",
|
|
7963
|
-
message:
|
|
9622
|
+
message: `${params.label} usage reporting is off (${params.offFlagHint})`
|
|
7964
9623
|
});
|
|
7965
9624
|
return null;
|
|
7966
9625
|
}
|
|
@@ -7969,7 +9628,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7969
9628
|
let rearmRequested = false;
|
|
7970
9629
|
const armProbe = () => {
|
|
7971
9630
|
phase = "probe-pending";
|
|
7972
|
-
|
|
9631
|
+
params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
|
|
7973
9632
|
};
|
|
7974
9633
|
const scheduleNextTick = () => {
|
|
7975
9634
|
if (rearmRequested) {
|
|
@@ -7978,7 +9637,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7978
9637
|
return;
|
|
7979
9638
|
}
|
|
7980
9639
|
phase = "steady-pending";
|
|
7981
|
-
|
|
9640
|
+
params.setTimer(setTimeout(() => void tick(false), params.nextDelayMs()));
|
|
7982
9641
|
};
|
|
7983
9642
|
const rearm = () => {
|
|
7984
9643
|
switch (phase) {
|
|
@@ -7988,9 +9647,9 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7988
9647
|
case "probe-pending":
|
|
7989
9648
|
return;
|
|
7990
9649
|
case "steady-pending":
|
|
7991
|
-
if (
|
|
7992
|
-
clearTimeout(
|
|
7993
|
-
|
|
9650
|
+
if (params.getTimer()) {
|
|
9651
|
+
clearTimeout(params.getTimer());
|
|
9652
|
+
params.setTimer(null);
|
|
7994
9653
|
}
|
|
7995
9654
|
rearmRequested = false;
|
|
7996
9655
|
armProbe();
|
|
@@ -8004,45 +9663,45 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8004
9663
|
const tick = async (isProbe) => {
|
|
8005
9664
|
phase = "tick-in-flight";
|
|
8006
9665
|
try {
|
|
8007
|
-
const usage = await
|
|
8008
|
-
const result = await
|
|
9666
|
+
const usage = await params.fetchUsage();
|
|
9667
|
+
const result = await params.report(usage);
|
|
8009
9668
|
if (result.ok) {
|
|
8010
9669
|
if (consecutiveFailures > 0) {
|
|
8011
9670
|
logActivity(state, {
|
|
8012
9671
|
type: "info",
|
|
8013
9672
|
level: "info",
|
|
8014
|
-
message:
|
|
9673
|
+
message: `${params.label} usage reporting recovered`
|
|
8015
9674
|
});
|
|
8016
9675
|
}
|
|
8017
9676
|
consecutiveFailures = 0;
|
|
8018
9677
|
logActivity(state, {
|
|
8019
9678
|
type: "info",
|
|
8020
9679
|
level: "debug",
|
|
8021
|
-
message:
|
|
9680
|
+
message: `Reported ${params.label} usage to Evident`
|
|
8022
9681
|
});
|
|
8023
9682
|
} else {
|
|
8024
9683
|
consecutiveFailures++;
|
|
8025
9684
|
logActivity(state, {
|
|
8026
9685
|
type: "info",
|
|
8027
|
-
level:
|
|
8028
|
-
message: `Failed to report
|
|
9686
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
9687
|
+
message: `Failed to report ${params.label} usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
8029
9688
|
});
|
|
8030
9689
|
}
|
|
8031
9690
|
scheduleNextTick();
|
|
8032
9691
|
} catch (error2) {
|
|
8033
|
-
if (
|
|
9692
|
+
if (params.isLocalCredentialProblem(error2)) {
|
|
8034
9693
|
if (mode === "on") {
|
|
8035
9694
|
logActivity(state, {
|
|
8036
9695
|
type: "info",
|
|
8037
9696
|
level: "warn",
|
|
8038
|
-
message:
|
|
9697
|
+
message: `${params.label} usage reporting is forced on but no usable login was found \u2014 ${params.forcedOnHint}; reporting will keep retrying`
|
|
8039
9698
|
});
|
|
8040
9699
|
scheduleNextTick();
|
|
8041
9700
|
} else if (isProbe) {
|
|
8042
9701
|
logActivity(state, {
|
|
8043
9702
|
type: "info",
|
|
8044
9703
|
level: "debug",
|
|
8045
|
-
message:
|
|
9704
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
8046
9705
|
});
|
|
8047
9706
|
phase = "dormant";
|
|
8048
9707
|
if (rearmRequested) rearm();
|
|
@@ -8050,7 +9709,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8050
9709
|
logActivity(state, {
|
|
8051
9710
|
type: "info",
|
|
8052
9711
|
level: "debug",
|
|
8053
|
-
message:
|
|
9712
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
8054
9713
|
});
|
|
8055
9714
|
scheduleNextTick();
|
|
8056
9715
|
}
|
|
@@ -8059,8 +9718,8 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8059
9718
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8060
9719
|
logActivity(state, {
|
|
8061
9720
|
type: "info",
|
|
8062
|
-
level:
|
|
8063
|
-
message:
|
|
9721
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
9722
|
+
message: `${params.label} usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
8064
9723
|
});
|
|
8065
9724
|
scheduleNextTick();
|
|
8066
9725
|
}
|
|
@@ -8069,6 +9728,34 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8069
9728
|
armProbe();
|
|
8070
9729
|
return rearm;
|
|
8071
9730
|
}
|
|
9731
|
+
function scheduleClaudeUsageReporting(state, options) {
|
|
9732
|
+
return scheduleUsageReporting(state, {
|
|
9733
|
+
label: "Claude",
|
|
9734
|
+
resolved: resolveClaudeUsageReportingMode(options.claudeUsageReporting, process.env),
|
|
9735
|
+
offFlagHint: "--claude-usage-reporting off",
|
|
9736
|
+
getTimer: () => state.claudeUsageTimer,
|
|
9737
|
+
setTimer: (timer) => {
|
|
9738
|
+
state.claudeUsageTimer = timer;
|
|
9739
|
+
},
|
|
9740
|
+
fetchUsage: async () => {
|
|
9741
|
+
const usage = await getClaudeUsage();
|
|
9742
|
+
if (usage.ownerLookupError) {
|
|
9743
|
+
logActivity(state, {
|
|
9744
|
+
type: "info",
|
|
9745
|
+
level: "debug",
|
|
9746
|
+
message: `Claude usage owner lookup failed: ${usage.ownerLookupError}`
|
|
9747
|
+
});
|
|
9748
|
+
}
|
|
9749
|
+
return usage;
|
|
9750
|
+
},
|
|
9751
|
+
report: (usage) => reportClaudeUsage(state.agentId, state.authHeader, usage),
|
|
9752
|
+
isLocalCredentialProblem,
|
|
9753
|
+
forcedOnHint: "run `claude` to sign in",
|
|
9754
|
+
firstDelayMs: () => FIRST_REPORT_DELAY_MS,
|
|
9755
|
+
nextDelayMs: nextReportDelayMs,
|
|
9756
|
+
failureLogLevel: claudeUsageFailureLogLevel
|
|
9757
|
+
});
|
|
9758
|
+
}
|
|
8072
9759
|
var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
8073
9760
|
var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
8074
9761
|
var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
@@ -8092,7 +9779,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
8092
9779
|
});
|
|
8093
9780
|
return;
|
|
8094
9781
|
}
|
|
8095
|
-
const collect = createResourceUsageCollector(
|
|
9782
|
+
const collect = createResourceUsageCollector(homedir5());
|
|
8096
9783
|
let consecutiveFailures = 0;
|
|
8097
9784
|
const tick = async () => {
|
|
8098
9785
|
try {
|
|
@@ -8193,6 +9880,11 @@ async function cleanup(state, opts = {}) {
|
|
|
8193
9880
|
state.claudeUsageTimer = null;
|
|
8194
9881
|
}
|
|
8195
9882
|
state.claudeUsageRearm = null;
|
|
9883
|
+
if (state.openaiUsageTimer) {
|
|
9884
|
+
clearTimeout(state.openaiUsageTimer);
|
|
9885
|
+
state.openaiUsageTimer = null;
|
|
9886
|
+
}
|
|
9887
|
+
state.openaiUsageRearm = null;
|
|
8196
9888
|
if (state.resourceUsageTimer) {
|
|
8197
9889
|
clearTimeout(state.resourceUsageTimer);
|
|
8198
9890
|
state.resourceUsageTimer = null;
|
|
@@ -8227,15 +9919,31 @@ async function cleanup(state, opts = {}) {
|
|
|
8227
9919
|
}
|
|
8228
9920
|
if (state.opencodeProcess) {
|
|
8229
9921
|
const opencodeProcess = state.opencodeProcess;
|
|
8230
|
-
|
|
9922
|
+
const result = await timeShutdownPhase(
|
|
9923
|
+
state,
|
|
9924
|
+
durations,
|
|
9925
|
+
"opencode_stop",
|
|
9926
|
+
() => stopOpenCodeAndWait(opencodeProcess, CHILD_STOP_TIMEOUT_MS)
|
|
9927
|
+
);
|
|
8231
9928
|
if (state.interactive) {
|
|
8232
|
-
logActivity(state, { type: "info", message:
|
|
9929
|
+
logActivity(state, { type: "info", message: `Stopped OpenCode process (${result.outcome})` });
|
|
8233
9930
|
displayStatus(state);
|
|
8234
9931
|
} else {
|
|
8235
|
-
log2(state,
|
|
9932
|
+
log2(state, `Stopped OpenCode process (${result.outcome})`);
|
|
8236
9933
|
}
|
|
8237
9934
|
state.opencodeProcess = null;
|
|
8238
9935
|
}
|
|
9936
|
+
if (state.litestreamProcess) {
|
|
9937
|
+
const litestreamProcess = state.litestreamProcess;
|
|
9938
|
+
const result = await timeShutdownPhase(
|
|
9939
|
+
state,
|
|
9940
|
+
durations,
|
|
9941
|
+
"litestream_stop",
|
|
9942
|
+
() => stopSessionDbReplication(litestreamProcess, CHILD_STOP_TIMEOUT_MS)
|
|
9943
|
+
);
|
|
9944
|
+
log2(state, `Stopped litestream replication (${result.outcome})`);
|
|
9945
|
+
state.litestreamProcess = null;
|
|
9946
|
+
}
|
|
8239
9947
|
return durations;
|
|
8240
9948
|
}
|
|
8241
9949
|
async function run(options) {
|
|
@@ -8244,7 +9952,12 @@ async function run(options) {
|
|
|
8244
9952
|
let fileSyncDirectories;
|
|
8245
9953
|
try {
|
|
8246
9954
|
logLevel = resolveLogLevel(options);
|
|
8247
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
9955
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir5());
|
|
9956
|
+
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
9957
|
+
throw new Error(
|
|
9958
|
+
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
9959
|
+
);
|
|
9960
|
+
}
|
|
8248
9961
|
} catch (error2) {
|
|
8249
9962
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8250
9963
|
if (options.json) {
|
|
@@ -8268,7 +9981,9 @@ async function run(options) {
|
|
|
8268
9981
|
connected: false,
|
|
8269
9982
|
opencodeConnected: false,
|
|
8270
9983
|
opencodeVersion: null,
|
|
9984
|
+
sessionDbProvenanceAnomaly: false,
|
|
8271
9985
|
opencodeProcess: null,
|
|
9986
|
+
litestreamProcess: null,
|
|
8272
9987
|
connection: null,
|
|
8273
9988
|
channelDriver: null,
|
|
8274
9989
|
running: true,
|
|
@@ -8279,6 +9994,8 @@ async function run(options) {
|
|
|
8279
9994
|
sessionCleanupTimers: [],
|
|
8280
9995
|
claudeUsageTimer: null,
|
|
8281
9996
|
claudeUsageRearm: null,
|
|
9997
|
+
openaiUsageTimer: null,
|
|
9998
|
+
openaiUsageRearm: null,
|
|
8282
9999
|
resourceUsageTimer: null,
|
|
8283
10000
|
authHeader: ""
|
|
8284
10001
|
};
|
|
@@ -8333,8 +10050,8 @@ async function run(options) {
|
|
|
8333
10050
|
return true;
|
|
8334
10051
|
}
|
|
8335
10052
|
);
|
|
8336
|
-
const timedOut = new Promise((
|
|
8337
|
-
timer = setTimeout(() =>
|
|
10053
|
+
const timedOut = new Promise((resolve4) => {
|
|
10054
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
8338
10055
|
});
|
|
8339
10056
|
if (!await Promise.race([flushed, timedOut])) {
|
|
8340
10057
|
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
@@ -8474,6 +10191,67 @@ async function run(options) {
|
|
|
8474
10191
|
} else {
|
|
8475
10192
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
8476
10193
|
}
|
|
10194
|
+
if (options.restoreRunnerCredentials) {
|
|
10195
|
+
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
10196
|
+
const credentialContext = {
|
|
10197
|
+
env: process.env,
|
|
10198
|
+
log: (message, level = "info") => {
|
|
10199
|
+
if (level === "error") {
|
|
10200
|
+
logActivity(state, { type: "error", error: message });
|
|
10201
|
+
} else {
|
|
10202
|
+
logActivity(state, { type: "info", level, message });
|
|
10203
|
+
}
|
|
10204
|
+
}
|
|
10205
|
+
};
|
|
10206
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
10207
|
+
await restoreCredentialStores(credentialContext);
|
|
10208
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
10209
|
+
}
|
|
10210
|
+
let sessionDbVerifyFatal = false;
|
|
10211
|
+
if (!options.restoreSessionDb) {
|
|
10212
|
+
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
10213
|
+
} else {
|
|
10214
|
+
const health = await checkOpenCodeHealth(state.port);
|
|
10215
|
+
if (health.healthy) {
|
|
10216
|
+
log2(
|
|
10217
|
+
state,
|
|
10218
|
+
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
10219
|
+
"debug"
|
|
10220
|
+
);
|
|
10221
|
+
} else {
|
|
10222
|
+
const result = await restoreAndVerifySessionDb({
|
|
10223
|
+
dbPath: sessionDbPath(),
|
|
10224
|
+
litestreamConfig: options.litestreamConfig,
|
|
10225
|
+
noReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
10226
|
+
env: process.env,
|
|
10227
|
+
log: (message, level = "info") => {
|
|
10228
|
+
if (level === "error") {
|
|
10229
|
+
logActivity(state, { type: "error", error: message });
|
|
10230
|
+
} else {
|
|
10231
|
+
logActivity(state, { type: "info", level, message });
|
|
10232
|
+
}
|
|
10233
|
+
},
|
|
10234
|
+
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
10235
|
+
});
|
|
10236
|
+
sessionDbVerifyFatal = result.verifyFatal;
|
|
10237
|
+
}
|
|
10238
|
+
}
|
|
10239
|
+
reportSessionDbRecovery(state);
|
|
10240
|
+
if (sessionDbVerifyFatal) {
|
|
10241
|
+
throw new Error(
|
|
10242
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
10243
|
+
);
|
|
10244
|
+
}
|
|
10245
|
+
applyRunnerOpenCodeConfig({
|
|
10246
|
+
overlayPath: options.opencodeConfigOverlay,
|
|
10247
|
+
log: (message, level = "info") => {
|
|
10248
|
+
if (level === "error") {
|
|
10249
|
+
logActivity(state, { type: "error", error: message });
|
|
10250
|
+
} else {
|
|
10251
|
+
logActivity(state, { type: "info", level, message });
|
|
10252
|
+
}
|
|
10253
|
+
}
|
|
10254
|
+
});
|
|
8477
10255
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
8478
10256
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
8479
10257
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
@@ -8482,6 +10260,7 @@ async function run(options) {
|
|
|
8482
10260
|
for (const warning2 of maxActiveSessionsWarnings) {
|
|
8483
10261
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
8484
10262
|
}
|
|
10263
|
+
const preBootMigrationIds = readSessionDbMigrationIds(sessionDbPath());
|
|
8485
10264
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
8486
10265
|
try {
|
|
8487
10266
|
const oc = await ensureOpenCodeRunning({
|
|
@@ -8489,11 +10268,41 @@ async function run(options) {
|
|
|
8489
10268
|
interactive: state.interactive,
|
|
8490
10269
|
agentId: state.agentId,
|
|
8491
10270
|
log: (message) => log2(state, message),
|
|
8492
|
-
startTimeoutMs: opencodeStartTimeoutMs
|
|
10271
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
10272
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
8493
10273
|
});
|
|
8494
10274
|
state.port = oc.port;
|
|
8495
|
-
state.opencodeProcess = oc.process;
|
|
10275
|
+
state.opencodeProcess = options.opencodePidFile ? null : oc.process;
|
|
8496
10276
|
state.opencodeVersion = oc.version;
|
|
10277
|
+
if (options.opencodePidFile && oc.process?.pid !== void 0) {
|
|
10278
|
+
try {
|
|
10279
|
+
writeFileSync5(options.opencodePidFile, `${oc.process.pid}
|
|
10280
|
+
`, { mode: 384 });
|
|
10281
|
+
chmodSync3(options.opencodePidFile, 384);
|
|
10282
|
+
} catch (error2) {
|
|
10283
|
+
logActivity(state, {
|
|
10284
|
+
type: "error",
|
|
10285
|
+
error: `Failed to write OpenCode pid file ${options.opencodePidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10286
|
+
});
|
|
10287
|
+
}
|
|
10288
|
+
}
|
|
10289
|
+
if (state.opencodeVersion !== null) {
|
|
10290
|
+
const provenance = checkSessionDbProvenance({
|
|
10291
|
+
dbPath: sessionDbPath(),
|
|
10292
|
+
currentVersion: state.opencodeVersion,
|
|
10293
|
+
homeDir: homedir5(),
|
|
10294
|
+
env: process.env
|
|
10295
|
+
});
|
|
10296
|
+
if (provenance.anomaly) {
|
|
10297
|
+
state.sessionDbProvenanceAnomaly = true;
|
|
10298
|
+
logSessionDbProvenanceMismatch(
|
|
10299
|
+
state,
|
|
10300
|
+
provenance,
|
|
10301
|
+
state.opencodeVersion,
|
|
10302
|
+
preBootMigrationIds?.length ?? null
|
|
10303
|
+
);
|
|
10304
|
+
}
|
|
10305
|
+
}
|
|
8497
10306
|
state.opencodeConnected = oc.notReadyReason === null;
|
|
8498
10307
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
8499
10308
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
@@ -8530,6 +10339,108 @@ async function run(options) {
|
|
|
8530
10339
|
ocSpinner?.fail(error2.message);
|
|
8531
10340
|
throw error2;
|
|
8532
10341
|
}
|
|
10342
|
+
if (options.litestreamPidFile) {
|
|
10343
|
+
if (options.sessionDbNoReplicateMarker && existsSync3(options.sessionDbNoReplicateMarker)) {
|
|
10344
|
+
log2(
|
|
10345
|
+
state,
|
|
10346
|
+
`Skipping Litestream replication because ${options.sessionDbNoReplicateMarker} marks this session database unsafe to replicate`
|
|
10347
|
+
);
|
|
10348
|
+
} else if (!options.litestreamConfig) {
|
|
10349
|
+
logActivity(state, {
|
|
10350
|
+
type: "info",
|
|
10351
|
+
level: "warn",
|
|
10352
|
+
message: "Skipping Litestream replication because no configuration file was provided"
|
|
10353
|
+
});
|
|
10354
|
+
} else {
|
|
10355
|
+
let existingPid;
|
|
10356
|
+
if (existsSync3(options.litestreamPidFile)) {
|
|
10357
|
+
try {
|
|
10358
|
+
const rawPid = readFileSync6(options.litestreamPidFile, "utf8").trim();
|
|
10359
|
+
const parsedPid = Number(rawPid);
|
|
10360
|
+
if (/^\d+$/.test(rawPid) && Number.isSafeInteger(parsedPid) && parsedPid > 0) {
|
|
10361
|
+
existingPid = parsedPid;
|
|
10362
|
+
}
|
|
10363
|
+
} catch (error2) {
|
|
10364
|
+
logActivity(state, {
|
|
10365
|
+
type: "info",
|
|
10366
|
+
level: "warn",
|
|
10367
|
+
message: `Could not read Litestream pid file ${options.litestreamPidFile}; checking for a new process: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10368
|
+
});
|
|
10369
|
+
}
|
|
10370
|
+
}
|
|
10371
|
+
if (existingPid !== void 0 && isProcessAlive(existingPid)) {
|
|
10372
|
+
log2(state, `Litestream replication is already running with pid ${existingPid}`);
|
|
10373
|
+
} else {
|
|
10374
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
10375
|
+
state.litestreamProcess = null;
|
|
10376
|
+
let failureHandled = false;
|
|
10377
|
+
const reportImageOwnedReplicationFailure = (message) => {
|
|
10378
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
10379
|
+
failureHandled = true;
|
|
10380
|
+
logActivity(state, { type: "error", error: message });
|
|
10381
|
+
if (state.interactive) displayStatus(state);
|
|
10382
|
+
};
|
|
10383
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
10384
|
+
reportImageOwnedReplicationFailure(
|
|
10385
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
10386
|
+
);
|
|
10387
|
+
});
|
|
10388
|
+
litestreamProcess.on("error", (error2) => {
|
|
10389
|
+
reportImageOwnedReplicationFailure(
|
|
10390
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10391
|
+
);
|
|
10392
|
+
});
|
|
10393
|
+
try {
|
|
10394
|
+
if (litestreamProcess.pid !== void 0) {
|
|
10395
|
+
writeFileSync5(options.litestreamPidFile, `${litestreamProcess.pid}
|
|
10396
|
+
`, {
|
|
10397
|
+
mode: 384
|
|
10398
|
+
});
|
|
10399
|
+
chmodSync3(options.litestreamPidFile, 384);
|
|
10400
|
+
}
|
|
10401
|
+
} catch (error2) {
|
|
10402
|
+
logActivity(state, {
|
|
10403
|
+
type: "error",
|
|
10404
|
+
error: `Failed to write Litestream pid file ${options.litestreamPidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10405
|
+
});
|
|
10406
|
+
}
|
|
10407
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
10408
|
+
}
|
|
10409
|
+
}
|
|
10410
|
+
} else if (options.litestreamConfig) {
|
|
10411
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
10412
|
+
state.litestreamProcess = litestreamProcess;
|
|
10413
|
+
let failureHandled = false;
|
|
10414
|
+
const failRunForReplication = (message) => {
|
|
10415
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
10416
|
+
failureHandled = true;
|
|
10417
|
+
state.shuttingDown = true;
|
|
10418
|
+
logActivity(state, { type: "error", error: message });
|
|
10419
|
+
if (state.interactive) displayStatus(state);
|
|
10420
|
+
void (async () => {
|
|
10421
|
+
try {
|
|
10422
|
+
await cleanup(state);
|
|
10423
|
+
await shutdownTelemetry();
|
|
10424
|
+
} catch (error2) {
|
|
10425
|
+
console.error(
|
|
10426
|
+
`[run] replication failure cleanup failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10427
|
+
);
|
|
10428
|
+
}
|
|
10429
|
+
process.exit(1);
|
|
10430
|
+
})();
|
|
10431
|
+
};
|
|
10432
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
10433
|
+
failRunForReplication(
|
|
10434
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
10435
|
+
);
|
|
10436
|
+
});
|
|
10437
|
+
litestreamProcess.on("error", (error2) => {
|
|
10438
|
+
failRunForReplication(
|
|
10439
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10440
|
+
);
|
|
10441
|
+
});
|
|
10442
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
10443
|
+
}
|
|
8533
10444
|
const tunnelSpinner = interactive && !state.json ? ora3("Connecting tunnel...").start() : null;
|
|
8534
10445
|
const channelDriver = new ChannelDriver({
|
|
8535
10446
|
agentId: state.agentId,
|
|
@@ -8541,7 +10452,7 @@ async function run(options) {
|
|
|
8541
10452
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
8542
10453
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
8543
10454
|
fileSyncDirectories,
|
|
8544
|
-
homeDir:
|
|
10455
|
+
homeDir: homedir5(),
|
|
8545
10456
|
maxActiveSessions,
|
|
8546
10457
|
log: (entry) => (
|
|
8547
10458
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -8679,6 +10590,22 @@ async function run(options) {
|
|
|
8679
10590
|
}
|
|
8680
10591
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
8681
10592
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
10593
|
+
state.openaiUsageRearm = scheduleUsageReporting(state, {
|
|
10594
|
+
label: "OpenAI",
|
|
10595
|
+
resolved: resolveOpenAiUsageReportingMode(options.openaiUsageReporting, process.env),
|
|
10596
|
+
offFlagHint: "--openai-usage-reporting off",
|
|
10597
|
+
getTimer: () => state.openaiUsageTimer,
|
|
10598
|
+
setTimer: (timer) => {
|
|
10599
|
+
state.openaiUsageTimer = timer;
|
|
10600
|
+
},
|
|
10601
|
+
fetchUsage: () => getOpenAiUsage(state.port),
|
|
10602
|
+
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
10603
|
+
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
10604
|
+
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
10605
|
+
firstDelayMs: firstReportDelayMs,
|
|
10606
|
+
nextDelayMs: usageReportDelayMs,
|
|
10607
|
+
failureLogLevel: usageReportFailureLogLevel
|
|
10608
|
+
});
|
|
8682
10609
|
scheduleResourceUsageReporting(state, options);
|
|
8683
10610
|
if (!interactive || state.json) {
|
|
8684
10611
|
log2(state, "Driving channel messages...");
|
|
@@ -8717,7 +10644,7 @@ async function run(options) {
|
|
|
8717
10644
|
}
|
|
8718
10645
|
|
|
8719
10646
|
// src/index.ts
|
|
8720
|
-
var { version } =
|
|
10647
|
+
var { version } = createRequire2(import.meta.url)("../package.json");
|
|
8721
10648
|
var program = new Command();
|
|
8722
10649
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
8723
10650
|
"--endpoint <url>",
|
|
@@ -8760,6 +10687,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8760
10687
|
).option(
|
|
8761
10688
|
"--claude-usage-reporting <mode>",
|
|
8762
10689
|
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
10690
|
+
).option(
|
|
10691
|
+
"--openai-usage-reporting <mode>",
|
|
10692
|
+
"Report OpenAI plan usage to Evident: auto | on | off (default: auto). Env: EVIDENT_OPENAI_USAGE_REPORTING"
|
|
8763
10693
|
).option(
|
|
8764
10694
|
"--no-resource-usage-reporting",
|
|
8765
10695
|
"Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
|
|
@@ -8771,6 +10701,27 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8771
10701
|
).option(
|
|
8772
10702
|
"--tunnel-ready-file <path>",
|
|
8773
10703
|
"Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
|
|
10704
|
+
).option(
|
|
10705
|
+
"--litestream-config <path>",
|
|
10706
|
+
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
10707
|
+
).option(
|
|
10708
|
+
"--opencode-pid-file <path>",
|
|
10709
|
+
"Record the OpenCode pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
10710
|
+
).option(
|
|
10711
|
+
"--litestream-pid-file <path>",
|
|
10712
|
+
"Record the Litestream pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
10713
|
+
).option(
|
|
10714
|
+
"--session-db-no-replicate-marker <path>",
|
|
10715
|
+
"Read this marker as a gate before starting Litestream; the session-DB restore writes it when this boot's database is not safe to replicate."
|
|
10716
|
+
).option(
|
|
10717
|
+
"--restore-session-db",
|
|
10718
|
+
"Restore and verify the OpenCode session database before starting OpenCode; requires --litestream-config for the replica and --session-db-no-replicate-marker for give-up records."
|
|
10719
|
+
).option(
|
|
10720
|
+
"--restore-runner-credentials",
|
|
10721
|
+
"Restore the hosted runner secret and persisted credential stores before starting OpenCode."
|
|
10722
|
+
).option(
|
|
10723
|
+
"--opencode-config-overlay <path>",
|
|
10724
|
+
"Apply this runner-provided OpenCode config before starting OpenCode."
|
|
8774
10725
|
).action(
|
|
8775
10726
|
(options) => {
|
|
8776
10727
|
run({
|
|
@@ -8795,13 +10746,21 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8795
10746
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
8796
10747
|
// (resolveClaudeUsageReportingMode).
|
|
8797
10748
|
claudeUsageReporting: options.claudeUsageReporting,
|
|
10749
|
+
openaiUsageReporting: options.openaiUsageReporting,
|
|
8798
10750
|
// Raw value — resolution is single-sourced in run.ts's
|
|
8799
10751
|
// resolveResourceUsageReportingEnabled.
|
|
8800
10752
|
resourceUsageReporting: options.resourceUsageReporting,
|
|
8801
10753
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
8802
10754
|
// resolveFileSyncDirectories.
|
|
8803
10755
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
8804
|
-
tunnelReadyFile: options.tunnelReadyFile
|
|
10756
|
+
tunnelReadyFile: options.tunnelReadyFile,
|
|
10757
|
+
litestreamConfig: options.litestreamConfig,
|
|
10758
|
+
opencodePidFile: options.opencodePidFile,
|
|
10759
|
+
litestreamPidFile: options.litestreamPidFile,
|
|
10760
|
+
sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
10761
|
+
restoreSessionDb: options.restoreSessionDb,
|
|
10762
|
+
restoreRunnerCredentials: options.restoreRunnerCredentials,
|
|
10763
|
+
opencodeConfigOverlay: options.opencodeConfigOverlay
|
|
8805
10764
|
});
|
|
8806
10765
|
}
|
|
8807
10766
|
);
|