@evident-ai/cli 3.4.1-dev.b1bf8c5 → 3.4.1-dev.b9f71fa
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/index.js +2658 -381
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire } from "module";
|
|
4
|
+
import { createRequire as createRequire2 } from "module";
|
|
5
5
|
import { Command } from "commander";
|
|
6
6
|
|
|
7
7
|
// src/commands/login.ts
|
|
@@ -371,14 +371,14 @@ function blank() {
|
|
|
371
371
|
console.log();
|
|
372
372
|
}
|
|
373
373
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
374
|
-
return new Promise((
|
|
374
|
+
return new Promise((resolve4) => {
|
|
375
375
|
process.stdout.write(chalk.dim(prompt));
|
|
376
376
|
const handler = () => {
|
|
377
377
|
process.stdin.removeListener("data", handler);
|
|
378
378
|
process.stdin.setRawMode?.(false);
|
|
379
379
|
process.stdin.pause();
|
|
380
380
|
console.log();
|
|
381
|
-
|
|
381
|
+
resolve4();
|
|
382
382
|
};
|
|
383
383
|
if (process.stdin.isTTY) {
|
|
384
384
|
process.stdin.setRawMode?.(true);
|
|
@@ -388,7 +388,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
388
388
|
});
|
|
389
389
|
}
|
|
390
390
|
function sleep(ms) {
|
|
391
|
-
return new Promise((
|
|
391
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
392
392
|
}
|
|
393
393
|
|
|
394
394
|
// src/commands/login.ts
|
|
@@ -466,19 +466,19 @@ async function tokenLogin() {
|
|
|
466
466
|
);
|
|
467
467
|
blank();
|
|
468
468
|
process.stdout.write("Paste token: ");
|
|
469
|
-
const token = await new Promise((
|
|
469
|
+
const token = await new Promise((resolve4) => {
|
|
470
470
|
let data = "";
|
|
471
471
|
process.stdin.setEncoding("utf8");
|
|
472
472
|
process.stdin.on("data", (chunk) => {
|
|
473
473
|
data += chunk;
|
|
474
474
|
});
|
|
475
475
|
process.stdin.on("end", () => {
|
|
476
|
-
|
|
476
|
+
resolve4(data.trim());
|
|
477
477
|
});
|
|
478
478
|
if (process.stdin.isTTY) {
|
|
479
479
|
process.stdin.once("data", (chunk) => {
|
|
480
480
|
process.stdin.pause();
|
|
481
|
-
|
|
481
|
+
resolve4(chunk.toString().trim());
|
|
482
482
|
});
|
|
483
483
|
process.stdin.resume();
|
|
484
484
|
}
|
|
@@ -722,6 +722,14 @@ function toReportedWindow(window) {
|
|
|
722
722
|
if (!window) return null;
|
|
723
723
|
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
724
724
|
}
|
|
725
|
+
function toReportedOwner(snapshot) {
|
|
726
|
+
if (!snapshot.owner) return null;
|
|
727
|
+
return {
|
|
728
|
+
email: snapshot.owner.email,
|
|
729
|
+
organization_name: snapshot.owner.organizationName,
|
|
730
|
+
rate_limit_tier: snapshot.owner.rateLimitTier
|
|
731
|
+
};
|
|
732
|
+
}
|
|
725
733
|
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
726
734
|
try {
|
|
727
735
|
const apiUrl = getApiUrlConfig();
|
|
@@ -730,7 +738,8 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
730
738
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
731
739
|
body: JSON.stringify({
|
|
732
740
|
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
733
|
-
seven_day: toReportedWindow(snapshot.sevenDay)
|
|
741
|
+
seven_day: toReportedWindow(snapshot.sevenDay),
|
|
742
|
+
owner: toReportedOwner(snapshot)
|
|
734
743
|
}),
|
|
735
744
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
736
745
|
});
|
|
@@ -754,6 +763,13 @@ function toReportedOpenAiWindow(window) {
|
|
|
754
763
|
resets_at: window.resetsAt
|
|
755
764
|
};
|
|
756
765
|
}
|
|
766
|
+
function toReportedOpenAiSubscription(snapshot) {
|
|
767
|
+
if (!snapshot.subscription) return null;
|
|
768
|
+
return {
|
|
769
|
+
owner_email: snapshot.subscription.ownerEmail,
|
|
770
|
+
plan_type: snapshot.subscription.planType
|
|
771
|
+
};
|
|
772
|
+
}
|
|
757
773
|
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
758
774
|
try {
|
|
759
775
|
const apiUrl = getApiUrlConfig();
|
|
@@ -764,7 +780,8 @@ async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
|
764
780
|
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
765
781
|
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
766
782
|
has_credits: snapshot.hasCredits,
|
|
767
|
-
credits_unlimited: snapshot.creditsUnlimited
|
|
783
|
+
credits_unlimited: snapshot.creditsUnlimited,
|
|
784
|
+
subscription: toReportedOpenAiSubscription(snapshot)
|
|
768
785
|
}),
|
|
769
786
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
770
787
|
});
|
|
@@ -788,6 +805,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
788
805
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
789
806
|
body: JSON.stringify({
|
|
790
807
|
cpu_percent: usage.cpuPercent,
|
|
808
|
+
cpu_peak_percent: usage.cpuPeakPercent,
|
|
791
809
|
cpu_count: usage.cpuCount,
|
|
792
810
|
memory_total_bytes: usage.memoryTotalBytes,
|
|
793
811
|
memory_available_bytes: usage.memoryAvailableBytes,
|
|
@@ -996,7 +1014,10 @@ import { readFileSync } from "fs";
|
|
|
996
1014
|
import { homedir } from "os";
|
|
997
1015
|
import { join } from "path";
|
|
998
1016
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
1017
|
+
var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
|
|
1018
|
+
var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
|
|
999
1019
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
1020
|
+
var cachedOwner = null;
|
|
1000
1021
|
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
1001
1022
|
function parseClaudeCliCredentials(raw) {
|
|
1002
1023
|
let parsed;
|
|
@@ -1070,6 +1091,47 @@ function toWindow(value) {
|
|
|
1070
1091
|
}
|
|
1071
1092
|
return { utilization: window.utilization, resetsAt };
|
|
1072
1093
|
}
|
|
1094
|
+
function ownerLookupFailure(error2) {
|
|
1095
|
+
const name = error2?.name;
|
|
1096
|
+
return name === "TimeoutError" || name === "AbortError" ? "timed out" : "request failed";
|
|
1097
|
+
}
|
|
1098
|
+
async function getClaudeUsageOwner(accessToken) {
|
|
1099
|
+
if (cachedOwner?.accessToken === accessToken) {
|
|
1100
|
+
return { owner: cachedOwner.owner, ownerLookupError: null };
|
|
1101
|
+
}
|
|
1102
|
+
try {
|
|
1103
|
+
const response = await fetch(CLAUDE_PROFILE_URL, {
|
|
1104
|
+
headers: {
|
|
1105
|
+
Authorization: `Bearer ${accessToken}`,
|
|
1106
|
+
"Content-Type": "application/json",
|
|
1107
|
+
"anthropic-version": "2023-06-01"
|
|
1108
|
+
},
|
|
1109
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1110
|
+
});
|
|
1111
|
+
if (!response.ok) {
|
|
1112
|
+
return { owner: null, ownerLookupError: `HTTP ${response.status}` };
|
|
1113
|
+
}
|
|
1114
|
+
let body;
|
|
1115
|
+
try {
|
|
1116
|
+
body = await response.json();
|
|
1117
|
+
} catch (error2) {
|
|
1118
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1119
|
+
}
|
|
1120
|
+
const profile = body;
|
|
1121
|
+
if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
|
|
1122
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1123
|
+
}
|
|
1124
|
+
const owner = {
|
|
1125
|
+
email: profile.account.email,
|
|
1126
|
+
organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
|
|
1127
|
+
rateLimitTier: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
|
|
1128
|
+
};
|
|
1129
|
+
cachedOwner = { accessToken, owner };
|
|
1130
|
+
return { owner, ownerLookupError: null };
|
|
1131
|
+
} catch (error2) {
|
|
1132
|
+
return { owner: null, ownerLookupError: ownerLookupFailure(error2) };
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1073
1135
|
async function getClaudeUsage() {
|
|
1074
1136
|
const credentials2 = readClaudeCliCredentials();
|
|
1075
1137
|
if (!credentials2) {
|
|
@@ -1089,15 +1151,19 @@ async function getClaudeUsage() {
|
|
|
1089
1151
|
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
1090
1152
|
"Content-Type": "application/json",
|
|
1091
1153
|
"anthropic-version": "2023-06-01"
|
|
1092
|
-
}
|
|
1154
|
+
},
|
|
1155
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1093
1156
|
});
|
|
1094
1157
|
if (!res.ok) {
|
|
1095
1158
|
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
1096
1159
|
}
|
|
1097
1160
|
const body = await res.json();
|
|
1161
|
+
const { owner, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
|
|
1098
1162
|
return {
|
|
1099
1163
|
fiveHour: toWindow(body.five_hour),
|
|
1100
|
-
sevenDay: toWindow(body.seven_day)
|
|
1164
|
+
sevenDay: toWindow(body.seven_day),
|
|
1165
|
+
owner,
|
|
1166
|
+
ownerLookupError
|
|
1101
1167
|
};
|
|
1102
1168
|
}
|
|
1103
1169
|
|
|
@@ -1126,8 +1192,9 @@ async function claudeUsage() {
|
|
|
1126
1192
|
}
|
|
1127
1193
|
|
|
1128
1194
|
// src/commands/run.ts
|
|
1129
|
-
import {
|
|
1130
|
-
import {
|
|
1195
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "fs";
|
|
1196
|
+
import { homedir as homedir5 } from "os";
|
|
1197
|
+
import { isAbsolute as isAbsolute3, join as join9, parse, resolve as resolvePath2 } from "path";
|
|
1131
1198
|
import chalk6 from "chalk";
|
|
1132
1199
|
|
|
1133
1200
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1149,6 +1216,7 @@ var TelemetryEventTypes = {
|
|
|
1149
1216
|
// ../../packages/types/src/tunnel/index.ts
|
|
1150
1217
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1151
1218
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1219
|
+
var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
|
|
1152
1220
|
|
|
1153
1221
|
// ../../packages/types/src/runner-files.ts
|
|
1154
1222
|
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
@@ -1360,6 +1428,8 @@ var SEVERITY_BY_LEVEL = {
|
|
|
1360
1428
|
error: "error"
|
|
1361
1429
|
};
|
|
1362
1430
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1431
|
+
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1432
|
+
var MAX_METADATA_ENTRIES = 20;
|
|
1363
1433
|
var TRUNCATION_MARKER = "\u2026";
|
|
1364
1434
|
function redact(message) {
|
|
1365
1435
|
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
@@ -1368,6 +1438,24 @@ function truncate(message) {
|
|
|
1368
1438
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1369
1439
|
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
1370
1440
|
}
|
|
1441
|
+
function sanitiseMetadata(metadata) {
|
|
1442
|
+
if (!metadata) return {};
|
|
1443
|
+
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1444
|
+
if (Object.keys(metadata).length > entries.length) {
|
|
1445
|
+
console.error(
|
|
1446
|
+
`[runner-activity-telemetry] dropped ${Object.keys(metadata).length - entries.length} metadata entries`
|
|
1447
|
+
);
|
|
1448
|
+
}
|
|
1449
|
+
const sanitised = [];
|
|
1450
|
+
for (const [key, value] of entries) {
|
|
1451
|
+
if (typeof value === "string") {
|
|
1452
|
+
sanitised.push([key, truncate(redact(value).slice(0, MAX_METADATA_VALUE_LENGTH))]);
|
|
1453
|
+
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1454
|
+
sanitised.push([key, value]);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
return Object.fromEntries(sanitised);
|
|
1458
|
+
}
|
|
1371
1459
|
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1372
1460
|
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1373
1461
|
var windowStartedAt = 0;
|
|
@@ -1406,7 +1494,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1406
1494
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1407
1495
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1408
1496
|
message,
|
|
1409
|
-
metadata: { source: "cli.run" },
|
|
1497
|
+
metadata: { ...sanitiseMetadata(entry.metadata), source: "cli.run" },
|
|
1410
1498
|
agentId: context.agentId
|
|
1411
1499
|
});
|
|
1412
1500
|
} catch (err) {
|
|
@@ -1416,6 +1504,191 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1416
1504
|
}
|
|
1417
1505
|
}
|
|
1418
1506
|
|
|
1507
|
+
// src/lib/opencode/session-db-recovery-report.ts
|
|
1508
|
+
import { readFileSync as readFileSync2, unlinkSync } from "fs";
|
|
1509
|
+
import { join as join2 } from "path";
|
|
1510
|
+
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1511
|
+
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1512
|
+
return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1513
|
+
}
|
|
1514
|
+
function drainSessionDbRecoveryReport({
|
|
1515
|
+
homeDir,
|
|
1516
|
+
env
|
|
1517
|
+
}) {
|
|
1518
|
+
const path = sessionDbRecoveryReportPath(homeDir, env);
|
|
1519
|
+
let content;
|
|
1520
|
+
try {
|
|
1521
|
+
content = readFileSync2(path, "utf8");
|
|
1522
|
+
} catch (error2) {
|
|
1523
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")
|
|
1524
|
+
return { path, records: [], skippedLines: 0, readError: null };
|
|
1525
|
+
const readError = error2 instanceof Error ? error2.message : String(error2);
|
|
1526
|
+
console.error(`[session-db-recovery-report] could not read ${path}: ${readError}`);
|
|
1527
|
+
return { path, records: [], skippedLines: 0, readError };
|
|
1528
|
+
}
|
|
1529
|
+
let skippedLines = 0;
|
|
1530
|
+
const records = content.split("\n").flatMap((line) => {
|
|
1531
|
+
if (!line.trim()) return [];
|
|
1532
|
+
try {
|
|
1533
|
+
const value = JSON.parse(line);
|
|
1534
|
+
if (!isSessionDbRecoveryRecord(value)) {
|
|
1535
|
+
skippedLines++;
|
|
1536
|
+
return [];
|
|
1537
|
+
}
|
|
1538
|
+
return [
|
|
1539
|
+
{
|
|
1540
|
+
...value,
|
|
1541
|
+
provenance_reason: value.provenance_reason ?? null,
|
|
1542
|
+
provenance_migration_delta: value.provenance_migration_delta ?? null,
|
|
1543
|
+
replication_suspended: value.replication_suspended ?? false
|
|
1544
|
+
}
|
|
1545
|
+
];
|
|
1546
|
+
} catch (error2) {
|
|
1547
|
+
skippedLines++;
|
|
1548
|
+
console.error(
|
|
1549
|
+
`[session-db-recovery-report] skipped malformed record in ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1550
|
+
);
|
|
1551
|
+
return [];
|
|
1552
|
+
}
|
|
1553
|
+
});
|
|
1554
|
+
return { path, records, skippedLines, readError: null };
|
|
1555
|
+
}
|
|
1556
|
+
function acknowledgeSessionDbRecoveryReport(path) {
|
|
1557
|
+
try {
|
|
1558
|
+
unlinkSync(path);
|
|
1559
|
+
} catch (error2) {
|
|
1560
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1561
|
+
console.error(
|
|
1562
|
+
`[session-db-recovery-report] could not remove ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1563
|
+
);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
function buildSessionDbRecoveryActivity(record) {
|
|
1567
|
+
const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
|
|
1568
|
+
if (!level) return null;
|
|
1569
|
+
const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
|
|
1570
|
+
const replication = record.replication_suspended ? " This start is not backing up its new session history; restart after fixing the cause." : "";
|
|
1571
|
+
const giveupMessage = (() => {
|
|
1572
|
+
switch (record.reason) {
|
|
1573
|
+
case "restore_deadline_exceeded":
|
|
1574
|
+
return "The restore did not finish before this startup deadline. Earlier sessions are unavailable for this start; collect the SESSION-DB-RESTORE-TRUNCATED boot log before retrying.";
|
|
1575
|
+
case "restore_tool_unusable":
|
|
1576
|
+
case "classification_unrecognised":
|
|
1577
|
+
return "The restore or classification tool produced an unsupported result. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1578
|
+
case "synchroniser_config_unevaluable":
|
|
1579
|
+
case "synchroniser_config_incomplete":
|
|
1580
|
+
return "The installed hook and synchroniser bundle could not agree on configuration. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1581
|
+
case "synchroniser_config_unresolved":
|
|
1582
|
+
return "The synchroniser configuration could not be resolved. Inspect the preceding boot error and repair the identified runner image, bundle, or local environment before restarting.";
|
|
1583
|
+
case "litestream_config_unavailable":
|
|
1584
|
+
return "The local Litestream configuration could not be generated or written. Inspect the preceding boot error and repair the runner image, bundle, local environment, or write permissions before restarting.";
|
|
1585
|
+
case "classification_fatal":
|
|
1586
|
+
return "Session history could not be restored because backup setup could not be established. Inspect the named boot marker or log and repair the identified backup configuration or permissions.";
|
|
1587
|
+
default:
|
|
1588
|
+
return null;
|
|
1589
|
+
}
|
|
1590
|
+
})();
|
|
1591
|
+
if (giveupMessage)
|
|
1592
|
+
return {
|
|
1593
|
+
level,
|
|
1594
|
+
metadata: withoutContractFields(record),
|
|
1595
|
+
message: `${giveupMessage}${replication}`
|
|
1596
|
+
};
|
|
1597
|
+
switch (record.outcome) {
|
|
1598
|
+
case "fresh_session_db":
|
|
1599
|
+
return {
|
|
1600
|
+
level,
|
|
1601
|
+
metadata: withoutContractFields(record),
|
|
1602
|
+
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.${replication}`
|
|
1603
|
+
};
|
|
1604
|
+
case "restore_retried":
|
|
1605
|
+
return {
|
|
1606
|
+
level,
|
|
1607
|
+
metadata: withoutContractFields(record),
|
|
1608
|
+
message: "The initial session database restore failed; the runner will retry it. Monitor the runner backup for another restore failure."
|
|
1609
|
+
};
|
|
1610
|
+
case "replica_recovered":
|
|
1611
|
+
if (record.reason === "quarantine")
|
|
1612
|
+
return {
|
|
1613
|
+
level,
|
|
1614
|
+
metadata: withoutContractFields(record),
|
|
1615
|
+
message: `Session database recovery quarantined ${record.quarantined_objects ?? "unknown"} objects (${record.quarantined_bytes ?? "unknown"} bytes); ${record.quarantine_failed_objects ?? "unknown"} moves failed. Review the preserved backup at ${record.quarantine_destination ?? "an unknown destination"} before deleting it.`
|
|
1616
|
+
};
|
|
1617
|
+
if (record.reason === "prune")
|
|
1618
|
+
return {
|
|
1619
|
+
level,
|
|
1620
|
+
metadata: withoutContractFields(record),
|
|
1621
|
+
message: "Session database recovery discarded a damaged newest backup and retried. Sessions recorded after the previous backup point may be unavailable. Review the runner backup for another restore failure."
|
|
1622
|
+
};
|
|
1623
|
+
if (record.reason === "clear")
|
|
1624
|
+
return {
|
|
1625
|
+
level,
|
|
1626
|
+
metadata: withoutContractFields(record),
|
|
1627
|
+
message: "Session database recovery deleted the damaged backup and prior session history is unavailable. Review the runner backup configuration before relying on restored session history."
|
|
1628
|
+
};
|
|
1629
|
+
return null;
|
|
1630
|
+
case "history_rolled_back":
|
|
1631
|
+
return {
|
|
1632
|
+
level,
|
|
1633
|
+
metadata: withoutContractFields(record),
|
|
1634
|
+
message: `Session history was rolled back to verified restore point ${record.verified_restore_point ?? "unknown"}; everything after it is unavailable. Review the runner backup for another restore failure.`
|
|
1635
|
+
};
|
|
1636
|
+
case "restore_misconfigured":
|
|
1637
|
+
return {
|
|
1638
|
+
level,
|
|
1639
|
+
metadata: withoutContractFields(record),
|
|
1640
|
+
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1641
|
+
};
|
|
1642
|
+
case "session_db_boot_refused":
|
|
1643
|
+
return {
|
|
1644
|
+
level,
|
|
1645
|
+
metadata: withoutContractFields(record),
|
|
1646
|
+
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1647
|
+
};
|
|
1648
|
+
case "schema_provenance_mismatch":
|
|
1649
|
+
return {
|
|
1650
|
+
level,
|
|
1651
|
+
metadata: withoutContractFields(record),
|
|
1652
|
+
message: `Session database schema provenance mismatch for ${record.dbPath ?? record.db_path ?? "unknown"}: recorded version=${record.recorded_version ?? "unknown"}, current version=${record.current_version ?? "unknown"}, reason=${record.provenance_reason ?? "unknown"}, migration delta=${record.provenance_migration_delta ?? "unknown"}. Inspect the session database and runner backup before continuing.`
|
|
1653
|
+
};
|
|
1654
|
+
default:
|
|
1655
|
+
return null;
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
function withoutContractFields(record) {
|
|
1659
|
+
const { v: _v, event: _event, ...metadata } = record;
|
|
1660
|
+
return metadata;
|
|
1661
|
+
}
|
|
1662
|
+
var OUTCOMES = /* @__PURE__ */ new Set([
|
|
1663
|
+
"replica_recovered",
|
|
1664
|
+
"restore_retried",
|
|
1665
|
+
"fresh_session_db",
|
|
1666
|
+
"history_rolled_back",
|
|
1667
|
+
"restore_misconfigured",
|
|
1668
|
+
"session_db_boot_refused",
|
|
1669
|
+
"schema_provenance_mismatch"
|
|
1670
|
+
]);
|
|
1671
|
+
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1672
|
+
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
1673
|
+
var NUMBER_FIELDS = [
|
|
1674
|
+
"litestream_exit_code",
|
|
1675
|
+
"attempt",
|
|
1676
|
+
"replica_objects",
|
|
1677
|
+
"replica_bytes",
|
|
1678
|
+
"quarantined_objects",
|
|
1679
|
+
"quarantine_failed_objects",
|
|
1680
|
+
"quarantined_bytes",
|
|
1681
|
+
"restore_points_tried"
|
|
1682
|
+
];
|
|
1683
|
+
var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"];
|
|
1684
|
+
function isSessionDbRecoveryRecord(value) {
|
|
1685
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1686
|
+
const record = value;
|
|
1687
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && (record.provenance_reason === void 0 || record.provenance_reason === null || typeof record.provenance_reason === "string") && (record.provenance_migration_delta === void 0 || record.provenance_migration_delta === null || Number.isInteger(record.provenance_migration_delta)) && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1688
|
+
(field) => record[field] === null || typeof record[field] === "string"
|
|
1689
|
+
);
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1419
1692
|
// src/lib/opencode/health.ts
|
|
1420
1693
|
async function checkOpenCodeHealth(port) {
|
|
1421
1694
|
try {
|
|
@@ -1440,145 +1713,857 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1440
1713
|
if (health.healthy) {
|
|
1441
1714
|
return health;
|
|
1442
1715
|
}
|
|
1443
|
-
await new Promise((
|
|
1716
|
+
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
1444
1717
|
}
|
|
1445
1718
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1446
1719
|
}
|
|
1447
1720
|
|
|
1448
|
-
// src/lib/opencode/
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
}
|
|
1454
|
-
function buildOpenCodeVersionWarning(version2) {
|
|
1455
|
-
if (isQueueValidatedVersion(version2)) return null;
|
|
1456
|
-
const detected = version2 ? `v${version2}` : "unknown";
|
|
1457
|
-
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
1458
|
-
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
1459
|
-
}
|
|
1721
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1722
|
+
import { spawn as spawn2 } from "child_process";
|
|
1723
|
+
import { mkdirSync, statSync as statSync2, unlinkSync as unlinkSync2, writeFileSync } from "fs";
|
|
1724
|
+
import { homedir as homedir2 } from "os";
|
|
1725
|
+
import { dirname as dirname2, resolve as resolvePath } from "path";
|
|
1460
1726
|
|
|
1461
|
-
// src/lib/
|
|
1462
|
-
import {
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1727
|
+
// src/lib/runner-synchroniser.ts
|
|
1728
|
+
import { spawn } from "child_process";
|
|
1729
|
+
function appendError(stderr, error2) {
|
|
1730
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1731
|
+
return stderr === "" ? message : `${stderr}
|
|
1732
|
+
${message}`;
|
|
1733
|
+
}
|
|
1734
|
+
function runSynchroniser(args, opts) {
|
|
1735
|
+
return new Promise((resolve4) => {
|
|
1736
|
+
let child;
|
|
1737
|
+
let stdout = "";
|
|
1738
|
+
let stderr = "";
|
|
1739
|
+
let settled = false;
|
|
1740
|
+
const timer = {};
|
|
1741
|
+
let abortListener;
|
|
1742
|
+
let spawnListener;
|
|
1743
|
+
const finish = (result) => {
|
|
1744
|
+
if (settled) return;
|
|
1745
|
+
settled = true;
|
|
1746
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
1747
|
+
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
1748
|
+
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
1749
|
+
resolve4(result);
|
|
1750
|
+
};
|
|
1751
|
+
try {
|
|
1752
|
+
child = spawn("runner-synchroniser", args, {
|
|
1753
|
+
env: opts.env ?? process.env,
|
|
1754
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1755
|
+
});
|
|
1756
|
+
} catch (error2) {
|
|
1757
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
child.stdout?.setEncoding("utf8");
|
|
1761
|
+
child.stdout?.on("data", (chunk) => {
|
|
1762
|
+
stdout += chunk;
|
|
1763
|
+
});
|
|
1764
|
+
child.stderr?.setEncoding("utf8");
|
|
1765
|
+
child.stderr?.on("data", (chunk) => {
|
|
1766
|
+
stderr += chunk;
|
|
1767
|
+
});
|
|
1768
|
+
child.once("error", (error2) => {
|
|
1769
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1770
|
+
});
|
|
1771
|
+
child.once("close", (code) => {
|
|
1772
|
+
finish({ code, stdout, stderr, timedOut: false });
|
|
1773
|
+
});
|
|
1774
|
+
if (opts.signal) {
|
|
1775
|
+
const killChild = () => {
|
|
1776
|
+
if (child.pid === void 0) {
|
|
1777
|
+
if (!spawnListener) {
|
|
1778
|
+
spawnListener = killChild;
|
|
1779
|
+
child.once("spawn", spawnListener);
|
|
1780
|
+
}
|
|
1781
|
+
return;
|
|
1476
1782
|
}
|
|
1783
|
+
child.kill("SIGKILL");
|
|
1784
|
+
};
|
|
1785
|
+
abortListener = killChild;
|
|
1786
|
+
if (opts.signal.aborted) {
|
|
1787
|
+
abortListener();
|
|
1788
|
+
} else {
|
|
1789
|
+
opts.signal.addEventListener("abort", abortListener, { once: true });
|
|
1790
|
+
if (opts.signal.aborted) abortListener();
|
|
1477
1791
|
}
|
|
1478
|
-
} else if (platform === "linux") {
|
|
1479
|
-
const output = execSync(`readlink /proc/${pid}/cwd 2>/dev/null`, {
|
|
1480
|
-
encoding: "utf-8",
|
|
1481
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1482
|
-
}).trim();
|
|
1483
|
-
if (output) return output;
|
|
1484
1792
|
}
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1793
|
+
timer.handle = setTimeout(
|
|
1794
|
+
() => {
|
|
1795
|
+
child.kill("SIGKILL");
|
|
1796
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1797
|
+
},
|
|
1798
|
+
Math.max(0, opts.timeoutMs)
|
|
1799
|
+
);
|
|
1800
|
+
});
|
|
1488
1801
|
}
|
|
1489
|
-
|
|
1490
|
-
|
|
1802
|
+
|
|
1803
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1804
|
+
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
1805
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
|
|
1806
|
+
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
1807
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
|
|
1808
|
+
function commandError(result) {
|
|
1809
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
1810
|
+
}
|
|
1811
|
+
function reportRecord(stage, outcome, reason, litestreamExitCode, options) {
|
|
1812
|
+
options.reportRecovery({
|
|
1813
|
+
v: 1,
|
|
1814
|
+
event: "session_db_recovery",
|
|
1815
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1816
|
+
stage,
|
|
1817
|
+
outcome,
|
|
1818
|
+
severity: "error",
|
|
1819
|
+
reason,
|
|
1820
|
+
litestream_exit_code: litestreamExitCode,
|
|
1821
|
+
attempt: null,
|
|
1822
|
+
replica_objects: null,
|
|
1823
|
+
replica_bytes: null,
|
|
1824
|
+
quarantine_destination: null,
|
|
1825
|
+
quarantined_objects: null,
|
|
1826
|
+
quarantine_failed_objects: null,
|
|
1827
|
+
quarantined_bytes: null,
|
|
1828
|
+
verified_restore_point: null,
|
|
1829
|
+
restore_points_tried: null,
|
|
1830
|
+
provenance_reason: null,
|
|
1831
|
+
provenance_migration_delta: null,
|
|
1832
|
+
replication_suspended: stage === "restore"
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
function clearMarker(options) {
|
|
1836
|
+
if (!options.noReplicateMarker) return;
|
|
1491
1837
|
try {
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
})
|
|
1497
|
-
|
|
1838
|
+
unlinkSync2(options.noReplicateMarker);
|
|
1839
|
+
} catch (error2) {
|
|
1840
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1841
|
+
options.log(
|
|
1842
|
+
`Could not clear the previous session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1843
|
+
"warn"
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
function markNoReplicate(options, message) {
|
|
1848
|
+
if (options.noReplicateMarker) {
|
|
1849
|
+
try {
|
|
1850
|
+
mkdirSync(dirname2(options.noReplicateMarker), { recursive: true });
|
|
1851
|
+
writeFileSync(options.noReplicateMarker, "");
|
|
1852
|
+
} catch (error2) {
|
|
1853
|
+
options.log(
|
|
1854
|
+
`Could not write session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1855
|
+
"error"
|
|
1856
|
+
);
|
|
1498
1857
|
}
|
|
1499
|
-
} catch {
|
|
1500
1858
|
}
|
|
1501
|
-
|
|
1859
|
+
options.log(`SESSION-DB-NO-REPLICATE: ${message}`, "warn");
|
|
1502
1860
|
}
|
|
1503
|
-
function
|
|
1504
|
-
for (
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1861
|
+
function discardSessionDbDebris(options) {
|
|
1862
|
+
for (const path of [options.dbPath, `${options.dbPath}-wal`, `${options.dbPath}-shm`]) {
|
|
1863
|
+
try {
|
|
1864
|
+
unlinkSync2(path);
|
|
1865
|
+
} catch (error2) {
|
|
1866
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") continue;
|
|
1867
|
+
options.log(
|
|
1868
|
+
`Could not remove session-DB debris ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1869
|
+
"warn"
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
function splitDiagnostics(text) {
|
|
1875
|
+
return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
1876
|
+
}
|
|
1877
|
+
function logSynchroniserDiagnostics(result, options) {
|
|
1878
|
+
for (const line of splitDiagnostics(result.stderr)) options.log(line, "warn");
|
|
1879
|
+
}
|
|
1880
|
+
function parseSingleQuotedAssignment(line) {
|
|
1881
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
1882
|
+
if (!match || !match[2].startsWith("'")) return null;
|
|
1883
|
+
const valueSource = match[2];
|
|
1884
|
+
let value = "";
|
|
1885
|
+
for (let index = 1; index < valueSource.length; index++) {
|
|
1886
|
+
const character = valueSource[index];
|
|
1887
|
+
if (character !== "'") {
|
|
1888
|
+
value += character;
|
|
1889
|
+
continue;
|
|
1508
1890
|
}
|
|
1891
|
+
if (index === valueSource.length - 1) return [match[1], value];
|
|
1892
|
+
if (valueSource.slice(index + 1, index + 4) !== "\\''") return null;
|
|
1893
|
+
value += "'";
|
|
1894
|
+
index += 3;
|
|
1509
1895
|
}
|
|
1510
1896
|
return null;
|
|
1511
1897
|
}
|
|
1512
|
-
function
|
|
1513
|
-
const
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
for (const pid of pids) {
|
|
1548
|
-
try {
|
|
1549
|
-
const lsofOutput = execSync(`lsof -Pan -p ${pid} -i TCP -sTCP:LISTEN 2>/dev/null`, {
|
|
1550
|
-
encoding: "utf-8",
|
|
1551
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1552
|
-
}).trim();
|
|
1553
|
-
for (const line of lsofOutput.split("\n")) {
|
|
1554
|
-
const portMatch = line.match(/:(\d+)\s+\(LISTEN\)/);
|
|
1555
|
-
if (portMatch) {
|
|
1556
|
-
const port = parseInt(portMatch[1], 10);
|
|
1557
|
-
if (!isNaN(port) && !instances.some((i) => i.port === port)) {
|
|
1558
|
-
const cwd = getProcessCwd(pid);
|
|
1559
|
-
instances.push({ pid, port, cwd });
|
|
1560
|
-
}
|
|
1561
|
-
}
|
|
1562
|
-
}
|
|
1563
|
-
} catch {
|
|
1564
|
-
}
|
|
1565
|
-
}
|
|
1898
|
+
function parseSynchroniserEnv(stdout) {
|
|
1899
|
+
const values = {};
|
|
1900
|
+
for (const line of stdout.split("\n")) {
|
|
1901
|
+
if (line.trim() === "") continue;
|
|
1902
|
+
const assignment = parseSingleQuotedAssignment(line);
|
|
1903
|
+
if (!assignment) return null;
|
|
1904
|
+
values[assignment[0]] = assignment[1];
|
|
1905
|
+
}
|
|
1906
|
+
return values;
|
|
1907
|
+
}
|
|
1908
|
+
function runCommand(command, args, options) {
|
|
1909
|
+
return new Promise((resolve4) => {
|
|
1910
|
+
let child;
|
|
1911
|
+
let stdout = "";
|
|
1912
|
+
let stderr = "";
|
|
1913
|
+
let settled = false;
|
|
1914
|
+
const finish = (result) => {
|
|
1915
|
+
if (settled) return;
|
|
1916
|
+
settled = true;
|
|
1917
|
+
if (timer) clearTimeout(timer);
|
|
1918
|
+
resolve4(result);
|
|
1919
|
+
};
|
|
1920
|
+
try {
|
|
1921
|
+
child = spawn2(command, args, {
|
|
1922
|
+
env: options.env,
|
|
1923
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1924
|
+
});
|
|
1925
|
+
} catch (error2) {
|
|
1926
|
+
resolve4({
|
|
1927
|
+
code: null,
|
|
1928
|
+
stdout,
|
|
1929
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
1930
|
+
timedOut: false
|
|
1931
|
+
});
|
|
1932
|
+
return;
|
|
1566
1933
|
}
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1934
|
+
child.stdout?.setEncoding("utf8");
|
|
1935
|
+
child.stdout?.on("data", (chunk) => {
|
|
1936
|
+
stdout += chunk;
|
|
1937
|
+
});
|
|
1938
|
+
child.stderr?.setEncoding("utf8");
|
|
1939
|
+
child.stderr?.on("data", (chunk) => {
|
|
1940
|
+
stderr += chunk;
|
|
1941
|
+
});
|
|
1942
|
+
child.once("error", (error2) => {
|
|
1943
|
+
finish({
|
|
1944
|
+
code: null,
|
|
1945
|
+
stdout,
|
|
1946
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
1947
|
+
${error2.message}`,
|
|
1948
|
+
timedOut: false
|
|
1949
|
+
});
|
|
1950
|
+
});
|
|
1951
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
1952
|
+
const timer = setTimeout(
|
|
1953
|
+
() => {
|
|
1954
|
+
child.kill("SIGKILL");
|
|
1955
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1956
|
+
},
|
|
1957
|
+
Math.max(0, options.timeoutMs)
|
|
1570
1958
|
);
|
|
1571
|
-
}
|
|
1572
|
-
return instances;
|
|
1959
|
+
});
|
|
1573
1960
|
}
|
|
1574
|
-
async function
|
|
1575
|
-
const
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1961
|
+
async function ensureLitestreamConfig(options, env) {
|
|
1962
|
+
const configPath = options.litestreamConfig;
|
|
1963
|
+
if (!configPath) {
|
|
1964
|
+
markNoReplicate(options, "no Litestream configuration path was provided");
|
|
1965
|
+
reportRecord(
|
|
1966
|
+
"restore",
|
|
1967
|
+
"restore_misconfigured",
|
|
1968
|
+
"litestream_config_unavailable",
|
|
1969
|
+
null,
|
|
1970
|
+
options
|
|
1971
|
+
);
|
|
1972
|
+
return null;
|
|
1973
|
+
}
|
|
1974
|
+
try {
|
|
1975
|
+
if (statSync2(configPath).size > 0) return configPath;
|
|
1976
|
+
} catch (error2) {
|
|
1977
|
+
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
1978
|
+
options.log(
|
|
1979
|
+
`Could not inspect ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1980
|
+
"warn"
|
|
1981
|
+
);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
const rendered = await runSynchroniser(["litestream-config"], {
|
|
1985
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
1986
|
+
env
|
|
1987
|
+
});
|
|
1988
|
+
logSynchroniserDiagnostics(rendered, options);
|
|
1989
|
+
if (rendered.timedOut || rendered.code !== 0) {
|
|
1990
|
+
options.log(
|
|
1991
|
+
`Could not generate ${configPath}: runner-synchroniser litestream-config failed (${commandError(rendered)})`,
|
|
1992
|
+
"error"
|
|
1993
|
+
);
|
|
1994
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
1995
|
+
reportRecord(
|
|
1996
|
+
"restore",
|
|
1997
|
+
"restore_misconfigured",
|
|
1998
|
+
"litestream_config_unavailable",
|
|
1999
|
+
null,
|
|
2000
|
+
options
|
|
2001
|
+
);
|
|
2002
|
+
return null;
|
|
2003
|
+
}
|
|
2004
|
+
try {
|
|
2005
|
+
mkdirSync(dirname2(configPath), { recursive: true });
|
|
2006
|
+
writeFileSync(configPath, rendered.stdout);
|
|
2007
|
+
} catch (error2) {
|
|
2008
|
+
options.log(
|
|
2009
|
+
`Could not write ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2010
|
+
"error"
|
|
2011
|
+
);
|
|
2012
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2013
|
+
reportRecord(
|
|
2014
|
+
"restore",
|
|
2015
|
+
"restore_misconfigured",
|
|
2016
|
+
"litestream_config_unavailable",
|
|
2017
|
+
null,
|
|
2018
|
+
options
|
|
2019
|
+
);
|
|
2020
|
+
return null;
|
|
2021
|
+
}
|
|
2022
|
+
const version2 = await runCommand("litestream", ["version"], {
|
|
2023
|
+
env,
|
|
2024
|
+
timeoutMs: 1e4
|
|
2025
|
+
});
|
|
2026
|
+
const litestreamVersion = version2.code === 0 ? version2.stdout.trim() || "unknown" : "unknown";
|
|
2027
|
+
const regionEmpty = !/^\s*region:\s*\S+/m.test(rendered.stdout);
|
|
2028
|
+
options.log(
|
|
2029
|
+
`litestream ${litestreamVersion}; AWS_REGION=${env.AWS_REGION ?? "<unset>"} AWS_DEFAULT_REGION=${env.AWS_DEFAULT_REGION ?? "<unset>"}; rendered litestream.yml region empty: ${regionEmpty ? "yes" : "no"}`
|
|
2030
|
+
);
|
|
2031
|
+
return configPath;
|
|
2032
|
+
}
|
|
2033
|
+
function restoreGiveUp(options, reason, message, litestreamExitCode, outcome = "fresh_session_db") {
|
|
2034
|
+
discardSessionDbDebris(options);
|
|
2035
|
+
markNoReplicate(options, message);
|
|
2036
|
+
reportRecord("restore", outcome, reason, litestreamExitCode, options);
|
|
2037
|
+
}
|
|
2038
|
+
async function restoreSessionDb(options, configPath, env) {
|
|
2039
|
+
const restored = await runCommand(
|
|
2040
|
+
"litestream",
|
|
2041
|
+
["restore", "-config", configPath, "-if-db-not-exists", "-if-replica-exists", options.dbPath],
|
|
2042
|
+
{ env, timeoutMs: SESSION_DB_RESTORE_TIMEOUT_MS }
|
|
2043
|
+
);
|
|
2044
|
+
for (const line of splitDiagnostics(restored.stderr)) options.log(line, "warn");
|
|
2045
|
+
if (restored.timedOut || restored.code === 124 || restored.code === 137) {
|
|
2046
|
+
restoreGiveUp(
|
|
2047
|
+
options,
|
|
2048
|
+
"restore_deadline_exceeded",
|
|
2049
|
+
`SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within ${SESSION_DB_RESTORE_TIMEOUT_MS}ms; opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2050
|
+
restored.code ?? 124
|
|
2051
|
+
);
|
|
2052
|
+
return;
|
|
2053
|
+
}
|
|
2054
|
+
if (restored.code === null || restored.code === 125 || restored.code === 126 || restored.code === 127) {
|
|
2055
|
+
options.log(`litestream restore could not run (${commandError(restored)})`, "error");
|
|
2056
|
+
restoreGiveUp(
|
|
2057
|
+
options,
|
|
2058
|
+
"restore_tool_unusable",
|
|
2059
|
+
`restore tool is broken (${commandError(restored)}); opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2060
|
+
restored.code
|
|
2061
|
+
);
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
const classified = await runSynchroniser(
|
|
2065
|
+
[
|
|
2066
|
+
"session-db-classify",
|
|
2067
|
+
String(restored.code ?? 1),
|
|
2068
|
+
"1",
|
|
2069
|
+
"--on-unusable-replica=leave",
|
|
2070
|
+
"--fresh-db-fallback"
|
|
2071
|
+
],
|
|
2072
|
+
{ timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS, env }
|
|
2073
|
+
);
|
|
2074
|
+
logSynchroniserDiagnostics(classified, options);
|
|
2075
|
+
const classifyCode = classified.code;
|
|
2076
|
+
switch (classifyCode) {
|
|
2077
|
+
case 0:
|
|
2078
|
+
return;
|
|
2079
|
+
case 31:
|
|
2080
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2081
|
+
options.log(
|
|
2082
|
+
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
2083
|
+
"warn"
|
|
2084
|
+
);
|
|
2085
|
+
return;
|
|
2086
|
+
case 32:
|
|
2087
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2088
|
+
discardSessionDbDebris(options);
|
|
2089
|
+
markNoReplicate(
|
|
2090
|
+
options,
|
|
2091
|
+
"session-db-classify asked for another restore attempt (32), but this boot has budget for only one; treating it as a give-up rather than retrying"
|
|
2092
|
+
);
|
|
2093
|
+
return;
|
|
2094
|
+
case 30:
|
|
2095
|
+
restoreGiveUp(
|
|
2096
|
+
options,
|
|
2097
|
+
"classification_fatal",
|
|
2098
|
+
"session-db-classify returned fatal (30); see the FATAL message above",
|
|
2099
|
+
restored.code,
|
|
2100
|
+
"restore_misconfigured"
|
|
2101
|
+
);
|
|
2102
|
+
return;
|
|
2103
|
+
default:
|
|
2104
|
+
restoreGiveUp(
|
|
2105
|
+
options,
|
|
2106
|
+
"classification_unrecognised",
|
|
2107
|
+
`session-db-classify exited ${classifyCode ?? "null"}, which is none of its documented answers`,
|
|
2108
|
+
restored.code
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
async function verifySessionDb(options, configPath, env) {
|
|
2113
|
+
if (!options.noReplicateMarker || !fileExists(options.noReplicateMarker)) {
|
|
2114
|
+
const result = await runSynchroniser(["session-db-verify", configPath], {
|
|
2115
|
+
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
2116
|
+
env: {
|
|
2117
|
+
...env,
|
|
2118
|
+
// The synchroniser reads this value in SECONDS. Keep this at 120, not
|
|
2119
|
+
// 120_000, so the walkback gives up before the outer process bound.
|
|
2120
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
2121
|
+
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
2122
|
+
)
|
|
2123
|
+
}
|
|
2124
|
+
});
|
|
2125
|
+
logSynchroniserDiagnostics(result, options);
|
|
2126
|
+
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
2127
|
+
options.log(
|
|
2128
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
2129
|
+
"warn"
|
|
2130
|
+
);
|
|
2131
|
+
return false;
|
|
2132
|
+
}
|
|
2133
|
+
if (result.code === 34) {
|
|
2134
|
+
reportRecord(
|
|
2135
|
+
"verify",
|
|
2136
|
+
"session_db_boot_refused",
|
|
2137
|
+
result.stderr.includes("SESSION-DB-LOCAL-DISCARD-FAILED") ? "local_discard_failed" : "replica_separation_unproven",
|
|
2138
|
+
null,
|
|
2139
|
+
options
|
|
2140
|
+
);
|
|
2141
|
+
return true;
|
|
2142
|
+
}
|
|
2143
|
+
if (result.code === 33) {
|
|
2144
|
+
options.log(
|
|
2145
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
2146
|
+
"warn"
|
|
2147
|
+
);
|
|
2148
|
+
return false;
|
|
2149
|
+
}
|
|
2150
|
+
if (result.code !== 0) {
|
|
2151
|
+
options.log(
|
|
2152
|
+
`SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${result.code ?? "null"}, which is none of its documented answers; continuing with the restored opencode.db as-is`,
|
|
2153
|
+
"warn"
|
|
2154
|
+
);
|
|
2155
|
+
}
|
|
2156
|
+
return false;
|
|
2157
|
+
}
|
|
2158
|
+
options.log(
|
|
2159
|
+
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
2160
|
+
"debug"
|
|
2161
|
+
);
|
|
2162
|
+
return false;
|
|
2163
|
+
}
|
|
2164
|
+
function fileExists(path) {
|
|
2165
|
+
try {
|
|
2166
|
+
statSync2(path);
|
|
2167
|
+
return true;
|
|
2168
|
+
} catch (error2) {
|
|
2169
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
2170
|
+
return true;
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
async function restoreAndVerifySessionDb(options) {
|
|
2174
|
+
const env = options.env ?? process.env;
|
|
2175
|
+
clearMarker(options);
|
|
2176
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2177
|
+
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
2178
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2179
|
+
env
|
|
2180
|
+
});
|
|
2181
|
+
logSynchroniserDiagnostics(synchroniserEnv, options);
|
|
2182
|
+
if (synchroniserEnv.timedOut || synchroniserEnv.code !== 0) {
|
|
2183
|
+
options.log(
|
|
2184
|
+
`runner-synchroniser env could not resolve the session-DB configuration (${commandError(synchroniserEnv)})`,
|
|
2185
|
+
"error"
|
|
2186
|
+
);
|
|
2187
|
+
markNoReplicate(
|
|
2188
|
+
options,
|
|
2189
|
+
"could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
2190
|
+
);
|
|
2191
|
+
reportRecord(
|
|
2192
|
+
"restore",
|
|
2193
|
+
"restore_misconfigured",
|
|
2194
|
+
"synchroniser_config_unresolved",
|
|
2195
|
+
null,
|
|
2196
|
+
options
|
|
2197
|
+
);
|
|
2198
|
+
return { verifyFatal: false };
|
|
2199
|
+
}
|
|
2200
|
+
const values = parseSynchroniserEnv(synchroniserEnv.stdout);
|
|
2201
|
+
if (!values) {
|
|
2202
|
+
markNoReplicate(
|
|
2203
|
+
options,
|
|
2204
|
+
"the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
2205
|
+
);
|
|
2206
|
+
reportRecord(
|
|
2207
|
+
"restore",
|
|
2208
|
+
"restore_misconfigured",
|
|
2209
|
+
"synchroniser_config_unevaluable",
|
|
2210
|
+
null,
|
|
2211
|
+
options
|
|
2212
|
+
);
|
|
2213
|
+
return { verifyFatal: false };
|
|
2214
|
+
}
|
|
2215
|
+
const synchroniserDbPath = values.OPENCODE_DB_PATH;
|
|
2216
|
+
if (!synchroniserDbPath) {
|
|
2217
|
+
markNoReplicate(
|
|
2218
|
+
options,
|
|
2219
|
+
"run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
2220
|
+
);
|
|
2221
|
+
reportRecord(
|
|
2222
|
+
"restore",
|
|
2223
|
+
"restore_misconfigured",
|
|
2224
|
+
"synchroniser_config_incomplete",
|
|
2225
|
+
null,
|
|
2226
|
+
options
|
|
2227
|
+
);
|
|
2228
|
+
return { verifyFatal: false };
|
|
2229
|
+
}
|
|
2230
|
+
if (resolvePath(synchroniserDbPath) !== resolvePath(options.dbPath)) {
|
|
2231
|
+
options.log(
|
|
2232
|
+
`runner-synchroniser reported OPENCODE_DB_PATH=${synchroniserDbPath}, but OpenCode uses ${options.dbPath}; continuing with OpenCode's session-DB path`,
|
|
2233
|
+
"warn"
|
|
2234
|
+
);
|
|
2235
|
+
}
|
|
2236
|
+
if (!values.PERSISTENCE_BUCKET) {
|
|
2237
|
+
options.log(
|
|
2238
|
+
"SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated.",
|
|
2239
|
+
"warn"
|
|
2240
|
+
);
|
|
2241
|
+
return { verifyFatal: false };
|
|
2242
|
+
}
|
|
2243
|
+
const configPath = await ensureLitestreamConfig(options, env);
|
|
2244
|
+
if (!configPath) return { verifyFatal: false };
|
|
2245
|
+
await restoreSessionDb(options, configPath, env);
|
|
2246
|
+
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
2247
|
+
return { verifyFatal: false };
|
|
2248
|
+
}
|
|
2249
|
+
return { verifyFatal: await verifySessionDb(options, configPath, env) };
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// src/lib/opencode/session-db-provenance.ts
|
|
2253
|
+
import { createRequire } from "module";
|
|
2254
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
2255
|
+
import { dirname as dirname3, join as join3 } from "path";
|
|
2256
|
+
var require2 = createRequire(import.meta.url);
|
|
2257
|
+
function readSessionDbMigrationIds(dbPath) {
|
|
2258
|
+
let db;
|
|
2259
|
+
try {
|
|
2260
|
+
const { DatabaseSync } = require2("node:sqlite");
|
|
2261
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
2262
|
+
const columns = db.prepare("PRAGMA table_info(migration)").all();
|
|
2263
|
+
const hasExpectedShape = columns.length === 2 && columns.some(
|
|
2264
|
+
(column) => column.name === "id" && typeof column.type === "string" && column.type.toUpperCase() === "TEXT" && column.pk === 1
|
|
2265
|
+
) && columns.some(
|
|
2266
|
+
(column) => column.name === "time_completed" && typeof column.type === "string" && column.type.toUpperCase() === "INTEGER" && column.notnull === 1 && column.pk === 0
|
|
2267
|
+
);
|
|
2268
|
+
if (!hasExpectedShape) {
|
|
2269
|
+
console.warn(`[readSessionDbMigrationIds] unsupported migration table shape in ${dbPath}`);
|
|
2270
|
+
return null;
|
|
2271
|
+
}
|
|
2272
|
+
const rows = db.prepare("SELECT id FROM migration ORDER BY id").all();
|
|
2273
|
+
if (rows.some((row) => typeof row.id !== "string")) return null;
|
|
2274
|
+
return rows.map((row) => row.id);
|
|
2275
|
+
} catch (error2) {
|
|
2276
|
+
console.warn(
|
|
2277
|
+
`[readSessionDbMigrationIds] could not read migration history from ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2278
|
+
);
|
|
2279
|
+
return null;
|
|
2280
|
+
} finally {
|
|
2281
|
+
try {
|
|
2282
|
+
db?.close();
|
|
2283
|
+
} catch (error2) {
|
|
2284
|
+
console.warn(
|
|
2285
|
+
`[readSessionDbMigrationIds] could not close ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2286
|
+
);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
2291
|
+
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
2292
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
2293
|
+
}
|
|
2294
|
+
function loadSessionDbProvenanceState(path) {
|
|
2295
|
+
let value;
|
|
2296
|
+
try {
|
|
2297
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
2298
|
+
} catch (error2) {
|
|
2299
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return {};
|
|
2300
|
+
console.error(
|
|
2301
|
+
`[session-db-provenance] could not read ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2302
|
+
);
|
|
2303
|
+
return {};
|
|
2304
|
+
}
|
|
2305
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2306
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2307
|
+
return {};
|
|
2308
|
+
}
|
|
2309
|
+
const state = {};
|
|
2310
|
+
for (const [dbPath, record] of Object.entries(value)) {
|
|
2311
|
+
if (!isSessionDbProvenanceRecord(record)) {
|
|
2312
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2313
|
+
return {};
|
|
2314
|
+
}
|
|
2315
|
+
state[dbPath] = record;
|
|
2316
|
+
}
|
|
2317
|
+
return state;
|
|
2318
|
+
}
|
|
2319
|
+
function saveSessionDbProvenanceState(path, state) {
|
|
2320
|
+
try {
|
|
2321
|
+
mkdirSync2(dirname3(path), { recursive: true });
|
|
2322
|
+
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
2323
|
+
`, "utf8");
|
|
2324
|
+
} catch (error2) {
|
|
2325
|
+
console.error(
|
|
2326
|
+
`[session-db-provenance] could not write ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2327
|
+
);
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
function evaluateSessionDbProvenance(input) {
|
|
2331
|
+
const { currentVersion, currentIds, previous } = input;
|
|
2332
|
+
if (!previous) return { anomaly: false, reason: null };
|
|
2333
|
+
const current = new Set(currentIds);
|
|
2334
|
+
const prior = new Set(previous.migrationIds);
|
|
2335
|
+
for (const id of prior) {
|
|
2336
|
+
if (!current.has(id)) return { anomaly: true, reason: "migration-history-regressed" };
|
|
2337
|
+
}
|
|
2338
|
+
if (current.size > prior.size && previous.opencodeVersion === currentVersion) {
|
|
2339
|
+
return { anomaly: true, reason: "foreign-version-migrations" };
|
|
2340
|
+
}
|
|
2341
|
+
return { anomaly: false, reason: null };
|
|
2342
|
+
}
|
|
2343
|
+
function checkSessionDbProvenance(input) {
|
|
2344
|
+
const { dbPath, currentVersion, homeDir, env } = input;
|
|
2345
|
+
const path = sessionDbProvenanceStatePath(homeDir, env);
|
|
2346
|
+
const state = loadSessionDbProvenanceState(path);
|
|
2347
|
+
const previous = state[dbPath];
|
|
2348
|
+
const currentIds = readSessionDbMigrationIds(dbPath);
|
|
2349
|
+
if (currentIds === null) {
|
|
2350
|
+
return {
|
|
2351
|
+
anomaly: false,
|
|
2352
|
+
reason: null,
|
|
2353
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2354
|
+
migrationDelta: null
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
const decision = evaluateSessionDbProvenance({ currentVersion, currentIds, previous });
|
|
2358
|
+
const migrationDelta = previous ? new Set(currentIds).size - new Set(previous.migrationIds).size : null;
|
|
2359
|
+
state[dbPath] = {
|
|
2360
|
+
opencodeVersion: currentVersion,
|
|
2361
|
+
migrationIds: currentIds,
|
|
2362
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2363
|
+
};
|
|
2364
|
+
saveSessionDbProvenanceState(path, state);
|
|
2365
|
+
return {
|
|
2366
|
+
...decision,
|
|
2367
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2368
|
+
migrationDelta
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
function isSessionDbProvenanceRecord(value) {
|
|
2372
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2373
|
+
const record = value;
|
|
2374
|
+
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
// src/lib/opencode/opencode-version-gate.ts
|
|
2378
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
2379
|
+
function isQueueValidatedVersion(version2) {
|
|
2380
|
+
if (!version2) return false;
|
|
2381
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
2382
|
+
}
|
|
2383
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
2384
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
2385
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
2386
|
+
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
2387
|
+
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
// src/lib/opencode/process.ts
|
|
2391
|
+
import { execSync, spawn as spawn3 } from "child_process";
|
|
2392
|
+
|
|
2393
|
+
// src/lib/process-stop.ts
|
|
2394
|
+
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
2395
|
+
if (!child.pid) {
|
|
2396
|
+
return { outcome: "not-running", code: child.exitCode, signal: child.signalCode };
|
|
2397
|
+
}
|
|
2398
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
2399
|
+
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
2400
|
+
}
|
|
2401
|
+
return new Promise((resolve4, reject) => {
|
|
2402
|
+
let forced = false;
|
|
2403
|
+
let settled = false;
|
|
2404
|
+
const timer = setTimeout(() => {
|
|
2405
|
+
forced = true;
|
|
2406
|
+
try {
|
|
2407
|
+
sendKill();
|
|
2408
|
+
} catch (error2) {
|
|
2409
|
+
if (error2.code === "ESRCH") {
|
|
2410
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2411
|
+
} else {
|
|
2412
|
+
fail(error2);
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
}, timeoutMs);
|
|
2416
|
+
const finish = (result) => {
|
|
2417
|
+
if (settled) return;
|
|
2418
|
+
settled = true;
|
|
2419
|
+
clearTimeout(timer);
|
|
2420
|
+
child.removeListener("exit", onExit);
|
|
2421
|
+
resolve4(result);
|
|
2422
|
+
};
|
|
2423
|
+
const fail = (error2) => {
|
|
2424
|
+
if (settled) return;
|
|
2425
|
+
settled = true;
|
|
2426
|
+
clearTimeout(timer);
|
|
2427
|
+
child.removeListener("exit", onExit);
|
|
2428
|
+
reject(error2);
|
|
2429
|
+
};
|
|
2430
|
+
const onExit = (code, signal) => {
|
|
2431
|
+
finish({ outcome: forced ? "killed" : "exited", code, signal });
|
|
2432
|
+
};
|
|
2433
|
+
child.once("exit", onExit);
|
|
2434
|
+
try {
|
|
2435
|
+
sendTerm();
|
|
2436
|
+
} catch (error2) {
|
|
2437
|
+
if (error2.code === "ESRCH") {
|
|
2438
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2439
|
+
} else {
|
|
2440
|
+
fail(error2);
|
|
2441
|
+
}
|
|
2442
|
+
return;
|
|
2443
|
+
}
|
|
2444
|
+
});
|
|
2445
|
+
}
|
|
2446
|
+
|
|
2447
|
+
// src/lib/opencode/process.ts
|
|
2448
|
+
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
2449
|
+
function getProcessCwd(pid) {
|
|
2450
|
+
const platform = process.platform;
|
|
2451
|
+
try {
|
|
2452
|
+
if (platform === "darwin") {
|
|
2453
|
+
const output = execSync(`lsof -a -p ${pid} -d cwd -Fn 2>/dev/null`, {
|
|
2454
|
+
encoding: "utf-8",
|
|
2455
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2456
|
+
}).trim();
|
|
2457
|
+
const lines = output.split("\n");
|
|
2458
|
+
for (const line of lines) {
|
|
2459
|
+
if (line.startsWith("n") && !line.startsWith("n ")) {
|
|
2460
|
+
return line.slice(1);
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
} else if (platform === "linux") {
|
|
2464
|
+
const output = execSync(`readlink /proc/${pid}/cwd 2>/dev/null`, {
|
|
2465
|
+
encoding: "utf-8",
|
|
2466
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2467
|
+
}).trim();
|
|
2468
|
+
if (output) return output;
|
|
2469
|
+
}
|
|
2470
|
+
} catch {
|
|
2471
|
+
}
|
|
2472
|
+
return void 0;
|
|
2473
|
+
}
|
|
2474
|
+
function isPortInUse(port) {
|
|
2475
|
+
const platform = process.platform;
|
|
2476
|
+
try {
|
|
2477
|
+
if (platform === "darwin" || platform === "linux") {
|
|
2478
|
+
execSync(`lsof -i :${port} -sTCP:LISTEN 2>/dev/null`, {
|
|
2479
|
+
encoding: "utf-8",
|
|
2480
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2481
|
+
});
|
|
2482
|
+
return true;
|
|
2483
|
+
}
|
|
2484
|
+
} catch {
|
|
2485
|
+
}
|
|
2486
|
+
return false;
|
|
2487
|
+
}
|
|
2488
|
+
function findAvailablePort(startPort, maxAttempts = 10) {
|
|
2489
|
+
for (let i = 0; i < maxAttempts; i++) {
|
|
2490
|
+
const port = startPort + i;
|
|
2491
|
+
if (!isPortInUse(port)) {
|
|
2492
|
+
return port;
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
return null;
|
|
2496
|
+
}
|
|
2497
|
+
function findOpenCodeProcesses() {
|
|
2498
|
+
const instances = [];
|
|
2499
|
+
try {
|
|
2500
|
+
const platform = process.platform;
|
|
2501
|
+
if (platform === "darwin" || platform === "linux") {
|
|
2502
|
+
let pids = [];
|
|
2503
|
+
try {
|
|
2504
|
+
const pgrepOutput = execSync('pgrep -f "opencode serve|opencode-serve"', {
|
|
2505
|
+
encoding: "utf-8",
|
|
2506
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2507
|
+
}).trim();
|
|
2508
|
+
if (pgrepOutput) {
|
|
2509
|
+
pids = pgrepOutput.split("\n").map((p) => parseInt(p.trim(), 10)).filter((p) => !isNaN(p));
|
|
2510
|
+
}
|
|
2511
|
+
} catch {
|
|
2512
|
+
try {
|
|
2513
|
+
const psOutput = execSync('ps aux | grep -E "opencode (serve|--port)" | grep -v grep', {
|
|
2514
|
+
encoding: "utf-8",
|
|
2515
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2516
|
+
}).trim();
|
|
2517
|
+
if (psOutput) {
|
|
2518
|
+
for (const line of psOutput.split("\n")) {
|
|
2519
|
+
const parts = line.trim().split(/\s+/);
|
|
2520
|
+
if (parts.length >= 2) {
|
|
2521
|
+
const pid = parseInt(parts[1], 10);
|
|
2522
|
+
if (!isNaN(pid)) pids.push(pid);
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
} catch (err) {
|
|
2527
|
+
console.warn(
|
|
2528
|
+
`findOpenCodeProcesses: ps fallback failed: ${err instanceof Error ? err.message : String(err)}`
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
for (const pid of pids) {
|
|
2533
|
+
try {
|
|
2534
|
+
const lsofOutput = execSync(`lsof -Pan -p ${pid} -i TCP -sTCP:LISTEN 2>/dev/null`, {
|
|
2535
|
+
encoding: "utf-8",
|
|
2536
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2537
|
+
}).trim();
|
|
2538
|
+
for (const line of lsofOutput.split("\n")) {
|
|
2539
|
+
const portMatch = line.match(/:(\d+)\s+\(LISTEN\)/);
|
|
2540
|
+
if (portMatch) {
|
|
2541
|
+
const port = parseInt(portMatch[1], 10);
|
|
2542
|
+
if (!isNaN(port) && !instances.some((i) => i.port === port)) {
|
|
2543
|
+
const cwd = getProcessCwd(pid);
|
|
2544
|
+
instances.push({ pid, port, cwd });
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
} catch {
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2551
|
+
}
|
|
2552
|
+
} catch (err) {
|
|
2553
|
+
console.warn(
|
|
2554
|
+
`findOpenCodeProcesses: process detection failed: ${err instanceof Error ? err.message : String(err)}`
|
|
2555
|
+
);
|
|
2556
|
+
}
|
|
2557
|
+
return instances;
|
|
2558
|
+
}
|
|
2559
|
+
async function scanPortsForOpenCode() {
|
|
2560
|
+
const instances = [];
|
|
2561
|
+
const checks = OPENCODE_PORT_RANGE.map(async (port) => {
|
|
2562
|
+
const health = await checkOpenCodeHealth(port);
|
|
2563
|
+
if (health.healthy) {
|
|
2564
|
+
let pid = 0;
|
|
2565
|
+
try {
|
|
2566
|
+
const lsofOutput = execSync(`lsof -ti :${port} -sTCP:LISTEN 2>/dev/null`, {
|
|
1582
2567
|
encoding: "utf-8",
|
|
1583
2568
|
stdio: ["pipe", "pipe", "pipe"]
|
|
1584
2569
|
}).trim();
|
|
@@ -1615,39 +2600,45 @@ async function findHealthyOpenCodeInstances() {
|
|
|
1615
2600
|
}
|
|
1616
2601
|
return healthy;
|
|
1617
2602
|
}
|
|
1618
|
-
async function startOpenCode(port) {
|
|
2603
|
+
async function startOpenCode(port, options = {}) {
|
|
1619
2604
|
let command = "opencode";
|
|
1620
|
-
|
|
2605
|
+
const printLogs = options.inheritStdio ? ["--print-logs"] : [];
|
|
2606
|
+
let args = ["serve", "--port", port.toString(), "--hostname", "127.0.0.1", ...printLogs];
|
|
1621
2607
|
try {
|
|
1622
2608
|
execSync("which opencode", { stdio: "ignore" });
|
|
1623
2609
|
} catch {
|
|
1624
2610
|
command = "npx";
|
|
1625
|
-
args = [
|
|
1626
|
-
|
|
1627
|
-
|
|
2611
|
+
args = [
|
|
2612
|
+
"opencode",
|
|
2613
|
+
"serve",
|
|
2614
|
+
"--port",
|
|
2615
|
+
port.toString(),
|
|
2616
|
+
"--hostname",
|
|
2617
|
+
"127.0.0.1",
|
|
2618
|
+
...printLogs
|
|
2619
|
+
];
|
|
2620
|
+
}
|
|
2621
|
+
const child = spawn3(command, args, {
|
|
1628
2622
|
detached: true,
|
|
1629
|
-
stdio: "ignore",
|
|
2623
|
+
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
1630
2624
|
cwd: process.cwd()
|
|
1631
2625
|
});
|
|
1632
2626
|
return child;
|
|
1633
2627
|
}
|
|
1634
|
-
function
|
|
1635
|
-
|
|
1636
|
-
return;
|
|
1637
|
-
}
|
|
1638
|
-
try {
|
|
2628
|
+
function stopOpenCodeAndWait(opencodeProcess, timeoutMs) {
|
|
2629
|
+
const sendSignal = (signal) => {
|
|
1639
2630
|
if (process.platform === "win32") {
|
|
1640
|
-
opencodeProcess.kill(
|
|
2631
|
+
opencodeProcess.kill(signal);
|
|
1641
2632
|
} else {
|
|
1642
|
-
process.kill(-opencodeProcess.pid,
|
|
2633
|
+
process.kill(-opencodeProcess.pid, signal);
|
|
1643
2634
|
}
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
2635
|
+
};
|
|
2636
|
+
return stopProcessAndWait(
|
|
2637
|
+
opencodeProcess,
|
|
2638
|
+
timeoutMs,
|
|
2639
|
+
() => sendSignal("SIGTERM"),
|
|
2640
|
+
() => sendSignal("SIGKILL")
|
|
2641
|
+
);
|
|
1651
2642
|
}
|
|
1652
2643
|
|
|
1653
2644
|
// src/lib/opencode/install.ts
|
|
@@ -1934,6 +2925,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1934
2925
|
return data.id;
|
|
1935
2926
|
}
|
|
1936
2927
|
async function getModelAttachmentCapability(port, model) {
|
|
2928
|
+
const { model: baseModel } = splitModelVariant(model);
|
|
1937
2929
|
try {
|
|
1938
2930
|
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1939
2931
|
if (!res.ok) {
|
|
@@ -1950,9 +2942,9 @@ async function getModelAttachmentCapability(port, model) {
|
|
|
1950
2942
|
);
|
|
1951
2943
|
return null;
|
|
1952
2944
|
}
|
|
1953
|
-
const slash =
|
|
1954
|
-
const providerId = slash > 0 ?
|
|
1955
|
-
let modelId = slash > 0 ?
|
|
2945
|
+
const slash = baseModel ? baseModel.indexOf("/") : -1;
|
|
2946
|
+
const providerId = slash > 0 ? baseModel.slice(0, slash) : void 0;
|
|
2947
|
+
let modelId = slash > 0 ? baseModel.slice(slash + 1) : void 0;
|
|
1956
2948
|
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
1957
2949
|
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
1958
2950
|
if (!provider && !providerId) {
|
|
@@ -2032,6 +3024,29 @@ async function buildFileParts(attachments, capable) {
|
|
|
2032
3024
|
}
|
|
2033
3025
|
return { parts, outcomes, capabilityUnknown };
|
|
2034
3026
|
}
|
|
3027
|
+
function splitModelVariant(raw) {
|
|
3028
|
+
const value = raw?.trim();
|
|
3029
|
+
if (!value) return {};
|
|
3030
|
+
const hashIndex = value.indexOf("#");
|
|
3031
|
+
if (hashIndex === -1) return { model: value };
|
|
3032
|
+
const model = value.slice(0, hashIndex).trim() || void 0;
|
|
3033
|
+
const variant = value.slice(hashIndex + 1).trim() || void 0;
|
|
3034
|
+
return { model, variant };
|
|
3035
|
+
}
|
|
3036
|
+
function applyModelOptions(body, options) {
|
|
3037
|
+
if (options?.agent) body.agent = options.agent;
|
|
3038
|
+
const { model, variant } = splitModelVariant(options?.model);
|
|
3039
|
+
if (model) {
|
|
3040
|
+
const slashIndex = model.indexOf("/");
|
|
3041
|
+
if (slashIndex !== -1) {
|
|
3042
|
+
body.model = {
|
|
3043
|
+
providerID: model.substring(0, slashIndex),
|
|
3044
|
+
modelID: model.substring(slashIndex + 1)
|
|
3045
|
+
};
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3048
|
+
if (variant) body.variant = variant;
|
|
3049
|
+
}
|
|
2035
3050
|
function messageText(m) {
|
|
2036
3051
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2037
3052
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
@@ -2056,18 +3071,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2056
3071
|
const body = {
|
|
2057
3072
|
parts
|
|
2058
3073
|
};
|
|
2059
|
-
|
|
2060
|
-
body.agent = options.agent;
|
|
2061
|
-
}
|
|
2062
|
-
if (options?.model) {
|
|
2063
|
-
const slashIndex = options.model.indexOf("/");
|
|
2064
|
-
if (slashIndex !== -1) {
|
|
2065
|
-
body.model = {
|
|
2066
|
-
providerID: options.model.substring(0, slashIndex),
|
|
2067
|
-
modelID: options.model.substring(slashIndex + 1)
|
|
2068
|
-
};
|
|
2069
|
-
}
|
|
2070
|
-
}
|
|
3074
|
+
applyModelOptions(body, options);
|
|
2071
3075
|
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
2072
3076
|
method: "POST",
|
|
2073
3077
|
headers: { "Content-Type": "application/json" },
|
|
@@ -2075,7 +3079,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2075
3079
|
});
|
|
2076
3080
|
if (res.status < 200 || res.status >= 300) {
|
|
2077
3081
|
const text = await res.text().catch(() => "");
|
|
2078
|
-
|
|
3082
|
+
const { variant } = splitModelVariant(options?.model);
|
|
3083
|
+
throw new Error(
|
|
3084
|
+
`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}${variant ? ` (variant: ${variant})` : ""}`
|
|
3085
|
+
);
|
|
2079
3086
|
}
|
|
2080
3087
|
const READ_BACK_ATTEMPTS = 5;
|
|
2081
3088
|
const READ_BACK_DELAY_MS = 150;
|
|
@@ -2099,7 +3106,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2099
3106
|
}
|
|
2100
3107
|
}
|
|
2101
3108
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2102
|
-
await new Promise((
|
|
3109
|
+
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
2103
3110
|
}
|
|
2104
3111
|
}
|
|
2105
3112
|
return null;
|
|
@@ -2145,6 +3152,44 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2145
3152
|
}
|
|
2146
3153
|
return lastOk ?? last;
|
|
2147
3154
|
}
|
|
3155
|
+
function collectSubagentSessions(messages, userMessageId) {
|
|
3156
|
+
if (!messages || messages.length === 0) return [];
|
|
3157
|
+
const byParent = messages.filter(
|
|
3158
|
+
(message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
|
|
3159
|
+
);
|
|
3160
|
+
const assistants = byParent.length > 0 ? byParent : [];
|
|
3161
|
+
if (assistants.length === 0) {
|
|
3162
|
+
const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
|
|
3163
|
+
if (userIndex === -1) return [];
|
|
3164
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
3165
|
+
const message = messages[i];
|
|
3166
|
+
if (roleOf(message) === "user") break;
|
|
3167
|
+
if (roleOf(message) === "assistant") assistants.push(message);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
const refs = [];
|
|
3171
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3172
|
+
for (const message of assistants) {
|
|
3173
|
+
const parts = Array.isArray(message.parts) ? message.parts : [];
|
|
3174
|
+
for (const part of parts) {
|
|
3175
|
+
if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
|
|
3176
|
+
continue;
|
|
3177
|
+
const state = part.state;
|
|
3178
|
+
if (!state || typeof state !== "object") continue;
|
|
3179
|
+
const metadata = state.metadata;
|
|
3180
|
+
if (!metadata || typeof metadata !== "object") continue;
|
|
3181
|
+
const sessionId = metadata.sessionId;
|
|
3182
|
+
if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
|
|
3183
|
+
seen.add(sessionId);
|
|
3184
|
+
const start = state.time?.start;
|
|
3185
|
+
refs.push({
|
|
3186
|
+
sessionId,
|
|
3187
|
+
startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
return refs;
|
|
3192
|
+
}
|
|
2148
3193
|
function messageUsage(messages, userMessageId) {
|
|
2149
3194
|
if (!messages || messages.length === 0) return null;
|
|
2150
3195
|
const byParentAll = messages.filter(
|
|
@@ -2230,7 +3275,7 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
2230
3275
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
2231
3276
|
}
|
|
2232
3277
|
function isB2AbandonmentConfirmed(params) {
|
|
2233
|
-
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
3278
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false && params.rootOngoing === false;
|
|
2234
3279
|
}
|
|
2235
3280
|
function isAmbiguousTerminalFinish(m) {
|
|
2236
3281
|
if (completedOf(m) == null) return false;
|
|
@@ -2243,7 +3288,7 @@ function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
|
|
|
2243
3288
|
return isAmbiguousTerminalFinish(reply);
|
|
2244
3289
|
}
|
|
2245
3290
|
function isAmbiguousFinishResolved(params) {
|
|
2246
|
-
return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
|
|
3291
|
+
return params.sessionOngoing === false || params.sessionOngoing !== true && params.pinnedForMs >= params.maxPinnedMs;
|
|
2247
3292
|
}
|
|
2248
3293
|
function messageError(messages, userMessageId) {
|
|
2249
3294
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2273,8 +3318,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2273
3318
|
}
|
|
2274
3319
|
return false;
|
|
2275
3320
|
}
|
|
2276
|
-
function
|
|
2277
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3321
|
+
function classifyReplyAuthError(reply) {
|
|
2278
3322
|
const error2 = errorOf(reply);
|
|
2279
3323
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2280
3324
|
const e = error2;
|
|
@@ -2299,6 +3343,32 @@ function messageFailure(messages, userMessageId) {
|
|
|
2299
3343
|
}
|
|
2300
3344
|
return null;
|
|
2301
3345
|
}
|
|
3346
|
+
function messageFailure(messages, userMessageId) {
|
|
3347
|
+
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
3348
|
+
}
|
|
3349
|
+
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
3350
|
+
if (!messages || messages.length === 0) return null;
|
|
3351
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3352
|
+
const message = messages[i];
|
|
3353
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3354
|
+
const created = createdOf(message);
|
|
3355
|
+
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
3356
|
+
const failure = classifyReplyAuthError(message);
|
|
3357
|
+
if (failure) {
|
|
3358
|
+
if (!failure.providerId) return null;
|
|
3359
|
+
return { providerId: failure.providerId, outcome: "failed", failure };
|
|
3360
|
+
}
|
|
3361
|
+
const providerId = message.info?.providerID;
|
|
3362
|
+
if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
|
|
3363
|
+
return { providerId, outcome: "succeeded" };
|
|
3364
|
+
}
|
|
3365
|
+
return null;
|
|
3366
|
+
}
|
|
3367
|
+
return null;
|
|
3368
|
+
}
|
|
3369
|
+
function findSubagentAuthOutcome(messages, sinceMs) {
|
|
3370
|
+
return findLatestSubagentAuthOutcome(messages, sinceMs);
|
|
3371
|
+
}
|
|
2302
3372
|
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2303
3373
|
if (classified != null) return classified;
|
|
2304
3374
|
if (hasConfiguredProvider !== false) return null;
|
|
@@ -2452,13 +3522,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2452
3522
|
}
|
|
2453
3523
|
|
|
2454
3524
|
// src/lib/opencode/session-db-size.ts
|
|
2455
|
-
import { statSync as
|
|
2456
|
-
import { join as
|
|
3525
|
+
import { statSync as statSync3 } from "fs";
|
|
3526
|
+
import { join as join4 } from "path";
|
|
2457
3527
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2458
3528
|
function statSessionDbBytes(homeDir) {
|
|
2459
|
-
const dbPath =
|
|
3529
|
+
const dbPath = join4(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2460
3530
|
try {
|
|
2461
|
-
return
|
|
3531
|
+
return statSync3(dbPath).size;
|
|
2462
3532
|
} catch (err) {
|
|
2463
3533
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2464
3534
|
if (!isMissingFile) {
|
|
@@ -2484,11 +3554,11 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
2484
3554
|
}
|
|
2485
3555
|
|
|
2486
3556
|
// src/lib/opencode/session-db-reclaim.ts
|
|
2487
|
-
import { statSync as
|
|
2488
|
-
import { dirname as
|
|
3557
|
+
import { statSync as statSync4, statfsSync } from "fs";
|
|
3558
|
+
import { dirname as dirname4 } from "path";
|
|
2489
3559
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2490
3560
|
try {
|
|
2491
|
-
const fsStats = statfsSync(
|
|
3561
|
+
const fsStats = statfsSync(dirname4(dbPath));
|
|
2492
3562
|
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2493
3563
|
if (availableBytes < requiredBytes) {
|
|
2494
3564
|
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
@@ -2557,7 +3627,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2557
3627
|
);
|
|
2558
3628
|
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2559
3629
|
}
|
|
2560
|
-
const fileBytesForGuard =
|
|
3630
|
+
const fileBytesForGuard = statSync4(dbPath).size;
|
|
2561
3631
|
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2562
3632
|
if (skipReason !== null) {
|
|
2563
3633
|
console.warn(
|
|
@@ -2629,7 +3699,6 @@ var StreamForwarder = class {
|
|
|
2629
3699
|
handleFrame(frame) {
|
|
2630
3700
|
switch (frame.type) {
|
|
2631
3701
|
case "open":
|
|
2632
|
-
this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
|
|
2633
3702
|
void this.handleOpen(frame);
|
|
2634
3703
|
break;
|
|
2635
3704
|
case "req_data":
|
|
@@ -2665,12 +3734,21 @@ var StreamForwarder = class {
|
|
|
2665
3734
|
const { sid, method, path, headers, has_body } = frame;
|
|
2666
3735
|
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
2667
3736
|
const startedAt = Date.now();
|
|
3737
|
+
if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3738
|
+
this.callbacks.onOpen?.(sid, method, path);
|
|
3739
|
+
}
|
|
2668
3740
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
2669
3741
|
this.callbacks.onDrainPing?.();
|
|
2670
3742
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
2671
3743
|
this.send({ type: "res_end", sid });
|
|
2672
3744
|
return;
|
|
2673
3745
|
}
|
|
3746
|
+
if (path === TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3747
|
+
this.callbacks.onUsageRearmPing?.();
|
|
3748
|
+
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
3749
|
+
this.send({ type: "res_end", sid });
|
|
3750
|
+
return;
|
|
3751
|
+
}
|
|
2674
3752
|
if (process.env.DEBUG) {
|
|
2675
3753
|
log("debug", "agent_request", {
|
|
2676
3754
|
correlation_id: correlationId,
|
|
@@ -2685,12 +3763,12 @@ var StreamForwarder = class {
|
|
|
2685
3763
|
let endBody;
|
|
2686
3764
|
if (has_body) {
|
|
2687
3765
|
const chunks = [];
|
|
2688
|
-
bodyPromise = new Promise((
|
|
3766
|
+
bodyPromise = new Promise((resolve4) => {
|
|
2689
3767
|
pushBody = (buf) => {
|
|
2690
3768
|
chunks.push(buf);
|
|
2691
3769
|
};
|
|
2692
3770
|
endBody = () => {
|
|
2693
|
-
|
|
3771
|
+
resolve4(Buffer.concat(chunks));
|
|
2694
3772
|
};
|
|
2695
3773
|
});
|
|
2696
3774
|
}
|
|
@@ -2815,11 +3893,12 @@ function connectTunnel(options) {
|
|
|
2815
3893
|
onResponse,
|
|
2816
3894
|
onInfo,
|
|
2817
3895
|
onWarning,
|
|
2818
|
-
onDrainPing
|
|
3896
|
+
onDrainPing,
|
|
3897
|
+
onUsageRearmPing
|
|
2819
3898
|
} = options;
|
|
2820
3899
|
const tunnelUrl = getTunnelUrlConfig();
|
|
2821
3900
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
2822
|
-
return new Promise((
|
|
3901
|
+
return new Promise((resolve4, reject) => {
|
|
2823
3902
|
const ws = new WebSocket2(url, {
|
|
2824
3903
|
headers: {
|
|
2825
3904
|
Authorization: authHeader
|
|
@@ -2827,7 +3906,8 @@ function connectTunnel(options) {
|
|
|
2827
3906
|
});
|
|
2828
3907
|
const forwarder = new StreamForwarder(ws, port, {
|
|
2829
3908
|
onHead: () => onResponse?.(),
|
|
2830
|
-
onDrainPing: () => onDrainPing?.()
|
|
3909
|
+
onDrainPing: () => onDrainPing?.(),
|
|
3910
|
+
onUsageRearmPing: () => onUsageRearmPing?.()
|
|
2831
3911
|
});
|
|
2832
3912
|
const connectionTimeout = setTimeout(() => {
|
|
2833
3913
|
ws.close();
|
|
@@ -2870,8 +3950,8 @@ function connectTunnel(options) {
|
|
|
2870
3950
|
try {
|
|
2871
3951
|
message = JSON.parse(data.toString());
|
|
2872
3952
|
} catch (error2) {
|
|
2873
|
-
const
|
|
2874
|
-
onError?.(`Failed to handle message: ${
|
|
3953
|
+
const errorMessage2 = error2 instanceof Error ? error2.message : "Unknown error";
|
|
3954
|
+
onError?.(`Failed to handle message: ${errorMessage2}`);
|
|
2875
3955
|
return;
|
|
2876
3956
|
}
|
|
2877
3957
|
if (isStreamFrame(message)) {
|
|
@@ -2883,7 +3963,7 @@ function connectTunnel(options) {
|
|
|
2883
3963
|
clearTimeout(connectionTimeout);
|
|
2884
3964
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
2885
3965
|
onConnected?.(connectedAgentId);
|
|
2886
|
-
|
|
3966
|
+
resolve4({
|
|
2887
3967
|
ws,
|
|
2888
3968
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
2889
3969
|
});
|
|
@@ -2988,6 +4068,7 @@ var RunnerConnection = class {
|
|
|
2988
4068
|
onError: (error2) => events.onError?.(error2),
|
|
2989
4069
|
onResponse: () => events.onResponse?.(),
|
|
2990
4070
|
onDrainPing: () => events.onDrainPing?.(),
|
|
4071
|
+
onUsageRearmPing: () => events.onUsageRearmPing?.(),
|
|
2991
4072
|
onInfo: (message) => events.onInfo?.(message),
|
|
2992
4073
|
onWarning: (message) => events.onWarning?.(message)
|
|
2993
4074
|
});
|
|
@@ -3014,10 +4095,10 @@ var RunnerConnection = class {
|
|
|
3014
4095
|
};
|
|
3015
4096
|
|
|
3016
4097
|
// src/lib/tunnel/ready-marker.ts
|
|
3017
|
-
import { writeFileSync } from "fs";
|
|
4098
|
+
import { writeFileSync as writeFileSync3 } from "fs";
|
|
3018
4099
|
function writeTunnelReadyMarker(path, agentId) {
|
|
3019
4100
|
try {
|
|
3020
|
-
|
|
4101
|
+
writeFileSync3(path, `${agentId}
|
|
3021
4102
|
`);
|
|
3022
4103
|
return { ok: true };
|
|
3023
4104
|
} catch (error2) {
|
|
@@ -3025,10 +4106,52 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
3025
4106
|
}
|
|
3026
4107
|
}
|
|
3027
4108
|
|
|
4109
|
+
// src/lib/replication.ts
|
|
4110
|
+
import { spawn as spawn4 } from "child_process";
|
|
4111
|
+
function startSessionDbReplication(configPath) {
|
|
4112
|
+
return spawn4("litestream", ["replicate", "-config", configPath], {
|
|
4113
|
+
stdio: "inherit"
|
|
4114
|
+
});
|
|
4115
|
+
}
|
|
4116
|
+
async function stopSessionDbReplication(child, timeoutMs) {
|
|
4117
|
+
return stopProcessAndWait(
|
|
4118
|
+
child,
|
|
4119
|
+
timeoutMs,
|
|
4120
|
+
() => child.kill("SIGTERM"),
|
|
4121
|
+
() => child.kill("SIGKILL")
|
|
4122
|
+
);
|
|
4123
|
+
}
|
|
4124
|
+
|
|
4125
|
+
// src/lib/process-liveness.ts
|
|
4126
|
+
import { readFileSync as readFileSync4 } from "fs";
|
|
4127
|
+
function isProcessAlive(pid) {
|
|
4128
|
+
try {
|
|
4129
|
+
process.kill(pid, 0);
|
|
4130
|
+
} catch (error2) {
|
|
4131
|
+
const code = error2.code;
|
|
4132
|
+
if (code === "ESRCH") return false;
|
|
4133
|
+
if (code === "EPERM") return true;
|
|
4134
|
+
console.error(
|
|
4135
|
+
`[process-liveness] could not probe process ${pid}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4136
|
+
);
|
|
4137
|
+
return false;
|
|
4138
|
+
}
|
|
4139
|
+
if (process.platform !== "linux") return true;
|
|
4140
|
+
try {
|
|
4141
|
+
const status2 = readFileSync4(`/proc/${pid}/status`, "utf8");
|
|
4142
|
+
return !/^State:\s+Z(?:\s|$)/m.test(status2);
|
|
4143
|
+
} catch (error2) {
|
|
4144
|
+
console.error(
|
|
4145
|
+
`[process-liveness] could not inspect /proc/${pid}/status: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4146
|
+
);
|
|
4147
|
+
return true;
|
|
4148
|
+
}
|
|
4149
|
+
}
|
|
4150
|
+
|
|
3028
4151
|
// src/lib/openai-usage.ts
|
|
3029
|
-
import { readFileSync as
|
|
3030
|
-
import { homedir as
|
|
3031
|
-
import { join as
|
|
4152
|
+
import { readFileSync as readFileSync5 } from "fs";
|
|
4153
|
+
import { homedir as homedir3 } from "os";
|
|
4154
|
+
import { join as join5 } from "path";
|
|
3032
4155
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3033
4156
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3034
4157
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3042,7 +4165,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3042
4165
|
}
|
|
3043
4166
|
function readOpenCodeChatGptCredentials() {
|
|
3044
4167
|
try {
|
|
3045
|
-
const raw =
|
|
4168
|
+
const raw = readFileSync5(join5(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3046
4169
|
let parsed;
|
|
3047
4170
|
try {
|
|
3048
4171
|
parsed = JSON.parse(raw);
|
|
@@ -3064,6 +4187,23 @@ function readOpenCodeChatGptCredentials() {
|
|
|
3064
4187
|
return null;
|
|
3065
4188
|
}
|
|
3066
4189
|
}
|
|
4190
|
+
function parseChatGptIdentity(accessToken) {
|
|
4191
|
+
const segments = accessToken.split(".");
|
|
4192
|
+
if (segments.length !== 3) return null;
|
|
4193
|
+
let payload;
|
|
4194
|
+
try {
|
|
4195
|
+
const parsed = JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8"));
|
|
4196
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
4197
|
+
payload = parsed;
|
|
4198
|
+
} catch {
|
|
4199
|
+
return null;
|
|
4200
|
+
}
|
|
4201
|
+
const profile = payload["https://api.openai.com/profile"];
|
|
4202
|
+
const auth = payload["https://api.openai.com/auth"];
|
|
4203
|
+
const ownerEmail = typeof profile?.email === "string" ? profile.email.trim() || null : null;
|
|
4204
|
+
const planType = typeof auth?.chatgpt_plan_type === "string" ? auth.chatgpt_plan_type.trim() || null : null;
|
|
4205
|
+
return ownerEmail === null && planType === null ? null : { ownerEmail, planType };
|
|
4206
|
+
}
|
|
3067
4207
|
function toWindow2(headers, name) {
|
|
3068
4208
|
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
3069
4209
|
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
@@ -3139,6 +4279,7 @@ async function getOpenAiUsage(port) {
|
|
|
3139
4279
|
"credentials_expired"
|
|
3140
4280
|
);
|
|
3141
4281
|
}
|
|
4282
|
+
const subscription = parseChatGptIdentity(credentials2.accessToken);
|
|
3142
4283
|
const models = await resolveProbeModels(port);
|
|
3143
4284
|
if (models.length === 0) {
|
|
3144
4285
|
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
@@ -3171,7 +4312,7 @@ async function getOpenAiUsage(port) {
|
|
|
3171
4312
|
"no_usable_window"
|
|
3172
4313
|
);
|
|
3173
4314
|
}
|
|
3174
|
-
return usage;
|
|
4315
|
+
return { ...usage, subscription };
|
|
3175
4316
|
}
|
|
3176
4317
|
if (res.status === 401) {
|
|
3177
4318
|
throw new OpenAiUsageError(
|
|
@@ -3372,58 +4513,97 @@ function readDisk(homeDir) {
|
|
|
3372
4513
|
};
|
|
3373
4514
|
}
|
|
3374
4515
|
}
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
4516
|
+
var CPU_PEAK_WINDOW_MS = 6e4;
|
|
4517
|
+
var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
|
|
4518
|
+
var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4519
|
+
function createCpuPeakSampler() {
|
|
4520
|
+
const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4521
|
+
sampleHistory[0] = readCpuSample();
|
|
4522
|
+
let nextSampleIndex = 1;
|
|
4523
|
+
let sampleCount = 1;
|
|
4524
|
+
let peak = null;
|
|
4525
|
+
const timer = setInterval(() => {
|
|
3378
4526
|
const current = readCpuSample();
|
|
3379
|
-
const
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
const warnings = [];
|
|
3386
|
-
if (disk.warning) warnings.push(disk.warning);
|
|
3387
|
-
if (ecsWarning) warnings.push(ecsWarning);
|
|
3388
|
-
let cpuPercent = hostCpuPercent;
|
|
3389
|
-
let cpuCount = hostCpuCount;
|
|
3390
|
-
let memoryTotalBytes = totalmem();
|
|
3391
|
-
let memoryAvailableBytes = freemem();
|
|
3392
|
-
if (limits !== null) {
|
|
3393
|
-
cpuCount = limits.cpuCount;
|
|
3394
|
-
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3395
|
-
memoryAvailableBytes = clamp(
|
|
3396
|
-
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3397
|
-
0,
|
|
3398
|
-
limits.memoryTotalBytes
|
|
3399
|
-
);
|
|
3400
|
-
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4527
|
+
const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
|
|
4528
|
+
if (sampleFromWindowAgo !== void 0) {
|
|
4529
|
+
const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
|
|
4530
|
+
if (percentage !== null) {
|
|
4531
|
+
peak = peak === null ? percentage : Math.max(peak, percentage);
|
|
4532
|
+
}
|
|
3401
4533
|
}
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
4534
|
+
sampleHistory[nextSampleIndex] = current;
|
|
4535
|
+
nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4536
|
+
sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4537
|
+
}, CPU_PEAK_SAMPLE_INTERVAL_MS);
|
|
4538
|
+
return {
|
|
4539
|
+
takeAndReset: () => {
|
|
4540
|
+
const currentPeak = peak;
|
|
4541
|
+
peak = null;
|
|
4542
|
+
return currentPeak;
|
|
4543
|
+
},
|
|
4544
|
+
stop: () => clearInterval(timer)
|
|
4545
|
+
};
|
|
4546
|
+
}
|
|
4547
|
+
function createResourceUsageCollector(homeDir) {
|
|
4548
|
+
let previous = readCpuSample();
|
|
4549
|
+
const cpuPeakSampler = createCpuPeakSampler();
|
|
4550
|
+
return {
|
|
4551
|
+
collect: async () => {
|
|
4552
|
+
const current = readCpuSample();
|
|
4553
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
4554
|
+
const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
|
|
4555
|
+
const hostCpuCount = cpus().length;
|
|
4556
|
+
previous = current;
|
|
4557
|
+
const disk = readDisk(homeDir);
|
|
4558
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
4559
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
4560
|
+
const warnings = [];
|
|
4561
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
4562
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
4563
|
+
let cpuPercent = hostCpuPercent;
|
|
4564
|
+
let cpuPeakPercent = hostCpuPeakPercent;
|
|
4565
|
+
let cpuCount = hostCpuCount;
|
|
4566
|
+
let memoryTotalBytes = totalmem();
|
|
4567
|
+
let memoryAvailableBytes = freemem();
|
|
4568
|
+
if (limits !== null) {
|
|
4569
|
+
cpuCount = limits.cpuCount;
|
|
4570
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4571
|
+
memoryAvailableBytes = clamp(
|
|
4572
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4573
|
+
0,
|
|
4574
|
+
limits.memoryTotalBytes
|
|
4575
|
+
);
|
|
4576
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4577
|
+
cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4578
|
+
}
|
|
4579
|
+
return {
|
|
4580
|
+
usage: {
|
|
4581
|
+
cpuPercent,
|
|
4582
|
+
cpuPeakPercent,
|
|
4583
|
+
cpuCount,
|
|
4584
|
+
memoryTotalBytes,
|
|
4585
|
+
memoryAvailableBytes,
|
|
4586
|
+
diskTotalBytes: disk.totalBytes,
|
|
4587
|
+
diskFreeBytes: disk.freeBytes,
|
|
4588
|
+
opencodeDbBytes
|
|
4589
|
+
},
|
|
4590
|
+
warnings
|
|
4591
|
+
};
|
|
4592
|
+
},
|
|
4593
|
+
stop: cpuPeakSampler.stop
|
|
3414
4594
|
};
|
|
3415
4595
|
}
|
|
3416
4596
|
|
|
3417
4597
|
// src/lib/channels/driver.ts
|
|
3418
|
-
import { homedir as
|
|
4598
|
+
import { homedir as homedir4 } from "os";
|
|
3419
4599
|
|
|
3420
4600
|
// src/lib/runner-file-sync.ts
|
|
3421
|
-
import { join as
|
|
4601
|
+
import { join as join7 } from "path";
|
|
3422
4602
|
|
|
3423
4603
|
// src/lib/file-push.ts
|
|
3424
4604
|
import { randomUUID } from "crypto";
|
|
3425
4605
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
3426
|
-
import { basename, dirname as
|
|
4606
|
+
import { basename, dirname as dirname5, isAbsolute, join as join6, relative, resolve as resolve2, sep } from "path";
|
|
3427
4607
|
var FILE_MODE = 384;
|
|
3428
4608
|
var DIRECTORY_MODE = 448;
|
|
3429
4609
|
async function writePushedFile(request) {
|
|
@@ -3454,9 +4634,9 @@ async function writePushedFile(request) {
|
|
|
3454
4634
|
}
|
|
3455
4635
|
try {
|
|
3456
4636
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3457
|
-
|
|
4637
|
+
dirname5(candidate)
|
|
3458
4638
|
);
|
|
3459
|
-
const realTarget =
|
|
4639
|
+
const realTarget = join6(existingAncestor, ...missingSegments, basename(candidate));
|
|
3460
4640
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3461
4641
|
if (allowedDirectory === null) {
|
|
3462
4642
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3466,8 +4646,8 @@ async function writePushedFile(request) {
|
|
|
3466
4646
|
}
|
|
3467
4647
|
if (missingSegments.length > 0) {
|
|
3468
4648
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
3469
|
-
const realParent = await realpath(
|
|
3470
|
-
if (realParent !==
|
|
4649
|
+
const realParent = await realpath(dirname5(realTarget));
|
|
4650
|
+
if (realParent !== dirname5(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
3471
4651
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
3472
4652
|
path: realTarget,
|
|
3473
4653
|
bytes,
|
|
@@ -3492,7 +4672,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3492
4672
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3493
4673
|
return null;
|
|
3494
4674
|
}
|
|
3495
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4675
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3496
4676
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3497
4677
|
return null;
|
|
3498
4678
|
}
|
|
@@ -3510,7 +4690,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
3510
4690
|
try {
|
|
3511
4691
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
3512
4692
|
} catch (err) {
|
|
3513
|
-
const parent =
|
|
4693
|
+
const parent = dirname5(current);
|
|
3514
4694
|
if (err.code !== "ENOENT" || parent === current) {
|
|
3515
4695
|
throw err;
|
|
3516
4696
|
}
|
|
@@ -3565,13 +4745,13 @@ function contains(realDirectory, realTarget) {
|
|
|
3565
4745
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3566
4746
|
let current = existingAncestor;
|
|
3567
4747
|
for (const segment of missingSegments) {
|
|
3568
|
-
current =
|
|
4748
|
+
current = join6(current, segment);
|
|
3569
4749
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3570
4750
|
await chmod(current, DIRECTORY_MODE);
|
|
3571
4751
|
}
|
|
3572
4752
|
}
|
|
3573
4753
|
async function writeAtomically(realTarget, content) {
|
|
3574
|
-
const temporaryPath =
|
|
4754
|
+
const temporaryPath = join6(dirname5(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3575
4755
|
let handle;
|
|
3576
4756
|
try {
|
|
3577
4757
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -3701,12 +4881,12 @@ var NOT_APPLIED = {
|
|
|
3701
4881
|
opencodeAuthApplied: false
|
|
3702
4882
|
};
|
|
3703
4883
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3704
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3705
|
-
return expanded ===
|
|
4884
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4885
|
+
return expanded === join7(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3706
4886
|
}
|
|
3707
4887
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
3708
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3709
|
-
return expanded ===
|
|
4888
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4889
|
+
return expanded === join7(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
3710
4890
|
}
|
|
3711
4891
|
async function applyOne(options, file) {
|
|
3712
4892
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -4233,6 +5413,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4233
5413
|
* and stops opencode.
|
|
4234
5414
|
*/
|
|
4235
5415
|
stopped = false;
|
|
5416
|
+
recycleRequestedFlag = false;
|
|
4236
5417
|
constructor(config) {
|
|
4237
5418
|
this.agentId = config.agentId;
|
|
4238
5419
|
this.port = config.port;
|
|
@@ -4252,7 +5433,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4252
5433
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
4253
5434
|
this.now = config.now ?? (() => Date.now());
|
|
4254
5435
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
4255
|
-
this.homeDir = config.homeDir ??
|
|
5436
|
+
this.homeDir = config.homeDir ?? homedir4();
|
|
4256
5437
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4257
5438
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4258
5439
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
@@ -4338,6 +5519,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4338
5519
|
let dispatched = 0;
|
|
4339
5520
|
try {
|
|
4340
5521
|
const conversations = await this.getPendingConversations();
|
|
5522
|
+
if (this.recycleRequestedFlag) {
|
|
5523
|
+
this.stop();
|
|
5524
|
+
}
|
|
4341
5525
|
if (conversations.length > 0) {
|
|
4342
5526
|
const total = conversations.reduce((sum, c) => sum + (c.pending_message_count ?? 0), 0);
|
|
4343
5527
|
this.log({
|
|
@@ -4466,6 +5650,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4466
5650
|
stop() {
|
|
4467
5651
|
this.stopped = true;
|
|
4468
5652
|
}
|
|
5653
|
+
/**
|
|
5654
|
+
* The server clears this request when a new MicroVM identity is recorded, so a
|
|
5655
|
+
* same-VM tunnel reconnect does not consume it. This is a plain read rather
|
|
5656
|
+
* than a consume; `run.ts` guards the action once-only.
|
|
5657
|
+
*/
|
|
5658
|
+
get recycleRequested() {
|
|
5659
|
+
return this.recycleRequestedFlag;
|
|
5660
|
+
}
|
|
4469
5661
|
/**
|
|
4470
5662
|
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
4471
5663
|
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
@@ -4603,7 +5795,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4603
5795
|
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
4604
5796
|
break;
|
|
4605
5797
|
}
|
|
4606
|
-
const
|
|
5798
|
+
const errorMessage2 = err instanceof Error ? err.message : String(err);
|
|
4607
5799
|
this.sessions.delete(conv.id);
|
|
4608
5800
|
this.supersede(conv.id, sessionId);
|
|
4609
5801
|
this.log({
|
|
@@ -4612,7 +5804,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4612
5804
|
conversation_id: conv.id,
|
|
4613
5805
|
message_id: message.id
|
|
4614
5806
|
});
|
|
4615
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5807
|
+
await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
|
|
4616
5808
|
this.log({
|
|
4617
5809
|
level: "warn",
|
|
4618
5810
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4623,7 +5815,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4623
5815
|
});
|
|
4624
5816
|
this.log({
|
|
4625
5817
|
level: "error",
|
|
4626
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
5818
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage2}`,
|
|
4627
5819
|
conversation_id: conv.id,
|
|
4628
5820
|
message_id: message.id
|
|
4629
5821
|
});
|
|
@@ -4644,14 +5836,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4644
5836
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4645
5837
|
this.sessions.delete(conv.id);
|
|
4646
5838
|
this.supersede(conv.id, sessionId);
|
|
4647
|
-
const
|
|
5839
|
+
const errorMessage2 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
4648
5840
|
this.log({
|
|
4649
5841
|
level: "error",
|
|
4650
|
-
message:
|
|
5842
|
+
message: errorMessage2,
|
|
4651
5843
|
conversation_id: conv.id,
|
|
4652
5844
|
message_id: message.id
|
|
4653
5845
|
});
|
|
4654
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5846
|
+
await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
|
|
4655
5847
|
this.log({
|
|
4656
5848
|
level: "warn",
|
|
4657
5849
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4985,6 +6177,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4985
6177
|
});
|
|
4986
6178
|
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
4987
6179
|
}
|
|
6180
|
+
if (ocId !== null) {
|
|
6181
|
+
await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
|
|
6182
|
+
}
|
|
4988
6183
|
} catch (err) {
|
|
4989
6184
|
if (err instanceof ChannelAuthError) throw err;
|
|
4990
6185
|
this.log({
|
|
@@ -5519,6 +6714,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5519
6714
|
deliveryDeadlineAnchored: false,
|
|
5520
6715
|
b2PinnedSinceMs: 0,
|
|
5521
6716
|
b2LastDescendantCheckMs: 0,
|
|
6717
|
+
b2RootOngoingHeldLogged: false,
|
|
5522
6718
|
b2AbandonedSignalled: false,
|
|
5523
6719
|
ambiguousPinnedSinceMs: 0,
|
|
5524
6720
|
ambiguousResolved: false
|
|
@@ -5613,6 +6809,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5613
6809
|
deliveryDeadlineAnchored: false,
|
|
5614
6810
|
b2PinnedSinceMs: 0,
|
|
5615
6811
|
b2LastDescendantCheckMs: 0,
|
|
6812
|
+
b2RootOngoingHeldLogged: false,
|
|
5616
6813
|
b2AbandonedSignalled: false,
|
|
5617
6814
|
ambiguousPinnedSinceMs: 0,
|
|
5618
6815
|
ambiguousResolved: false
|
|
@@ -5947,6 +7144,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5947
7144
|
return;
|
|
5948
7145
|
}
|
|
5949
7146
|
inFlight.done = true;
|
|
7147
|
+
await this.reportSubagentAuthFailures(
|
|
7148
|
+
watcher.conv.id,
|
|
7149
|
+
inFlight.opencodeMessageId,
|
|
7150
|
+
inFlight.evidentMessageId,
|
|
7151
|
+
messages
|
|
7152
|
+
);
|
|
5950
7153
|
}
|
|
5951
7154
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
5952
7155
|
return;
|
|
@@ -5966,6 +7169,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5966
7169
|
if (snapshotReadable) {
|
|
5967
7170
|
inFlight.b2PinnedSinceMs = 0;
|
|
5968
7171
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
7172
|
+
inFlight.b2RootOngoingHeldLogged = false;
|
|
5969
7173
|
inFlight.b2AbandonedSignalled = false;
|
|
5970
7174
|
}
|
|
5971
7175
|
} else {
|
|
@@ -5977,11 +7181,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5977
7181
|
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
5978
7182
|
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
5979
7183
|
inFlight.b2LastDescendantCheckMs = this.now();
|
|
5980
|
-
const descendantOngoing = await
|
|
7184
|
+
const [descendantOngoing, rootOngoing] = await Promise.all([
|
|
7185
|
+
this.isAnyDescendantSessionOngoing(sessionId),
|
|
7186
|
+
isSessionOngoing(this.port, sessionId)
|
|
7187
|
+
]);
|
|
5981
7188
|
if (isB2AbandonmentConfirmed({
|
|
5982
7189
|
pinnedForMs,
|
|
5983
7190
|
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
5984
|
-
descendantOngoing
|
|
7191
|
+
descendantOngoing,
|
|
7192
|
+
rootOngoing
|
|
5985
7193
|
})) {
|
|
5986
7194
|
inFlight.b2AbandonedSignalled = true;
|
|
5987
7195
|
this.log({
|
|
@@ -5990,12 +7198,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5990
7198
|
conversation_id: conv.id,
|
|
5991
7199
|
message_id: id
|
|
5992
7200
|
});
|
|
7201
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
5993
7202
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
5994
|
-
watched_for_ms: pinnedForMs
|
|
7203
|
+
watched_for_ms: pinnedForMs,
|
|
7204
|
+
finish: reply?.info?.finish ?? reply?.finish,
|
|
7205
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
7206
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
7207
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
5995
7208
|
});
|
|
5996
7209
|
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
5997
7210
|
return;
|
|
5998
7211
|
}
|
|
7212
|
+
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
7213
|
+
inFlight.b2RootOngoingHeldLogged = true;
|
|
7214
|
+
this.log({
|
|
7215
|
+
level: "warn",
|
|
7216
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with root_ongoing=${rootOngoing} and descendant_ongoing=${descendantOngoing} \u2014 holding until OpenCode confirms the root is idle`,
|
|
7217
|
+
conversation_id: conv.id,
|
|
7218
|
+
message_id: id
|
|
7219
|
+
});
|
|
7220
|
+
}
|
|
5999
7221
|
}
|
|
6000
7222
|
}
|
|
6001
7223
|
const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
|
|
@@ -6170,6 +7392,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6170
7392
|
return;
|
|
6171
7393
|
}
|
|
6172
7394
|
inFlight.done = true;
|
|
7395
|
+
await this.reportSubagentAuthFailures(
|
|
7396
|
+
watcher.conv.id,
|
|
7397
|
+
inFlight.opencodeMessageId,
|
|
7398
|
+
inFlight.evidentMessageId,
|
|
7399
|
+
messages
|
|
7400
|
+
);
|
|
6173
7401
|
}
|
|
6174
7402
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6175
7403
|
}
|
|
@@ -6340,6 +7568,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6340
7568
|
});
|
|
6341
7569
|
return;
|
|
6342
7570
|
}
|
|
7571
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
6343
7572
|
this.dontRedispatch.delete(row.id);
|
|
6344
7573
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
6345
7574
|
return;
|
|
@@ -6501,6 +7730,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6501
7730
|
});
|
|
6502
7731
|
return;
|
|
6503
7732
|
}
|
|
7733
|
+
if (ocId !== null) {
|
|
7734
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
|
|
7735
|
+
}
|
|
6504
7736
|
this.dontRedispatch.delete(row.id);
|
|
6505
7737
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
6506
7738
|
}
|
|
@@ -6594,14 +7826,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6594
7826
|
this.unconfirmedDispatchFailures.delete(row.id);
|
|
6595
7827
|
this.sessions.delete(readoptConv.id);
|
|
6596
7828
|
this.supersede(readoptConv.id, sessionId);
|
|
6597
|
-
const
|
|
7829
|
+
const errorMessage2 = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
6598
7830
|
this.log({
|
|
6599
7831
|
level: "error",
|
|
6600
|
-
message:
|
|
7832
|
+
message: errorMessage2,
|
|
6601
7833
|
conversation_id: row.conversation_id,
|
|
6602
7834
|
message_id: row.id
|
|
6603
7835
|
});
|
|
6604
|
-
await this.markFailed(row.conversation_id, row.id, null,
|
|
7836
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage2).catch((markErr) => {
|
|
6605
7837
|
this.log({
|
|
6606
7838
|
level: "warn",
|
|
6607
7839
|
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -7216,6 +8448,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7216
8448
|
throw new Error(`Failed to get pending conversations: HTTP ${res.status}`);
|
|
7217
8449
|
}
|
|
7218
8450
|
const data = await res.json();
|
|
8451
|
+
this.recycleRequestedFlag = data.recycle_requested === true;
|
|
7219
8452
|
let conversations = data.conversations;
|
|
7220
8453
|
if (this.conversationFilter) {
|
|
7221
8454
|
conversations = conversations.filter((c) => c.id === this.conversationFilter);
|
|
@@ -7451,6 +8684,111 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7451
8684
|
reply?.info?.modelID ?? null
|
|
7452
8685
|
);
|
|
7453
8686
|
}
|
|
8687
|
+
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
8688
|
+
const providerId = failure.providerId ?? "(unknown)";
|
|
8689
|
+
try {
|
|
8690
|
+
const res = await this.fetchImpl(
|
|
8691
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
|
|
8692
|
+
{
|
|
8693
|
+
method: "POST",
|
|
8694
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
8695
|
+
body: JSON.stringify({
|
|
8696
|
+
provider_id: failure.providerId,
|
|
8697
|
+
model_id: failure.modelId,
|
|
8698
|
+
reason: failure.reason
|
|
8699
|
+
})
|
|
8700
|
+
}
|
|
8701
|
+
);
|
|
8702
|
+
if (!res.ok) {
|
|
8703
|
+
this.log({
|
|
8704
|
+
level: "warn",
|
|
8705
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8706
|
+
conversation_id: conversationId,
|
|
8707
|
+
message_id: messageId
|
|
8708
|
+
});
|
|
8709
|
+
}
|
|
8710
|
+
} catch (err) {
|
|
8711
|
+
this.log({
|
|
8712
|
+
level: "warn",
|
|
8713
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8714
|
+
conversation_id: conversationId,
|
|
8715
|
+
message_id: messageId
|
|
8716
|
+
});
|
|
8717
|
+
}
|
|
8718
|
+
}
|
|
8719
|
+
async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
|
|
8720
|
+
try {
|
|
8721
|
+
const res = await this.fetchImpl(
|
|
8722
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
|
|
8723
|
+
{
|
|
8724
|
+
method: "DELETE",
|
|
8725
|
+
headers: { Authorization: this.getAuthHeader() }
|
|
8726
|
+
}
|
|
8727
|
+
);
|
|
8728
|
+
if (!res.ok) {
|
|
8729
|
+
this.log({
|
|
8730
|
+
level: "warn",
|
|
8731
|
+
message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8732
|
+
conversation_id: conversationId,
|
|
8733
|
+
message_id: messageId
|
|
8734
|
+
});
|
|
8735
|
+
}
|
|
8736
|
+
} catch (err) {
|
|
8737
|
+
this.log({
|
|
8738
|
+
level: "warn",
|
|
8739
|
+
message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8740
|
+
conversation_id: conversationId,
|
|
8741
|
+
message_id: messageId
|
|
8742
|
+
});
|
|
8743
|
+
}
|
|
8744
|
+
}
|
|
8745
|
+
async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
|
|
8746
|
+
const refs = collectSubagentSessions(messages, opencodeMessageId);
|
|
8747
|
+
if (refs.length === 0) return;
|
|
8748
|
+
const failedProviders = /* @__PURE__ */ new Map();
|
|
8749
|
+
const succeededProviders = /* @__PURE__ */ new Set();
|
|
8750
|
+
for (const ref of refs) {
|
|
8751
|
+
try {
|
|
8752
|
+
const childMessages = await getSessionMessages(this.port, ref.sessionId);
|
|
8753
|
+
if (childMessages === null) {
|
|
8754
|
+
this.log({
|
|
8755
|
+
level: "debug",
|
|
8756
|
+
message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
|
|
8757
|
+
conversation_id: conversationId,
|
|
8758
|
+
message_id: evidentMessageId
|
|
8759
|
+
});
|
|
8760
|
+
continue;
|
|
8761
|
+
}
|
|
8762
|
+
const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
|
|
8763
|
+
if (!outcome) continue;
|
|
8764
|
+
if (outcome.outcome === "failed") {
|
|
8765
|
+
failedProviders.set(outcome.providerId, outcome.failure);
|
|
8766
|
+
} else {
|
|
8767
|
+
succeededProviders.add(outcome.providerId);
|
|
8768
|
+
}
|
|
8769
|
+
} catch (err) {
|
|
8770
|
+
this.log({
|
|
8771
|
+
level: "warn",
|
|
8772
|
+
message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8773
|
+
conversation_id: conversationId,
|
|
8774
|
+
message_id: evidentMessageId
|
|
8775
|
+
});
|
|
8776
|
+
}
|
|
8777
|
+
}
|
|
8778
|
+
for (const [providerId, failure] of failedProviders) {
|
|
8779
|
+
this.log({
|
|
8780
|
+
level: "warn",
|
|
8781
|
+
message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
|
|
8782
|
+
conversation_id: conversationId,
|
|
8783
|
+
message_id: evidentMessageId
|
|
8784
|
+
});
|
|
8785
|
+
await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
|
|
8786
|
+
}
|
|
8787
|
+
for (const providerId of succeededProviders) {
|
|
8788
|
+
if (failedProviders.has(providerId)) continue;
|
|
8789
|
+
await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
|
|
8790
|
+
}
|
|
8791
|
+
}
|
|
7454
8792
|
/**
|
|
7455
8793
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
7456
8794
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -7652,7 +8990,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
7652
8990
|
}
|
|
7653
8991
|
if (!ctx.interactive) {
|
|
7654
8992
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
7655
|
-
const proc = await startOpenCode(ctx.port);
|
|
8993
|
+
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
7656
8994
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
7657
8995
|
if (!health.healthy) {
|
|
7658
8996
|
return {
|
|
@@ -7670,66 +9008,611 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
7670
9008
|
notReadyReason: null
|
|
7671
9009
|
};
|
|
7672
9010
|
}
|
|
7673
|
-
let port = ctx.port;
|
|
7674
|
-
if (isPortInUse(port)) {
|
|
7675
|
-
console.log(chalk5.yellow(`
|
|
7676
|
-
Port ${port} is already in use.`));
|
|
7677
|
-
const alternativePort = findAvailablePort(port + 1);
|
|
7678
|
-
if (alternativePort) {
|
|
7679
|
-
const useAlternative = await select2({
|
|
7680
|
-
message: `Use port ${alternativePort} instead?`,
|
|
7681
|
-
choices: [
|
|
7682
|
-
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
7683
|
-
{ name: "No, I will free the port manually", value: "no" }
|
|
7684
|
-
]
|
|
7685
|
-
});
|
|
7686
|
-
if (useAlternative === "yes") {
|
|
7687
|
-
port = alternativePort;
|
|
7688
|
-
} else {
|
|
7689
|
-
throw new Error(`Port ${ctx.port} is in use`);
|
|
9011
|
+
let port = ctx.port;
|
|
9012
|
+
if (isPortInUse(port)) {
|
|
9013
|
+
console.log(chalk5.yellow(`
|
|
9014
|
+
Port ${port} is already in use.`));
|
|
9015
|
+
const alternativePort = findAvailablePort(port + 1);
|
|
9016
|
+
if (alternativePort) {
|
|
9017
|
+
const useAlternative = await select2({
|
|
9018
|
+
message: `Use port ${alternativePort} instead?`,
|
|
9019
|
+
choices: [
|
|
9020
|
+
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
9021
|
+
{ name: "No, I will free the port manually", value: "no" }
|
|
9022
|
+
]
|
|
9023
|
+
});
|
|
9024
|
+
if (useAlternative === "yes") {
|
|
9025
|
+
port = alternativePort;
|
|
9026
|
+
} else {
|
|
9027
|
+
throw new Error(`Port ${ctx.port} is in use`);
|
|
9028
|
+
}
|
|
9029
|
+
}
|
|
9030
|
+
}
|
|
9031
|
+
const action = await select2({
|
|
9032
|
+
message: "OpenCode is not running. What would you like to do?",
|
|
9033
|
+
choices: [
|
|
9034
|
+
{
|
|
9035
|
+
name: "Start OpenCode for me",
|
|
9036
|
+
value: "start",
|
|
9037
|
+
description: `Run 'opencode serve --port ${port}'`
|
|
9038
|
+
},
|
|
9039
|
+
{
|
|
9040
|
+
name: "Show me the command",
|
|
9041
|
+
value: "manual",
|
|
9042
|
+
description: "Display the command to run manually"
|
|
9043
|
+
},
|
|
9044
|
+
{
|
|
9045
|
+
name: "Continue without OpenCode",
|
|
9046
|
+
value: "continue",
|
|
9047
|
+
description: "Requests will fail until OpenCode starts"
|
|
9048
|
+
}
|
|
9049
|
+
]
|
|
9050
|
+
});
|
|
9051
|
+
if (action === "manual") {
|
|
9052
|
+
blank();
|
|
9053
|
+
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
9054
|
+
blank();
|
|
9055
|
+
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
9056
|
+
blank();
|
|
9057
|
+
throw new Error("Please start OpenCode manually");
|
|
9058
|
+
}
|
|
9059
|
+
if (action === "start") {
|
|
9060
|
+
const spinner = ora2("Starting OpenCode...").start();
|
|
9061
|
+
const proc = await startOpenCode(port, { inheritStdio: ctx.inheritStdio });
|
|
9062
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
9063
|
+
if (!health.healthy) {
|
|
9064
|
+
spinner.fail("Failed to start OpenCode");
|
|
9065
|
+
throw new Error("OpenCode failed to start");
|
|
9066
|
+
}
|
|
9067
|
+
spinner.stop();
|
|
9068
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
9069
|
+
}
|
|
9070
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
9071
|
+
}
|
|
9072
|
+
|
|
9073
|
+
// src/lib/runner-credentials.ts
|
|
9074
|
+
import { chmodSync as chmodSync2, writeFileSync as writeFileSync4 } from "fs";
|
|
9075
|
+
import { spawn as spawn5 } from "child_process";
|
|
9076
|
+
var RUNNER_SECRET_FETCH_TIMEOUT_MS = 6e4;
|
|
9077
|
+
var CREDENTIAL_RESTORE_TIMEOUT_MS = 6e4;
|
|
9078
|
+
var GITHUB_PROBE_TIMEOUT_MS = 1e4;
|
|
9079
|
+
var GIT_CONFIG_GLOBAL = "/tmp/gitconfig";
|
|
9080
|
+
var GIT_CREDENTIAL_HELPER = "/tmp/git-credential-helper.sh";
|
|
9081
|
+
function commandError2(result) {
|
|
9082
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
9083
|
+
}
|
|
9084
|
+
var runCommand2 = (command, args, opts) => {
|
|
9085
|
+
return new Promise((resolve4) => {
|
|
9086
|
+
let child;
|
|
9087
|
+
let stdout = "";
|
|
9088
|
+
let stderr = "";
|
|
9089
|
+
let settled = false;
|
|
9090
|
+
const timer = {};
|
|
9091
|
+
const finish = (result) => {
|
|
9092
|
+
if (settled) return;
|
|
9093
|
+
settled = true;
|
|
9094
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
9095
|
+
resolve4(result);
|
|
9096
|
+
};
|
|
9097
|
+
try {
|
|
9098
|
+
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
9099
|
+
} catch (error2) {
|
|
9100
|
+
finish({
|
|
9101
|
+
code: null,
|
|
9102
|
+
stdout,
|
|
9103
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
9104
|
+
timedOut: false
|
|
9105
|
+
});
|
|
9106
|
+
return;
|
|
9107
|
+
}
|
|
9108
|
+
child.stdout?.setEncoding("utf8");
|
|
9109
|
+
child.stdout?.on("data", (chunk) => {
|
|
9110
|
+
stdout += chunk;
|
|
9111
|
+
});
|
|
9112
|
+
child.stderr?.setEncoding("utf8");
|
|
9113
|
+
child.stderr?.on("data", (chunk) => {
|
|
9114
|
+
stderr += chunk;
|
|
9115
|
+
});
|
|
9116
|
+
child.once("error", (error2) => {
|
|
9117
|
+
finish({
|
|
9118
|
+
code: null,
|
|
9119
|
+
stdout,
|
|
9120
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
9121
|
+
${error2.message}`,
|
|
9122
|
+
timedOut: false
|
|
9123
|
+
});
|
|
9124
|
+
});
|
|
9125
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
9126
|
+
timer.handle = setTimeout(
|
|
9127
|
+
() => {
|
|
9128
|
+
child.kill("SIGKILL");
|
|
9129
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
9130
|
+
},
|
|
9131
|
+
Math.max(0, opts.timeoutMs)
|
|
9132
|
+
);
|
|
9133
|
+
});
|
|
9134
|
+
};
|
|
9135
|
+
function isEnvironmentObject(value) {
|
|
9136
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9137
|
+
}
|
|
9138
|
+
function secretFailure(marker, detail, log3) {
|
|
9139
|
+
const message = `${marker}: ${detail}`;
|
|
9140
|
+
log3(message, "error");
|
|
9141
|
+
return new Error(message);
|
|
9142
|
+
}
|
|
9143
|
+
async function installRunnerSecret({
|
|
9144
|
+
env,
|
|
9145
|
+
log: log3,
|
|
9146
|
+
commandRunner
|
|
9147
|
+
}) {
|
|
9148
|
+
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
9149
|
+
if (!arn) {
|
|
9150
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
9151
|
+
return false;
|
|
9152
|
+
}
|
|
9153
|
+
const result = await (commandRunner ?? runCommand2)(
|
|
9154
|
+
"aws",
|
|
9155
|
+
[
|
|
9156
|
+
"secretsmanager",
|
|
9157
|
+
"get-secret-value",
|
|
9158
|
+
"--secret-id",
|
|
9159
|
+
arn,
|
|
9160
|
+
"--query",
|
|
9161
|
+
"SecretString",
|
|
9162
|
+
"--output",
|
|
9163
|
+
"text"
|
|
9164
|
+
],
|
|
9165
|
+
{ env, timeoutMs: RUNNER_SECRET_FETCH_TIMEOUT_MS }
|
|
9166
|
+
);
|
|
9167
|
+
if (result.timedOut) {
|
|
9168
|
+
throw secretFailure(
|
|
9169
|
+
"CREDENTIAL-RESTORE-TIMEOUT",
|
|
9170
|
+
`runner-secret did not finish within ${RUNNER_SECRET_FETCH_TIMEOUT_MS}ms`,
|
|
9171
|
+
log3
|
|
9172
|
+
);
|
|
9173
|
+
}
|
|
9174
|
+
if (result.code !== 0) {
|
|
9175
|
+
throw secretFailure("RUNNER-SECRET-UNREADABLE", commandError2(result), log3);
|
|
9176
|
+
}
|
|
9177
|
+
let payload;
|
|
9178
|
+
try {
|
|
9179
|
+
payload = JSON.parse(result.stdout);
|
|
9180
|
+
} catch (error2) {
|
|
9181
|
+
log3(
|
|
9182
|
+
`RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9183
|
+
"warn"
|
|
9184
|
+
);
|
|
9185
|
+
return false;
|
|
9186
|
+
}
|
|
9187
|
+
if (!isEnvironmentObject(payload)) {
|
|
9188
|
+
log3("RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object", "warn");
|
|
9189
|
+
return false;
|
|
9190
|
+
}
|
|
9191
|
+
let populated = 0;
|
|
9192
|
+
let skipped = 0;
|
|
9193
|
+
let githubTokenPopulated = false;
|
|
9194
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
9195
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
9196
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
9197
|
+
log3(
|
|
9198
|
+
`RUNNER-SECRET-KEY-SKIPPED: ${JSON.stringify(key)} is not a valid environment variable name`,
|
|
9199
|
+
"warn"
|
|
9200
|
+
);
|
|
9201
|
+
skipped += 1;
|
|
9202
|
+
continue;
|
|
9203
|
+
}
|
|
9204
|
+
env[key] = value;
|
|
9205
|
+
populated += 1;
|
|
9206
|
+
if (key === "GH_TOKEN") githubTokenPopulated = true;
|
|
9207
|
+
}
|
|
9208
|
+
if (populated === 0) {
|
|
9209
|
+
log3(
|
|
9210
|
+
"RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md",
|
|
9211
|
+
"warn"
|
|
9212
|
+
);
|
|
9213
|
+
} else {
|
|
9214
|
+
log3(
|
|
9215
|
+
`RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names`
|
|
9216
|
+
);
|
|
9217
|
+
}
|
|
9218
|
+
return githubTokenPopulated;
|
|
9219
|
+
}
|
|
9220
|
+
function restoreFailure(operation, result, log3) {
|
|
9221
|
+
const message = `CREDENTIAL-RESTORE-FAILED: ${operation} failed: ${commandError2(result)}`;
|
|
9222
|
+
log3(message, "error");
|
|
9223
|
+
return new Error(message);
|
|
9224
|
+
}
|
|
9225
|
+
async function runCredentialRestore(args, operation, log3, synchroniserRunner) {
|
|
9226
|
+
const result = await synchroniserRunner(args, { timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS });
|
|
9227
|
+
if (result.timedOut) {
|
|
9228
|
+
log3(
|
|
9229
|
+
`CREDENTIAL-RESTORE-TIMEOUT: ${operation} did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9230
|
+
"warn"
|
|
9231
|
+
);
|
|
9232
|
+
return result;
|
|
9233
|
+
}
|
|
9234
|
+
if (result.code !== 0) throw restoreFailure(operation, result, log3);
|
|
9235
|
+
return result;
|
|
9236
|
+
}
|
|
9237
|
+
async function restoreCredentialStores({
|
|
9238
|
+
env,
|
|
9239
|
+
log: log3,
|
|
9240
|
+
synchroniserRunner = runSynchroniser
|
|
9241
|
+
}) {
|
|
9242
|
+
await runCredentialRestore(["restore", "claude"], "restore-claude", log3, synchroniserRunner);
|
|
9243
|
+
await runCredentialRestore(["restore", "opencode"], "restore-opencode", log3, synchroniserRunner);
|
|
9244
|
+
const result = await synchroniserRunner(["model-auth-ready"], {
|
|
9245
|
+
timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS
|
|
9246
|
+
});
|
|
9247
|
+
if (result.timedOut) {
|
|
9248
|
+
log3(
|
|
9249
|
+
`CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9250
|
+
"warn"
|
|
9251
|
+
);
|
|
9252
|
+
return;
|
|
9253
|
+
}
|
|
9254
|
+
switch (result.code) {
|
|
9255
|
+
case 0:
|
|
9256
|
+
return;
|
|
9257
|
+
case 10:
|
|
9258
|
+
log3(
|
|
9259
|
+
`no model credentials under s3://${env.LITESTREAM_BUCKET ?? ""}/${env.LITESTREAM_PREFIX ?? ""}/ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs a model provider fails until one is connected. See 'Seeding a credential store' in infrastructure/evident-runner/MICROVM.md.`,
|
|
9260
|
+
"warn"
|
|
9261
|
+
);
|
|
9262
|
+
return;
|
|
9263
|
+
default:
|
|
9264
|
+
log3("could not determine whether this VM has model credentials", "warn");
|
|
9265
|
+
}
|
|
9266
|
+
}
|
|
9267
|
+
var GIT_CREDENTIAL_HELPER_CONTENT = [
|
|
9268
|
+
"#!/usr/bin/env bash",
|
|
9269
|
+
'[ "$1" = get ] || exit 0',
|
|
9270
|
+
"echo username=x-access-token",
|
|
9271
|
+
'echo "password=${GH_TOKEN}"',
|
|
9272
|
+
""
|
|
9273
|
+
].join("\n");
|
|
9274
|
+
async function probeGitHubAccess({ env, log: log3 }) {
|
|
9275
|
+
const auth = await runCommand2("gh", ["api", "user", "--jq", ".login"], {
|
|
9276
|
+
env,
|
|
9277
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9278
|
+
});
|
|
9279
|
+
if (auth.timedOut) {
|
|
9280
|
+
log3(`GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`, "warn");
|
|
9281
|
+
return;
|
|
9282
|
+
}
|
|
9283
|
+
if (auth.code !== 0) {
|
|
9284
|
+
log3(`GITHUB-AUTH-REJECTED: ${commandError2(auth)}`, "warn");
|
|
9285
|
+
return;
|
|
9286
|
+
}
|
|
9287
|
+
log3(`GITHUB-AUTH-OK: ${auth.stdout.trim()}`, "debug");
|
|
9288
|
+
const remote = await runCommand2("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
|
9289
|
+
env,
|
|
9290
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9291
|
+
});
|
|
9292
|
+
if (remote.code !== 0 || remote.timedOut) return;
|
|
9293
|
+
const repo = remote.stdout.trim().replace(/^(?:https:\/\/github\.com\/|git@github\.com:)/, "").replace(/\.git$/, "");
|
|
9294
|
+
if (!repo) return;
|
|
9295
|
+
const repository = await runCommand2("gh", ["api", `repos/${repo}`, "--jq", ".full_name"], {
|
|
9296
|
+
env,
|
|
9297
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9298
|
+
});
|
|
9299
|
+
if (repository.timedOut) {
|
|
9300
|
+
log3(
|
|
9301
|
+
`GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`,
|
|
9302
|
+
"warn"
|
|
9303
|
+
);
|
|
9304
|
+
} else if (repository.code !== 0) {
|
|
9305
|
+
log3(`GITHUB-REPO-INACCESSIBLE: ${repo}: ${commandError2(repository)}`, "warn");
|
|
9306
|
+
}
|
|
9307
|
+
}
|
|
9308
|
+
async function configureGitHubAccess({ env, log: log3 }) {
|
|
9309
|
+
if (!env.GH_TOKEN) {
|
|
9310
|
+
log3("GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above", "warn");
|
|
9311
|
+
return;
|
|
9312
|
+
}
|
|
9313
|
+
try {
|
|
9314
|
+
env.GIT_CONFIG_GLOBAL = GIT_CONFIG_GLOBAL;
|
|
9315
|
+
writeFileSync4(GIT_CONFIG_GLOBAL, "");
|
|
9316
|
+
writeFileSync4(GIT_CREDENTIAL_HELPER, GIT_CREDENTIAL_HELPER_CONTENT, { mode: 448 });
|
|
9317
|
+
chmodSync2(GIT_CREDENTIAL_HELPER, 448);
|
|
9318
|
+
const config = [
|
|
9319
|
+
["user.name", env.GIT_USER_NAME ?? "evident-bot"],
|
|
9320
|
+
["user.email", env.GIT_USER_EMAIL ?? "evident-bot@users.noreply.github.com"],
|
|
9321
|
+
["init.defaultBranch", "main"],
|
|
9322
|
+
["credential.https://github.com.helper", GIT_CREDENTIAL_HELPER]
|
|
9323
|
+
];
|
|
9324
|
+
for (const [key, value] of config) {
|
|
9325
|
+
const result = await runCommand2("git", ["config", "--global", key, value], {
|
|
9326
|
+
env,
|
|
9327
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9328
|
+
});
|
|
9329
|
+
if (result.timedOut || result.code !== 0) throw new Error(commandError2(result));
|
|
9330
|
+
}
|
|
9331
|
+
} catch (error2) {
|
|
9332
|
+
log3(
|
|
9333
|
+
`GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9334
|
+
"warn"
|
|
9335
|
+
);
|
|
9336
|
+
return;
|
|
9337
|
+
}
|
|
9338
|
+
void probeGitHubAccess({ env, log: log3 }).catch((error2) => {
|
|
9339
|
+
log3(
|
|
9340
|
+
`GITHUB-SETUP-FAILED: GitHub access probe failed: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9341
|
+
"warn"
|
|
9342
|
+
);
|
|
9343
|
+
});
|
|
9344
|
+
}
|
|
9345
|
+
|
|
9346
|
+
// src/lib/opencode/config-overlay.ts
|
|
9347
|
+
import { execFileSync as execFileSync2 } from "child_process";
|
|
9348
|
+
import { copyFileSync, existsSync as existsSync2, statSync as statSync5 } from "fs";
|
|
9349
|
+
import { isAbsolute as isAbsolute2, join as join8, resolve as resolve3 } from "path";
|
|
9350
|
+
function isFile(filePath) {
|
|
9351
|
+
return existsSync2(filePath) && statSync5(filePath).isFile();
|
|
9352
|
+
}
|
|
9353
|
+
function applyRunnerOpenCodeConfig({
|
|
9354
|
+
overlayPath,
|
|
9355
|
+
cwd = process.cwd(),
|
|
9356
|
+
log: log3
|
|
9357
|
+
}) {
|
|
9358
|
+
if (!overlayPath) {
|
|
9359
|
+
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
9360
|
+
return;
|
|
9361
|
+
}
|
|
9362
|
+
const source = isAbsolute2(overlayPath) ? overlayPath : resolve3(cwd, overlayPath);
|
|
9363
|
+
const target = isFile(join8(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
9364
|
+
if (!isFile(source)) {
|
|
9365
|
+
log3(
|
|
9366
|
+
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
9367
|
+
"error"
|
|
9368
|
+
);
|
|
9369
|
+
return;
|
|
9370
|
+
}
|
|
9371
|
+
copyFileSync(source, join8(cwd, target));
|
|
9372
|
+
try {
|
|
9373
|
+
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
9374
|
+
stdio: "ignore"
|
|
9375
|
+
});
|
|
9376
|
+
} catch (error2) {
|
|
9377
|
+
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
9378
|
+
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
9379
|
+
}
|
|
9380
|
+
log3(`Applied runner OpenCode config ${source} to ${join8(cwd, target)}`);
|
|
9381
|
+
}
|
|
9382
|
+
|
|
9383
|
+
// src/lib/credential-sync.ts
|
|
9384
|
+
import { renameSync, writeFileSync as writeFileSync5 } from "fs";
|
|
9385
|
+
var CREDENTIAL_SYNC_TICK_TIMEOUT_MS = 3e4;
|
|
9386
|
+
var CREDENTIAL_FLUSH_DEADLINE_MS = 8e3;
|
|
9387
|
+
var CREDENTIAL_FLUSH_ABORT_GRACE_MS = 500;
|
|
9388
|
+
var DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS = 60;
|
|
9389
|
+
var STORES = ["claude", "opencode"];
|
|
9390
|
+
var MAX_FLUSH_PASSES = 2;
|
|
9391
|
+
function outcomesWith(outcome) {
|
|
9392
|
+
return { claude: outcome, opencode: outcome };
|
|
9393
|
+
}
|
|
9394
|
+
function errorMessage(error2) {
|
|
9395
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
9396
|
+
}
|
|
9397
|
+
function waitForSettlement(promise, timeoutMs) {
|
|
9398
|
+
return new Promise((resolve4) => {
|
|
9399
|
+
let settled = false;
|
|
9400
|
+
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
9401
|
+
const finish = (value) => {
|
|
9402
|
+
if (settled) return;
|
|
9403
|
+
settled = true;
|
|
9404
|
+
clearTimeout(timer);
|
|
9405
|
+
resolve4(value);
|
|
9406
|
+
};
|
|
9407
|
+
promise.then(
|
|
9408
|
+
() => finish(true),
|
|
9409
|
+
() => finish(true)
|
|
9410
|
+
);
|
|
9411
|
+
});
|
|
9412
|
+
}
|
|
9413
|
+
function writeMarker(markerPath, outcomes, log3) {
|
|
9414
|
+
const body = `${STORES.map((store) => `${store}=${outcomes[store]}`).join("\n")}
|
|
9415
|
+
`;
|
|
9416
|
+
const temporaryPath = `${markerPath}.tmp`;
|
|
9417
|
+
try {
|
|
9418
|
+
writeFileSync5(temporaryPath, body, { mode: 384 });
|
|
9419
|
+
renameSync(temporaryPath, markerPath);
|
|
9420
|
+
} catch (error2) {
|
|
9421
|
+
log3(`CREDENTIAL-FLUSH-MARKER-UNWRITABLE: ${markerPath}: ${errorMessage(error2)}`, "warn");
|
|
9422
|
+
}
|
|
9423
|
+
}
|
|
9424
|
+
function intervalSeconds(env, log3) {
|
|
9425
|
+
const raw = env.CREDS_SYNC_INTERVAL;
|
|
9426
|
+
if (raw === void 0 || /^[1-9][0-9]*$/.test(raw) && Number.isSafeInteger(Number(raw))) {
|
|
9427
|
+
return raw === void 0 ? DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS : Number(raw);
|
|
9428
|
+
}
|
|
9429
|
+
log3(
|
|
9430
|
+
`CREDENTIAL-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${raw}' is not a positive integer; using ${DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS}s`,
|
|
9431
|
+
"warn"
|
|
9432
|
+
);
|
|
9433
|
+
return DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS;
|
|
9434
|
+
}
|
|
9435
|
+
async function runFlushStore(store, env, synchroniserRunner, log3, deadlineAt) {
|
|
9436
|
+
const remainingMs = deadlineAt - Date.now();
|
|
9437
|
+
if (remainingMs <= 0) return { outcome: "timeout", orphaned: false };
|
|
9438
|
+
const controller = new AbortController();
|
|
9439
|
+
let result;
|
|
9440
|
+
let failed = false;
|
|
9441
|
+
const completion = Promise.resolve().then(
|
|
9442
|
+
() => synchroniserRunner(["sync-once", store], {
|
|
9443
|
+
timeoutMs: remainingMs,
|
|
9444
|
+
env,
|
|
9445
|
+
signal: controller.signal
|
|
9446
|
+
})
|
|
9447
|
+
).then(
|
|
9448
|
+
(value) => {
|
|
9449
|
+
result = value;
|
|
9450
|
+
},
|
|
9451
|
+
(error2) => {
|
|
9452
|
+
failed = true;
|
|
9453
|
+
log3(`CREDENTIAL-FLUSH-ERROR: ${store}: ${errorMessage(error2)}`, "warn");
|
|
9454
|
+
}
|
|
9455
|
+
);
|
|
9456
|
+
const abortTimer = setTimeout(() => controller.abort(), remainingMs);
|
|
9457
|
+
const settledBeforeDeadline = await waitForSettlement(completion, remainingMs);
|
|
9458
|
+
clearTimeout(abortTimer);
|
|
9459
|
+
if (!settledBeforeDeadline) {
|
|
9460
|
+
controller.abort();
|
|
9461
|
+
const settledAfterAbort = await waitForSettlement(completion, CREDENTIAL_FLUSH_ABORT_GRACE_MS);
|
|
9462
|
+
if (!settledAfterAbort) return { outcome: "timeout", orphaned: true };
|
|
9463
|
+
return { outcome: "timeout", orphaned: false };
|
|
9464
|
+
}
|
|
9465
|
+
if (failed || !result) return { outcome: "failed", orphaned: false };
|
|
9466
|
+
if (result.timedOut || Date.now() >= deadlineAt) {
|
|
9467
|
+
return { outcome: "timeout", orphaned: false };
|
|
9468
|
+
}
|
|
9469
|
+
return { outcome: result.code === 0 ? "ok" : "failed", orphaned: false };
|
|
9470
|
+
}
|
|
9471
|
+
function createCredentialSync({
|
|
9472
|
+
markerPath,
|
|
9473
|
+
env,
|
|
9474
|
+
log: log3,
|
|
9475
|
+
synchroniserRunner = runSynchroniser
|
|
9476
|
+
}) {
|
|
9477
|
+
const persistenceDisabled = !env.PERSISTENCE_BUCKET;
|
|
9478
|
+
let disabled = persistenceDisabled;
|
|
9479
|
+
let armed = false;
|
|
9480
|
+
let stopped = false;
|
|
9481
|
+
let timer;
|
|
9482
|
+
let inFlight;
|
|
9483
|
+
let activeTickAbort;
|
|
9484
|
+
let lastTickFailed;
|
|
9485
|
+
let flushPromise;
|
|
9486
|
+
const scheduleTick = (intervalMs, startTick2) => {
|
|
9487
|
+
if (stopped) return;
|
|
9488
|
+
timer = setTimeout(() => {
|
|
9489
|
+
timer = void 0;
|
|
9490
|
+
startTick2();
|
|
9491
|
+
}, intervalMs);
|
|
9492
|
+
};
|
|
9493
|
+
const startTick = (intervalMs) => {
|
|
9494
|
+
if (stopped) return;
|
|
9495
|
+
const controller = new AbortController();
|
|
9496
|
+
activeTickAbort = controller;
|
|
9497
|
+
const tick = (async () => {
|
|
9498
|
+
const outcomes = {
|
|
9499
|
+
claude: "failed",
|
|
9500
|
+
opencode: "failed"
|
|
9501
|
+
};
|
|
9502
|
+
for (const store of STORES) {
|
|
9503
|
+
if (controller.signal.aborted) break;
|
|
9504
|
+
try {
|
|
9505
|
+
const result = await synchroniserRunner(["sync-once", store], {
|
|
9506
|
+
timeoutMs: CREDENTIAL_SYNC_TICK_TIMEOUT_MS,
|
|
9507
|
+
env,
|
|
9508
|
+
signal: controller.signal
|
|
9509
|
+
});
|
|
9510
|
+
outcomes[store] = !result.timedOut && result.code === 0 ? "ok" : "failed";
|
|
9511
|
+
} catch (error2) {
|
|
9512
|
+
outcomes[store] = "failed";
|
|
9513
|
+
log3(`CREDENTIAL-SYNC-TICK-ERROR: ${store}: ${errorMessage(error2)}`, "debug");
|
|
9514
|
+
}
|
|
9515
|
+
}
|
|
9516
|
+
const failed = STORES.some((store) => outcomes[store] === "failed");
|
|
9517
|
+
log3(
|
|
9518
|
+
`CREDENTIAL-SYNC-TICK: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
9519
|
+
"debug"
|
|
9520
|
+
);
|
|
9521
|
+
if (failed && lastTickFailed !== true) {
|
|
9522
|
+
log3(
|
|
9523
|
+
"CREDENTIAL-SYNC-FAILED: an interval credential sync failed; retrying on the next tick",
|
|
9524
|
+
"warn"
|
|
9525
|
+
);
|
|
9526
|
+
} else if (!failed && lastTickFailed === true) {
|
|
9527
|
+
log3("CREDENTIAL-SYNC-RECOVERED: interval credential sync succeeded again", "warn");
|
|
9528
|
+
}
|
|
9529
|
+
lastTickFailed = failed;
|
|
9530
|
+
})().finally(() => {
|
|
9531
|
+
if (activeTickAbort === controller) activeTickAbort = void 0;
|
|
9532
|
+
if (inFlight === tick) inFlight = void 0;
|
|
9533
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9534
|
+
});
|
|
9535
|
+
inFlight = tick;
|
|
9536
|
+
};
|
|
9537
|
+
const performFlush = async () => {
|
|
9538
|
+
stopped = true;
|
|
9539
|
+
if (timer) {
|
|
9540
|
+
clearTimeout(timer);
|
|
9541
|
+
timer = void 0;
|
|
9542
|
+
}
|
|
9543
|
+
const deadlineAt = Date.now() + CREDENTIAL_FLUSH_DEADLINE_MS;
|
|
9544
|
+
if (inFlight) {
|
|
9545
|
+
const settled = await waitForSettlement(inFlight, CREDENTIAL_FLUSH_DEADLINE_MS);
|
|
9546
|
+
if (!settled) {
|
|
9547
|
+
activeTickAbort?.abort();
|
|
9548
|
+
const settledAfterAbort = await waitForSettlement(
|
|
9549
|
+
inFlight,
|
|
9550
|
+
CREDENTIAL_FLUSH_ABORT_GRACE_MS
|
|
9551
|
+
);
|
|
9552
|
+
if (!settledAfterAbort) {
|
|
9553
|
+
log3(
|
|
9554
|
+
"CREDENTIAL-FLUSH-ORPHANED-TICK: a sync-once child could not be proven settled; skipping the boundary flush rather than becoming a second writer",
|
|
9555
|
+
"warn"
|
|
9556
|
+
);
|
|
9557
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9558
|
+
}
|
|
7690
9559
|
}
|
|
7691
9560
|
}
|
|
7692
|
-
|
|
7693
|
-
|
|
7694
|
-
|
|
7695
|
-
|
|
7696
|
-
{
|
|
7697
|
-
|
|
7698
|
-
|
|
7699
|
-
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
name: "Show me the command",
|
|
7703
|
-
value: "manual",
|
|
7704
|
-
description: "Display the command to run manually"
|
|
7705
|
-
},
|
|
7706
|
-
{
|
|
7707
|
-
name: "Continue without OpenCode",
|
|
7708
|
-
value: "continue",
|
|
7709
|
-
description: "Requests will fail until OpenCode starts"
|
|
9561
|
+
if (disabled) return { outcomes: outcomesWith("skipped"), orphaned: false };
|
|
9562
|
+
const outcomes = outcomesWith("timeout");
|
|
9563
|
+
for (const store of STORES) {
|
|
9564
|
+
const result = await runFlushStore(store, env, synchroniserRunner, log3, deadlineAt);
|
|
9565
|
+
if (result.orphaned) {
|
|
9566
|
+
log3(
|
|
9567
|
+
"CREDENTIAL-FLUSH-ORPHANED-CHILD: a flush sync-once child could not be proven settled; refusing another flush pass rather than becoming a second writer",
|
|
9568
|
+
"warn"
|
|
9569
|
+
);
|
|
9570
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
7710
9571
|
}
|
|
7711
|
-
|
|
7712
|
-
});
|
|
7713
|
-
if (action === "manual") {
|
|
7714
|
-
blank();
|
|
7715
|
-
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
7716
|
-
blank();
|
|
7717
|
-
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
7718
|
-
blank();
|
|
7719
|
-
throw new Error("Please start OpenCode manually");
|
|
7720
|
-
}
|
|
7721
|
-
if (action === "start") {
|
|
7722
|
-
const spinner = ora2("Starting OpenCode...").start();
|
|
7723
|
-
const proc = await startOpenCode(port);
|
|
7724
|
-
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
7725
|
-
if (!health.healthy) {
|
|
7726
|
-
spinner.fail("Failed to start OpenCode");
|
|
7727
|
-
throw new Error("OpenCode failed to start");
|
|
9572
|
+
outcomes[store] = result.outcome;
|
|
7728
9573
|
}
|
|
7729
|
-
|
|
7730
|
-
|
|
7731
|
-
|
|
7732
|
-
|
|
9574
|
+
return { outcomes, orphaned: false };
|
|
9575
|
+
};
|
|
9576
|
+
let flushPasses = 0;
|
|
9577
|
+
let lastFlush;
|
|
9578
|
+
return {
|
|
9579
|
+
arm() {
|
|
9580
|
+
if (stopped || armed) return;
|
|
9581
|
+
armed = true;
|
|
9582
|
+
if (persistenceDisabled) {
|
|
9583
|
+
disabled = true;
|
|
9584
|
+
log3(
|
|
9585
|
+
"CREDENTIAL-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot",
|
|
9586
|
+
"warn"
|
|
9587
|
+
);
|
|
9588
|
+
return;
|
|
9589
|
+
}
|
|
9590
|
+
disabled = false;
|
|
9591
|
+
const intervalMs = intervalSeconds(env, log3) * 1e3;
|
|
9592
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9593
|
+
},
|
|
9594
|
+
async stopAndFlush(publish) {
|
|
9595
|
+
let result;
|
|
9596
|
+
const runningFlush = flushPromise;
|
|
9597
|
+
if (runningFlush) {
|
|
9598
|
+
result = await runningFlush;
|
|
9599
|
+
} else if (flushPasses >= MAX_FLUSH_PASSES || lastFlush?.orphaned) {
|
|
9600
|
+
result = lastFlush ?? { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9601
|
+
} else {
|
|
9602
|
+
flushPasses++;
|
|
9603
|
+
const currentFlush = performFlush();
|
|
9604
|
+
flushPromise = currentFlush;
|
|
9605
|
+
try {
|
|
9606
|
+
result = await currentFlush;
|
|
9607
|
+
lastFlush = result;
|
|
9608
|
+
} finally {
|
|
9609
|
+
if (flushPromise === currentFlush) flushPromise = void 0;
|
|
9610
|
+
}
|
|
9611
|
+
}
|
|
9612
|
+
if (publish) writeMarker(markerPath, result.outcomes, log3);
|
|
9613
|
+
return result.outcomes;
|
|
9614
|
+
}
|
|
9615
|
+
};
|
|
7733
9616
|
}
|
|
7734
9617
|
|
|
7735
9618
|
// src/commands/run.ts
|
|
@@ -7738,6 +9621,7 @@ var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_
|
|
|
7738
9621
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
7739
9622
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
7740
9623
|
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
9624
|
+
var CHILD_STOP_TIMEOUT_MS = 1e4;
|
|
7741
9625
|
function resolveLogLevel(options) {
|
|
7742
9626
|
const accepted = Object.keys(LOG_LEVELS);
|
|
7743
9627
|
const validate = (value, source) => {
|
|
@@ -7768,11 +9652,11 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
7768
9652
|
if (trimmed === "") {
|
|
7769
9653
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
7770
9654
|
}
|
|
7771
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7772
|
-
if (!
|
|
9655
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join9(homeDir, trimmed.slice(2)) : trimmed;
|
|
9656
|
+
if (!isAbsolute3(expanded)) {
|
|
7773
9657
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
7774
9658
|
}
|
|
7775
|
-
const normalized =
|
|
9659
|
+
const normalized = resolvePath2(expanded);
|
|
7776
9660
|
if (parse(normalized).root === normalized) {
|
|
7777
9661
|
throw new Error(
|
|
7778
9662
|
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
@@ -7866,7 +9750,7 @@ function logActivity(state, entry) {
|
|
|
7866
9750
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
7867
9751
|
if (!meetsThreshold(state, level)) return;
|
|
7868
9752
|
forwardRunnerActivity(
|
|
7869
|
-
{ level, message: entry.message, error: entry.error },
|
|
9753
|
+
{ level, message: entry.message, error: entry.error, metadata: entry.metadata },
|
|
7870
9754
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
7871
9755
|
);
|
|
7872
9756
|
const fullEntry = {
|
|
@@ -7886,6 +9770,36 @@ function logActivity(state, entry) {
|
|
|
7886
9770
|
}
|
|
7887
9771
|
}
|
|
7888
9772
|
}
|
|
9773
|
+
function reportSessionDbRecovery(state) {
|
|
9774
|
+
try {
|
|
9775
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir5(), env: process.env });
|
|
9776
|
+
for (const record of report.records) {
|
|
9777
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9778
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9779
|
+
logActivity(state, {
|
|
9780
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9781
|
+
level: activity.level,
|
|
9782
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9783
|
+
metadata: activity.metadata
|
|
9784
|
+
});
|
|
9785
|
+
}
|
|
9786
|
+
acknowledgeSessionDbRecoveryReport(report.path);
|
|
9787
|
+
} catch (error2) {
|
|
9788
|
+
console.error(
|
|
9789
|
+
`[run] could not report session-DB recovery activity: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
9790
|
+
);
|
|
9791
|
+
}
|
|
9792
|
+
}
|
|
9793
|
+
function reportSessionDbRecoveryRecord(state, record) {
|
|
9794
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9795
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9796
|
+
logActivity(state, {
|
|
9797
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9798
|
+
level: activity.level,
|
|
9799
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9800
|
+
metadata: activity.metadata
|
|
9801
|
+
});
|
|
9802
|
+
}
|
|
7889
9803
|
function displayStatus(state) {
|
|
7890
9804
|
if (!state.interactive) return;
|
|
7891
9805
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
@@ -7998,6 +9912,10 @@ async function driveChannels(state, driver) {
|
|
|
7998
9912
|
consecutiveDrainFailures = 0;
|
|
7999
9913
|
unreachableMs = 0;
|
|
8000
9914
|
state.messageCount += processed;
|
|
9915
|
+
if (driver.recycleRequested) {
|
|
9916
|
+
await beginGracefulShutdown(state, "recycle");
|
|
9917
|
+
return;
|
|
9918
|
+
}
|
|
8001
9919
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
8002
9920
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
8003
9921
|
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
@@ -8040,8 +9958,8 @@ async function driveChannels(state, driver) {
|
|
|
8040
9958
|
state.running = false;
|
|
8041
9959
|
break;
|
|
8042
9960
|
}
|
|
8043
|
-
const
|
|
8044
|
-
logActivity(state, { type: "error", error: `Channel processing error: ${
|
|
9961
|
+
const errorMessage2 = error2 instanceof Error ? error2.message : String(error2);
|
|
9962
|
+
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage2}` });
|
|
8045
9963
|
if (state.interactive) displayStatus(state);
|
|
8046
9964
|
if (driver.hasInFlightWatchers()) {
|
|
8047
9965
|
consecutiveDrainFailures = 0;
|
|
@@ -8058,7 +9976,7 @@ async function driveChannels(state, driver) {
|
|
|
8058
9976
|
}
|
|
8059
9977
|
}
|
|
8060
9978
|
}
|
|
8061
|
-
await new Promise((
|
|
9979
|
+
await new Promise((resolve4) => setTimeout(resolve4, CHANNEL_POLL_INTERVAL_MS));
|
|
8062
9980
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
8063
9981
|
if (idleThisCycle) idleMs += cycleMs;
|
|
8064
9982
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -8081,7 +9999,43 @@ async function driveChannels(state, driver) {
|
|
|
8081
9999
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
8082
10000
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
8083
10001
|
function sessionDbPath() {
|
|
8084
|
-
return
|
|
10002
|
+
return join9(homedir5(), ".local", "share", "opencode", "opencode.db");
|
|
10003
|
+
}
|
|
10004
|
+
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
10005
|
+
const record = {
|
|
10006
|
+
v: 1,
|
|
10007
|
+
event: "session_db_recovery",
|
|
10008
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10009
|
+
stage: "verify",
|
|
10010
|
+
outcome: "schema_provenance_mismatch",
|
|
10011
|
+
severity: "error",
|
|
10012
|
+
reason: provenance.reason ?? "schema-provenance-mismatch",
|
|
10013
|
+
litestream_exit_code: null,
|
|
10014
|
+
attempt: null,
|
|
10015
|
+
replica_objects: null,
|
|
10016
|
+
replica_bytes: null,
|
|
10017
|
+
quarantine_destination: null,
|
|
10018
|
+
quarantined_objects: null,
|
|
10019
|
+
quarantine_failed_objects: null,
|
|
10020
|
+
quarantined_bytes: null,
|
|
10021
|
+
verified_restore_point: null,
|
|
10022
|
+
restore_points_tried: null,
|
|
10023
|
+
provenance_reason: provenance.reason,
|
|
10024
|
+
provenance_migration_delta: provenance.migrationDelta,
|
|
10025
|
+
replication_suspended: false,
|
|
10026
|
+
dbPath: sessionDbPath(),
|
|
10027
|
+
recorded_version: provenance.recordedVersion,
|
|
10028
|
+
current_version: currentVersion,
|
|
10029
|
+
provenance_pre_boot_migration_count: preBootMigrationCount
|
|
10030
|
+
};
|
|
10031
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10032
|
+
if (!activity) throw new Error("could not map session-DB provenance activity");
|
|
10033
|
+
logActivity(state, {
|
|
10034
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10035
|
+
level: activity.level,
|
|
10036
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10037
|
+
metadata: activity.metadata
|
|
10038
|
+
});
|
|
8085
10039
|
}
|
|
8086
10040
|
async function runSweep(state, driver, config) {
|
|
8087
10041
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -8128,7 +10082,7 @@ async function runSweep(state, driver, config) {
|
|
|
8128
10082
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
8129
10083
|
dbPath: sessionDbPath(),
|
|
8130
10084
|
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
8131
|
-
allowFullVacuum: protectedNow.size === 0
|
|
10085
|
+
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
8132
10086
|
});
|
|
8133
10087
|
if (reclaimResult.ok) {
|
|
8134
10088
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
@@ -8164,7 +10118,7 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
8164
10118
|
for (const warning2 of config.warnings) {
|
|
8165
10119
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
8166
10120
|
}
|
|
8167
|
-
const dbBytes = statSessionDbBytes(
|
|
10121
|
+
const dbBytes = statSessionDbBytes(homedir5());
|
|
8168
10122
|
void (async () => {
|
|
8169
10123
|
const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
8170
10124
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -8323,7 +10277,17 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8323
10277
|
setTimer: (timer) => {
|
|
8324
10278
|
state.claudeUsageTimer = timer;
|
|
8325
10279
|
},
|
|
8326
|
-
fetchUsage:
|
|
10280
|
+
fetchUsage: async () => {
|
|
10281
|
+
const usage = await getClaudeUsage();
|
|
10282
|
+
if (usage.ownerLookupError) {
|
|
10283
|
+
logActivity(state, {
|
|
10284
|
+
type: "info",
|
|
10285
|
+
level: "debug",
|
|
10286
|
+
message: `Claude usage owner lookup failed: ${usage.ownerLookupError}`
|
|
10287
|
+
});
|
|
10288
|
+
}
|
|
10289
|
+
return usage;
|
|
10290
|
+
},
|
|
8327
10291
|
report: (usage) => reportClaudeUsage(state.agentId, state.authHeader, usage),
|
|
8328
10292
|
isLocalCredentialProblem,
|
|
8329
10293
|
forcedOnHint: "run `claude` to sign in",
|
|
@@ -8355,7 +10319,8 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
8355
10319
|
});
|
|
8356
10320
|
return;
|
|
8357
10321
|
}
|
|
8358
|
-
const collect = createResourceUsageCollector(
|
|
10322
|
+
const { collect, stop } = createResourceUsageCollector(homedir5());
|
|
10323
|
+
state.stopResourceUsageSampling = stop;
|
|
8359
10324
|
let consecutiveFailures = 0;
|
|
8360
10325
|
const tick = async () => {
|
|
8361
10326
|
try {
|
|
@@ -8465,21 +10430,41 @@ async function cleanup(state, opts = {}) {
|
|
|
8465
10430
|
clearTimeout(state.resourceUsageTimer);
|
|
8466
10431
|
state.resourceUsageTimer = null;
|
|
8467
10432
|
}
|
|
10433
|
+
state.stopResourceUsageSampling?.();
|
|
10434
|
+
state.stopResourceUsageSampling = null;
|
|
10435
|
+
const credentialSync = state.credentialSync;
|
|
10436
|
+
const flushCredentials = credentialSync ? async (phase, publish) => {
|
|
10437
|
+
await timeShutdownPhase(state, durations, phase, async () => {
|
|
10438
|
+
const outcomes = await credentialSync.stopAndFlush(publish);
|
|
10439
|
+
const level = Object.values(outcomes).every((outcome) => outcome === "ok") ? "info" : "warn";
|
|
10440
|
+
log2(
|
|
10441
|
+
state,
|
|
10442
|
+
`Credential flush: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
10443
|
+
level
|
|
10444
|
+
);
|
|
10445
|
+
});
|
|
10446
|
+
} : void 0;
|
|
10447
|
+
let drainSettled = true;
|
|
8468
10448
|
if (opts.graceful && state.channelDriver) {
|
|
8469
10449
|
state.channelDriver.stop();
|
|
10450
|
+
}
|
|
10451
|
+
if (flushCredentials) {
|
|
10452
|
+
await flushCredentials("credential_flush", !(opts.graceful && state.channelDriver));
|
|
10453
|
+
}
|
|
10454
|
+
if (opts.graceful && state.channelDriver) {
|
|
8470
10455
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
8471
10456
|
if (state.interactive) {
|
|
8472
10457
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
8473
10458
|
displayStatus(state);
|
|
8474
10459
|
}
|
|
8475
10460
|
const driver = state.channelDriver;
|
|
8476
|
-
|
|
10461
|
+
drainSettled = await timeShutdownPhase(
|
|
8477
10462
|
state,
|
|
8478
10463
|
durations,
|
|
8479
10464
|
"drain",
|
|
8480
10465
|
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
8481
10466
|
);
|
|
8482
|
-
if (!
|
|
10467
|
+
if (!drainSettled) {
|
|
8483
10468
|
logActivity(state, {
|
|
8484
10469
|
type: "info",
|
|
8485
10470
|
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
@@ -8487,6 +10472,9 @@ async function cleanup(state, opts = {}) {
|
|
|
8487
10472
|
if (state.interactive) displayStatus(state);
|
|
8488
10473
|
}
|
|
8489
10474
|
}
|
|
10475
|
+
if (opts.graceful && state.channelDriver && flushCredentials && drainSettled) {
|
|
10476
|
+
await flushCredentials("credential_flush_final", true);
|
|
10477
|
+
}
|
|
8490
10478
|
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
8491
10479
|
if (state.connection) {
|
|
8492
10480
|
const connection = state.connection;
|
|
@@ -8495,24 +10483,83 @@ async function cleanup(state, opts = {}) {
|
|
|
8495
10483
|
}
|
|
8496
10484
|
if (state.opencodeProcess) {
|
|
8497
10485
|
const opencodeProcess = state.opencodeProcess;
|
|
8498
|
-
|
|
10486
|
+
const result = await timeShutdownPhase(
|
|
10487
|
+
state,
|
|
10488
|
+
durations,
|
|
10489
|
+
"opencode_stop",
|
|
10490
|
+
() => stopOpenCodeAndWait(opencodeProcess, CHILD_STOP_TIMEOUT_MS)
|
|
10491
|
+
);
|
|
8499
10492
|
if (state.interactive) {
|
|
8500
|
-
logActivity(state, { type: "info", message:
|
|
10493
|
+
logActivity(state, { type: "info", message: `Stopped OpenCode process (${result.outcome})` });
|
|
8501
10494
|
displayStatus(state);
|
|
8502
10495
|
} else {
|
|
8503
|
-
log2(state,
|
|
10496
|
+
log2(state, `Stopped OpenCode process (${result.outcome})`);
|
|
8504
10497
|
}
|
|
8505
10498
|
state.opencodeProcess = null;
|
|
8506
10499
|
}
|
|
10500
|
+
if (state.litestreamProcess) {
|
|
10501
|
+
const litestreamProcess = state.litestreamProcess;
|
|
10502
|
+
const result = await timeShutdownPhase(
|
|
10503
|
+
state,
|
|
10504
|
+
durations,
|
|
10505
|
+
"litestream_stop",
|
|
10506
|
+
() => stopSessionDbReplication(litestreamProcess, CHILD_STOP_TIMEOUT_MS)
|
|
10507
|
+
);
|
|
10508
|
+
log2(state, `Stopped litestream replication (${result.outcome})`);
|
|
10509
|
+
state.litestreamProcess = null;
|
|
10510
|
+
}
|
|
8507
10511
|
return durations;
|
|
8508
10512
|
}
|
|
10513
|
+
async function beginGracefulShutdown(state, trigger) {
|
|
10514
|
+
if (state.shuttingDown) return;
|
|
10515
|
+
state.shuttingDown = true;
|
|
10516
|
+
const shutdownStartedAt = Date.now();
|
|
10517
|
+
const shutdownMessage = trigger === "recycle" ? "Recycle requested \u2014 finishing in-flight work, then exiting" : "Shutting down...";
|
|
10518
|
+
if (state.interactive) {
|
|
10519
|
+
logActivity(state, { type: "info", message: shutdownMessage });
|
|
10520
|
+
displayStatus(state);
|
|
10521
|
+
} else {
|
|
10522
|
+
log2(state, shutdownMessage);
|
|
10523
|
+
}
|
|
10524
|
+
const durations = await cleanup(state, { graceful: true });
|
|
10525
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
10526
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
10527
|
+
let timer;
|
|
10528
|
+
const flushed = shutdownTelemetry().then(
|
|
10529
|
+
() => true,
|
|
10530
|
+
(error2) => {
|
|
10531
|
+
log2(
|
|
10532
|
+
state,
|
|
10533
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10534
|
+
"warn"
|
|
10535
|
+
);
|
|
10536
|
+
return true;
|
|
10537
|
+
}
|
|
10538
|
+
);
|
|
10539
|
+
const timedOut = new Promise((resolve4) => {
|
|
10540
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
10541
|
+
});
|
|
10542
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
10543
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
10544
|
+
}
|
|
10545
|
+
clearTimeout(timer);
|
|
10546
|
+
});
|
|
10547
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
10548
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
10549
|
+
process.exit(0);
|
|
10550
|
+
}
|
|
8509
10551
|
async function run(options) {
|
|
8510
10552
|
const interactive = isInteractive(options.json);
|
|
8511
10553
|
let logLevel;
|
|
8512
10554
|
let fileSyncDirectories;
|
|
8513
10555
|
try {
|
|
8514
10556
|
logLevel = resolveLogLevel(options);
|
|
8515
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
10557
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir5());
|
|
10558
|
+
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
10559
|
+
throw new Error(
|
|
10560
|
+
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
10561
|
+
);
|
|
10562
|
+
}
|
|
8516
10563
|
} catch (error2) {
|
|
8517
10564
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8518
10565
|
if (options.json) {
|
|
@@ -8536,7 +10583,9 @@ async function run(options) {
|
|
|
8536
10583
|
connected: false,
|
|
8537
10584
|
opencodeConnected: false,
|
|
8538
10585
|
opencodeVersion: null,
|
|
10586
|
+
sessionDbProvenanceAnomaly: false,
|
|
8539
10587
|
opencodeProcess: null,
|
|
10588
|
+
litestreamProcess: null,
|
|
8540
10589
|
connection: null,
|
|
8541
10590
|
channelDriver: null,
|
|
8542
10591
|
running: true,
|
|
@@ -8550,9 +10599,24 @@ async function run(options) {
|
|
|
8550
10599
|
openaiUsageTimer: null,
|
|
8551
10600
|
openaiUsageRearm: null,
|
|
8552
10601
|
resourceUsageTimer: null,
|
|
10602
|
+
stopResourceUsageSampling: null,
|
|
10603
|
+
credentialSync: null,
|
|
8553
10604
|
authHeader: ""
|
|
8554
10605
|
};
|
|
8555
10606
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
10607
|
+
if (options.credentialSyncMarker) {
|
|
10608
|
+
state.credentialSync = createCredentialSync({
|
|
10609
|
+
markerPath: options.credentialSyncMarker,
|
|
10610
|
+
env: process.env,
|
|
10611
|
+
log: (message, level = "info") => {
|
|
10612
|
+
if (level === "error") {
|
|
10613
|
+
logActivity(state, { type: "error", error: message });
|
|
10614
|
+
} else {
|
|
10615
|
+
logActivity(state, { type: "info", level, message });
|
|
10616
|
+
}
|
|
10617
|
+
}
|
|
10618
|
+
});
|
|
10619
|
+
}
|
|
8556
10620
|
if (fileSyncDirectories.length > 0) {
|
|
8557
10621
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
8558
10622
|
} else {
|
|
@@ -8578,43 +10642,7 @@ async function run(options) {
|
|
|
8578
10642
|
"warn"
|
|
8579
10643
|
);
|
|
8580
10644
|
}
|
|
8581
|
-
const handleSignal =
|
|
8582
|
-
if (state.shuttingDown) return;
|
|
8583
|
-
state.shuttingDown = true;
|
|
8584
|
-
const shutdownStartedAt = Date.now();
|
|
8585
|
-
if (state.interactive) {
|
|
8586
|
-
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
8587
|
-
displayStatus(state);
|
|
8588
|
-
} else {
|
|
8589
|
-
log2(state, "Shutting down...");
|
|
8590
|
-
}
|
|
8591
|
-
const durations = await cleanup(state, { graceful: true });
|
|
8592
|
-
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
8593
|
-
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
8594
|
-
let timer;
|
|
8595
|
-
const flushed = shutdownTelemetry().then(
|
|
8596
|
-
() => true,
|
|
8597
|
-
(error2) => {
|
|
8598
|
-
log2(
|
|
8599
|
-
state,
|
|
8600
|
-
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8601
|
-
"warn"
|
|
8602
|
-
);
|
|
8603
|
-
return true;
|
|
8604
|
-
}
|
|
8605
|
-
);
|
|
8606
|
-
const timedOut = new Promise((resolve3) => {
|
|
8607
|
-
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
8608
|
-
});
|
|
8609
|
-
if (!await Promise.race([flushed, timedOut])) {
|
|
8610
|
-
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
8611
|
-
}
|
|
8612
|
-
clearTimeout(timer);
|
|
8613
|
-
});
|
|
8614
|
-
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
8615
|
-
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
8616
|
-
process.exit(0);
|
|
8617
|
-
};
|
|
10645
|
+
const handleSignal = () => beginGracefulShutdown(state, "signal");
|
|
8618
10646
|
process.on("SIGINT", handleSignal);
|
|
8619
10647
|
process.on("SIGTERM", handleSignal);
|
|
8620
10648
|
try {
|
|
@@ -8744,6 +10772,68 @@ async function run(options) {
|
|
|
8744
10772
|
} else {
|
|
8745
10773
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
8746
10774
|
}
|
|
10775
|
+
if (options.restoreRunnerCredentials) {
|
|
10776
|
+
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
10777
|
+
const credentialContext = {
|
|
10778
|
+
env: process.env,
|
|
10779
|
+
log: (message, level = "info") => {
|
|
10780
|
+
if (level === "error") {
|
|
10781
|
+
logActivity(state, { type: "error", error: message });
|
|
10782
|
+
} else {
|
|
10783
|
+
logActivity(state, { type: "info", level, message });
|
|
10784
|
+
}
|
|
10785
|
+
}
|
|
10786
|
+
};
|
|
10787
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
10788
|
+
await restoreCredentialStores(credentialContext);
|
|
10789
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
10790
|
+
}
|
|
10791
|
+
state.credentialSync?.arm();
|
|
10792
|
+
let sessionDbVerifyFatal = false;
|
|
10793
|
+
if (!options.restoreSessionDb) {
|
|
10794
|
+
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
10795
|
+
} else {
|
|
10796
|
+
const health = await checkOpenCodeHealth(state.port);
|
|
10797
|
+
if (health.healthy) {
|
|
10798
|
+
log2(
|
|
10799
|
+
state,
|
|
10800
|
+
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
10801
|
+
"debug"
|
|
10802
|
+
);
|
|
10803
|
+
} else {
|
|
10804
|
+
const result = await restoreAndVerifySessionDb({
|
|
10805
|
+
dbPath: sessionDbPath(),
|
|
10806
|
+
litestreamConfig: options.litestreamConfig,
|
|
10807
|
+
noReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
10808
|
+
env: process.env,
|
|
10809
|
+
log: (message, level = "info") => {
|
|
10810
|
+
if (level === "error") {
|
|
10811
|
+
logActivity(state, { type: "error", error: message });
|
|
10812
|
+
} else {
|
|
10813
|
+
logActivity(state, { type: "info", level, message });
|
|
10814
|
+
}
|
|
10815
|
+
},
|
|
10816
|
+
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
10817
|
+
});
|
|
10818
|
+
sessionDbVerifyFatal = result.verifyFatal;
|
|
10819
|
+
}
|
|
10820
|
+
}
|
|
10821
|
+
reportSessionDbRecovery(state);
|
|
10822
|
+
if (sessionDbVerifyFatal) {
|
|
10823
|
+
throw new Error(
|
|
10824
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
10825
|
+
);
|
|
10826
|
+
}
|
|
10827
|
+
applyRunnerOpenCodeConfig({
|
|
10828
|
+
overlayPath: options.opencodeConfigOverlay,
|
|
10829
|
+
log: (message, level = "info") => {
|
|
10830
|
+
if (level === "error") {
|
|
10831
|
+
logActivity(state, { type: "error", error: message });
|
|
10832
|
+
} else {
|
|
10833
|
+
logActivity(state, { type: "info", level, message });
|
|
10834
|
+
}
|
|
10835
|
+
}
|
|
10836
|
+
});
|
|
8747
10837
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
8748
10838
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
8749
10839
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
@@ -8752,6 +10842,7 @@ async function run(options) {
|
|
|
8752
10842
|
for (const warning2 of maxActiveSessionsWarnings) {
|
|
8753
10843
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
8754
10844
|
}
|
|
10845
|
+
const preBootMigrationIds = readSessionDbMigrationIds(sessionDbPath());
|
|
8755
10846
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
8756
10847
|
try {
|
|
8757
10848
|
const oc = await ensureOpenCodeRunning({
|
|
@@ -8759,11 +10850,41 @@ async function run(options) {
|
|
|
8759
10850
|
interactive: state.interactive,
|
|
8760
10851
|
agentId: state.agentId,
|
|
8761
10852
|
log: (message) => log2(state, message),
|
|
8762
|
-
startTimeoutMs: opencodeStartTimeoutMs
|
|
10853
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
10854
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
8763
10855
|
});
|
|
8764
10856
|
state.port = oc.port;
|
|
8765
|
-
state.opencodeProcess = oc.process;
|
|
10857
|
+
state.opencodeProcess = options.opencodePidFile ? null : oc.process;
|
|
8766
10858
|
state.opencodeVersion = oc.version;
|
|
10859
|
+
if (options.opencodePidFile && oc.process?.pid !== void 0) {
|
|
10860
|
+
try {
|
|
10861
|
+
writeFileSync6(options.opencodePidFile, `${oc.process.pid}
|
|
10862
|
+
`, { mode: 384 });
|
|
10863
|
+
chmodSync3(options.opencodePidFile, 384);
|
|
10864
|
+
} catch (error2) {
|
|
10865
|
+
logActivity(state, {
|
|
10866
|
+
type: "error",
|
|
10867
|
+
error: `Failed to write OpenCode pid file ${options.opencodePidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10868
|
+
});
|
|
10869
|
+
}
|
|
10870
|
+
}
|
|
10871
|
+
if (state.opencodeVersion !== null) {
|
|
10872
|
+
const provenance = checkSessionDbProvenance({
|
|
10873
|
+
dbPath: sessionDbPath(),
|
|
10874
|
+
currentVersion: state.opencodeVersion,
|
|
10875
|
+
homeDir: homedir5(),
|
|
10876
|
+
env: process.env
|
|
10877
|
+
});
|
|
10878
|
+
if (provenance.anomaly) {
|
|
10879
|
+
state.sessionDbProvenanceAnomaly = true;
|
|
10880
|
+
logSessionDbProvenanceMismatch(
|
|
10881
|
+
state,
|
|
10882
|
+
provenance,
|
|
10883
|
+
state.opencodeVersion,
|
|
10884
|
+
preBootMigrationIds?.length ?? null
|
|
10885
|
+
);
|
|
10886
|
+
}
|
|
10887
|
+
}
|
|
8767
10888
|
state.opencodeConnected = oc.notReadyReason === null;
|
|
8768
10889
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
8769
10890
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
@@ -8800,6 +10921,108 @@ async function run(options) {
|
|
|
8800
10921
|
ocSpinner?.fail(error2.message);
|
|
8801
10922
|
throw error2;
|
|
8802
10923
|
}
|
|
10924
|
+
if (options.litestreamPidFile) {
|
|
10925
|
+
if (options.sessionDbNoReplicateMarker && existsSync3(options.sessionDbNoReplicateMarker)) {
|
|
10926
|
+
log2(
|
|
10927
|
+
state,
|
|
10928
|
+
`Skipping Litestream replication because ${options.sessionDbNoReplicateMarker} marks this session database unsafe to replicate`
|
|
10929
|
+
);
|
|
10930
|
+
} else if (!options.litestreamConfig) {
|
|
10931
|
+
logActivity(state, {
|
|
10932
|
+
type: "info",
|
|
10933
|
+
level: "warn",
|
|
10934
|
+
message: "Skipping Litestream replication because no configuration file was provided"
|
|
10935
|
+
});
|
|
10936
|
+
} else {
|
|
10937
|
+
let existingPid;
|
|
10938
|
+
if (existsSync3(options.litestreamPidFile)) {
|
|
10939
|
+
try {
|
|
10940
|
+
const rawPid = readFileSync6(options.litestreamPidFile, "utf8").trim();
|
|
10941
|
+
const parsedPid = Number(rawPid);
|
|
10942
|
+
if (/^\d+$/.test(rawPid) && Number.isSafeInteger(parsedPid) && parsedPid > 0) {
|
|
10943
|
+
existingPid = parsedPid;
|
|
10944
|
+
}
|
|
10945
|
+
} catch (error2) {
|
|
10946
|
+
logActivity(state, {
|
|
10947
|
+
type: "info",
|
|
10948
|
+
level: "warn",
|
|
10949
|
+
message: `Could not read Litestream pid file ${options.litestreamPidFile}; checking for a new process: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10950
|
+
});
|
|
10951
|
+
}
|
|
10952
|
+
}
|
|
10953
|
+
if (existingPid !== void 0 && isProcessAlive(existingPid)) {
|
|
10954
|
+
log2(state, `Litestream replication is already running with pid ${existingPid}`);
|
|
10955
|
+
} else {
|
|
10956
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
10957
|
+
state.litestreamProcess = null;
|
|
10958
|
+
let failureHandled = false;
|
|
10959
|
+
const reportImageOwnedReplicationFailure = (message) => {
|
|
10960
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
10961
|
+
failureHandled = true;
|
|
10962
|
+
logActivity(state, { type: "error", error: message });
|
|
10963
|
+
if (state.interactive) displayStatus(state);
|
|
10964
|
+
};
|
|
10965
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
10966
|
+
reportImageOwnedReplicationFailure(
|
|
10967
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
10968
|
+
);
|
|
10969
|
+
});
|
|
10970
|
+
litestreamProcess.on("error", (error2) => {
|
|
10971
|
+
reportImageOwnedReplicationFailure(
|
|
10972
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10973
|
+
);
|
|
10974
|
+
});
|
|
10975
|
+
try {
|
|
10976
|
+
if (litestreamProcess.pid !== void 0) {
|
|
10977
|
+
writeFileSync6(options.litestreamPidFile, `${litestreamProcess.pid}
|
|
10978
|
+
`, {
|
|
10979
|
+
mode: 384
|
|
10980
|
+
});
|
|
10981
|
+
chmodSync3(options.litestreamPidFile, 384);
|
|
10982
|
+
}
|
|
10983
|
+
} catch (error2) {
|
|
10984
|
+
logActivity(state, {
|
|
10985
|
+
type: "error",
|
|
10986
|
+
error: `Failed to write Litestream pid file ${options.litestreamPidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10987
|
+
});
|
|
10988
|
+
}
|
|
10989
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
10990
|
+
}
|
|
10991
|
+
}
|
|
10992
|
+
} else if (options.litestreamConfig) {
|
|
10993
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
10994
|
+
state.litestreamProcess = litestreamProcess;
|
|
10995
|
+
let failureHandled = false;
|
|
10996
|
+
const failRunForReplication = (message) => {
|
|
10997
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
10998
|
+
failureHandled = true;
|
|
10999
|
+
state.shuttingDown = true;
|
|
11000
|
+
logActivity(state, { type: "error", error: message });
|
|
11001
|
+
if (state.interactive) displayStatus(state);
|
|
11002
|
+
void (async () => {
|
|
11003
|
+
try {
|
|
11004
|
+
await cleanup(state);
|
|
11005
|
+
await shutdownTelemetry();
|
|
11006
|
+
} catch (error2) {
|
|
11007
|
+
console.error(
|
|
11008
|
+
`[run] replication failure cleanup failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11009
|
+
);
|
|
11010
|
+
}
|
|
11011
|
+
process.exit(1);
|
|
11012
|
+
})();
|
|
11013
|
+
};
|
|
11014
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
11015
|
+
failRunForReplication(
|
|
11016
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
11017
|
+
);
|
|
11018
|
+
});
|
|
11019
|
+
litestreamProcess.on("error", (error2) => {
|
|
11020
|
+
failRunForReplication(
|
|
11021
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11022
|
+
);
|
|
11023
|
+
});
|
|
11024
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
11025
|
+
}
|
|
8803
11026
|
const tunnelSpinner = interactive && !state.json ? ora3("Connecting tunnel...").start() : null;
|
|
8804
11027
|
const channelDriver = new ChannelDriver({
|
|
8805
11028
|
agentId: state.agentId,
|
|
@@ -8811,7 +11034,7 @@ async function run(options) {
|
|
|
8811
11034
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
8812
11035
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
8813
11036
|
fileSyncDirectories,
|
|
8814
|
-
homeDir:
|
|
11037
|
+
homeDir: homedir5(),
|
|
8815
11038
|
maxActiveSessions,
|
|
8816
11039
|
log: (entry) => (
|
|
8817
11040
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -8937,6 +11160,18 @@ async function run(options) {
|
|
|
8937
11160
|
if (state.interactive) displayStatus(state);
|
|
8938
11161
|
});
|
|
8939
11162
|
},
|
|
11163
|
+
// Both loops are rearmed because `rearm()` is idempotent for the
|
|
11164
|
+
// provider that did not just connect, and is a no-op when reporting is off.
|
|
11165
|
+
onUsageRearmPing: () => {
|
|
11166
|
+
if (!state.running) return;
|
|
11167
|
+
logActivity(state, {
|
|
11168
|
+
type: "info",
|
|
11169
|
+
level: "debug",
|
|
11170
|
+
message: "Usage rearm ping received"
|
|
11171
|
+
});
|
|
11172
|
+
state.claudeUsageRearm?.();
|
|
11173
|
+
state.openaiUsageRearm?.();
|
|
11174
|
+
},
|
|
8940
11175
|
onInfo: (message) => logActivity(state, { type: "info", message })
|
|
8941
11176
|
}
|
|
8942
11177
|
});
|
|
@@ -8957,7 +11192,17 @@ async function run(options) {
|
|
|
8957
11192
|
setTimer: (timer) => {
|
|
8958
11193
|
state.openaiUsageTimer = timer;
|
|
8959
11194
|
},
|
|
8960
|
-
fetchUsage: () =>
|
|
11195
|
+
fetchUsage: async () => {
|
|
11196
|
+
const usage = await getOpenAiUsage(state.port);
|
|
11197
|
+
if (usage.subscription === null) {
|
|
11198
|
+
logActivity(state, {
|
|
11199
|
+
type: "info",
|
|
11200
|
+
level: "debug",
|
|
11201
|
+
message: "OpenAI usage subscription could not be identified from the local credential"
|
|
11202
|
+
});
|
|
11203
|
+
}
|
|
11204
|
+
return usage;
|
|
11205
|
+
},
|
|
8961
11206
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
8962
11207
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
8963
11208
|
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
@@ -9003,7 +11248,7 @@ async function run(options) {
|
|
|
9003
11248
|
}
|
|
9004
11249
|
|
|
9005
11250
|
// src/index.ts
|
|
9006
|
-
var { version } =
|
|
11251
|
+
var { version } = createRequire2(import.meta.url)("../package.json");
|
|
9007
11252
|
var program = new Command();
|
|
9008
11253
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
9009
11254
|
"--endpoint <url>",
|
|
@@ -9060,6 +11305,30 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9060
11305
|
).option(
|
|
9061
11306
|
"--tunnel-ready-file <path>",
|
|
9062
11307
|
"Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
|
|
11308
|
+
).option(
|
|
11309
|
+
"--litestream-config <path>",
|
|
11310
|
+
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
11311
|
+
).option(
|
|
11312
|
+
"--opencode-pid-file <path>",
|
|
11313
|
+
"Record the OpenCode pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11314
|
+
).option(
|
|
11315
|
+
"--litestream-pid-file <path>",
|
|
11316
|
+
"Record the Litestream pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11317
|
+
).option(
|
|
11318
|
+
"--session-db-no-replicate-marker <path>",
|
|
11319
|
+
"Read this marker as a gate before starting Litestream; the session-DB restore writes it when this boot's database is not safe to replicate."
|
|
11320
|
+
).option(
|
|
11321
|
+
"--restore-session-db",
|
|
11322
|
+
"Restore and verify the OpenCode session database before starting OpenCode; requires --litestream-config for the replica and --session-db-no-replicate-marker for give-up records."
|
|
11323
|
+
).option(
|
|
11324
|
+
"--restore-runner-credentials",
|
|
11325
|
+
"Restore the hosted runner secret and persisted credential stores before starting OpenCode."
|
|
11326
|
+
).option(
|
|
11327
|
+
"--opencode-config-overlay <path>",
|
|
11328
|
+
"Apply this runner-provided OpenCode config before starting OpenCode."
|
|
11329
|
+
).option(
|
|
11330
|
+
"--credential-sync-marker <path>",
|
|
11331
|
+
"Own the interval credential sync and write this marker once the shutdown flush has finished, so the runner image's lifecycle hooks can wait on it."
|
|
9063
11332
|
).action(
|
|
9064
11333
|
(options) => {
|
|
9065
11334
|
run({
|
|
@@ -9091,7 +11360,15 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9091
11360
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
9092
11361
|
// resolveFileSyncDirectories.
|
|
9093
11362
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
9094
|
-
tunnelReadyFile: options.tunnelReadyFile
|
|
11363
|
+
tunnelReadyFile: options.tunnelReadyFile,
|
|
11364
|
+
litestreamConfig: options.litestreamConfig,
|
|
11365
|
+
opencodePidFile: options.opencodePidFile,
|
|
11366
|
+
litestreamPidFile: options.litestreamPidFile,
|
|
11367
|
+
sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
11368
|
+
restoreSessionDb: options.restoreSessionDb,
|
|
11369
|
+
restoreRunnerCredentials: options.restoreRunnerCredentials,
|
|
11370
|
+
opencodeConfigOverlay: options.opencodeConfigOverlay,
|
|
11371
|
+
credentialSyncMarker: options.credentialSyncMarker
|
|
9095
11372
|
});
|
|
9096
11373
|
}
|
|
9097
11374
|
);
|