@evident-ai/cli 3.4.1-dev.1549524 → 3.4.1-dev.1633d8c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/index.js +2599 -348
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire } from "module";
|
|
4
|
+
import { createRequire as createRequire2 } from "module";
|
|
5
5
|
import { Command } from "commander";
|
|
6
6
|
|
|
7
7
|
// src/commands/login.ts
|
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
-
import { dirname } from "path";
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "node:fs";
|
|
15
|
+
import { dirname } from "node:path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -371,14 +371,14 @@ function blank() {
|
|
|
371
371
|
console.log();
|
|
372
372
|
}
|
|
373
373
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
374
|
-
return new Promise((
|
|
374
|
+
return new Promise((resolve4) => {
|
|
375
375
|
process.stdout.write(chalk.dim(prompt));
|
|
376
376
|
const handler = () => {
|
|
377
377
|
process.stdin.removeListener("data", handler);
|
|
378
378
|
process.stdin.setRawMode?.(false);
|
|
379
379
|
process.stdin.pause();
|
|
380
380
|
console.log();
|
|
381
|
-
|
|
381
|
+
resolve4();
|
|
382
382
|
};
|
|
383
383
|
if (process.stdin.isTTY) {
|
|
384
384
|
process.stdin.setRawMode?.(true);
|
|
@@ -388,7 +388,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
388
388
|
});
|
|
389
389
|
}
|
|
390
390
|
function sleep(ms) {
|
|
391
|
-
return new Promise((
|
|
391
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
392
392
|
}
|
|
393
393
|
|
|
394
394
|
// src/commands/login.ts
|
|
@@ -466,19 +466,19 @@ async function tokenLogin() {
|
|
|
466
466
|
);
|
|
467
467
|
blank();
|
|
468
468
|
process.stdout.write("Paste token: ");
|
|
469
|
-
const token = await new Promise((
|
|
469
|
+
const token = await new Promise((resolve4) => {
|
|
470
470
|
let data = "";
|
|
471
471
|
process.stdin.setEncoding("utf8");
|
|
472
472
|
process.stdin.on("data", (chunk) => {
|
|
473
473
|
data += chunk;
|
|
474
474
|
});
|
|
475
475
|
process.stdin.on("end", () => {
|
|
476
|
-
|
|
476
|
+
resolve4(data.trim());
|
|
477
477
|
});
|
|
478
478
|
if (process.stdin.isTTY) {
|
|
479
479
|
process.stdin.once("data", (chunk) => {
|
|
480
480
|
process.stdin.pause();
|
|
481
|
-
|
|
481
|
+
resolve4(chunk.toString().trim());
|
|
482
482
|
});
|
|
483
483
|
process.stdin.resume();
|
|
484
484
|
}
|
|
@@ -623,6 +623,19 @@ function isInteractive(jsonOutput) {
|
|
|
623
623
|
return true;
|
|
624
624
|
}
|
|
625
625
|
|
|
626
|
+
// src/lib/subscription-usage-report.ts
|
|
627
|
+
function toReportedSubscription(collected) {
|
|
628
|
+
if (!collected) return null;
|
|
629
|
+
if (collected.ownerEmail === null && collected.planType === null && collected.organizationName === null) {
|
|
630
|
+
return null;
|
|
631
|
+
}
|
|
632
|
+
return {
|
|
633
|
+
owner_email: collected.ownerEmail,
|
|
634
|
+
plan_type: collected.planType,
|
|
635
|
+
organization_name: collected.organizationName
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
|
|
626
639
|
// src/commands/agent-lookup.ts
|
|
627
640
|
async function readErrorMessage(response) {
|
|
628
641
|
const text = await response.text().catch(() => "");
|
|
@@ -722,14 +735,6 @@ function toReportedWindow(window) {
|
|
|
722
735
|
if (!window) return null;
|
|
723
736
|
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
724
737
|
}
|
|
725
|
-
function toReportedOwner(snapshot) {
|
|
726
|
-
if (!snapshot.owner) return null;
|
|
727
|
-
return {
|
|
728
|
-
email: snapshot.owner.email,
|
|
729
|
-
organization_name: snapshot.owner.organizationName,
|
|
730
|
-
rate_limit_tier: snapshot.owner.rateLimitTier
|
|
731
|
-
};
|
|
732
|
-
}
|
|
733
738
|
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
734
739
|
try {
|
|
735
740
|
const apiUrl = getApiUrlConfig();
|
|
@@ -739,7 +744,7 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
739
744
|
body: JSON.stringify({
|
|
740
745
|
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
741
746
|
seven_day: toReportedWindow(snapshot.sevenDay),
|
|
742
|
-
|
|
747
|
+
subscription: toReportedSubscription(snapshot.subscription)
|
|
743
748
|
}),
|
|
744
749
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
745
750
|
});
|
|
@@ -773,7 +778,8 @@ async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
|
773
778
|
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
774
779
|
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
775
780
|
has_credits: snapshot.hasCredits,
|
|
776
|
-
credits_unlimited: snapshot.creditsUnlimited
|
|
781
|
+
credits_unlimited: snapshot.creditsUnlimited,
|
|
782
|
+
subscription: toReportedSubscription(snapshot.subscription)
|
|
777
783
|
}),
|
|
778
784
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
779
785
|
});
|
|
@@ -797,6 +803,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
797
803
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
798
804
|
body: JSON.stringify({
|
|
799
805
|
cpu_percent: usage.cpuPercent,
|
|
806
|
+
cpu_peak_percent: usage.cpuPeakPercent,
|
|
800
807
|
cpu_count: usage.cpuCount,
|
|
801
808
|
memory_total_bytes: usage.memoryTotalBytes,
|
|
802
809
|
memory_available_bytes: usage.memoryAvailableBytes,
|
|
@@ -1000,10 +1007,10 @@ async function status(options = {}) {
|
|
|
1000
1007
|
}
|
|
1001
1008
|
|
|
1002
1009
|
// src/lib/claude-usage.ts
|
|
1003
|
-
import { execFileSync } from "child_process";
|
|
1004
|
-
import { readFileSync } from "fs";
|
|
1005
|
-
import { homedir } from "os";
|
|
1006
|
-
import { join } from "path";
|
|
1010
|
+
import { execFileSync } from "node:child_process";
|
|
1011
|
+
import { readFileSync } from "node:fs";
|
|
1012
|
+
import { homedir } from "node:os";
|
|
1013
|
+
import { join } from "node:path";
|
|
1007
1014
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
1008
1015
|
var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
|
|
1009
1016
|
var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
|
|
@@ -1088,7 +1095,7 @@ function ownerLookupFailure(error2) {
|
|
|
1088
1095
|
}
|
|
1089
1096
|
async function getClaudeUsageOwner(accessToken) {
|
|
1090
1097
|
if (cachedOwner?.accessToken === accessToken) {
|
|
1091
|
-
return {
|
|
1098
|
+
return { subscription: cachedOwner.owner, ownerLookupError: null };
|
|
1092
1099
|
}
|
|
1093
1100
|
try {
|
|
1094
1101
|
const response = await fetch(CLAUDE_PROFILE_URL, {
|
|
@@ -1100,27 +1107,27 @@ async function getClaudeUsageOwner(accessToken) {
|
|
|
1100
1107
|
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1101
1108
|
});
|
|
1102
1109
|
if (!response.ok) {
|
|
1103
|
-
return {
|
|
1110
|
+
return { subscription: null, ownerLookupError: `HTTP ${response.status}` };
|
|
1104
1111
|
}
|
|
1105
1112
|
let body;
|
|
1106
1113
|
try {
|
|
1107
1114
|
body = await response.json();
|
|
1108
1115
|
} catch (error2) {
|
|
1109
|
-
return {
|
|
1116
|
+
return { subscription: null, ownerLookupError: "malformed response" };
|
|
1110
1117
|
}
|
|
1111
1118
|
const profile = body;
|
|
1112
1119
|
if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
|
|
1113
|
-
return {
|
|
1120
|
+
return { subscription: null, ownerLookupError: "malformed response" };
|
|
1114
1121
|
}
|
|
1115
|
-
const
|
|
1116
|
-
|
|
1122
|
+
const subscription = {
|
|
1123
|
+
ownerEmail: profile.account.email,
|
|
1117
1124
|
organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
|
|
1118
|
-
|
|
1125
|
+
planType: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
|
|
1119
1126
|
};
|
|
1120
|
-
cachedOwner = { accessToken, owner };
|
|
1121
|
-
return {
|
|
1127
|
+
cachedOwner = { accessToken, owner: subscription };
|
|
1128
|
+
return { subscription, ownerLookupError: null };
|
|
1122
1129
|
} catch (error2) {
|
|
1123
|
-
return {
|
|
1130
|
+
return { subscription: null, ownerLookupError: ownerLookupFailure(error2) };
|
|
1124
1131
|
}
|
|
1125
1132
|
}
|
|
1126
1133
|
async function getClaudeUsage() {
|
|
@@ -1149,11 +1156,11 @@ async function getClaudeUsage() {
|
|
|
1149
1156
|
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
1150
1157
|
}
|
|
1151
1158
|
const body = await res.json();
|
|
1152
|
-
const {
|
|
1159
|
+
const { subscription, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
|
|
1153
1160
|
return {
|
|
1154
1161
|
fiveHour: toWindow(body.five_hour),
|
|
1155
1162
|
sevenDay: toWindow(body.seven_day),
|
|
1156
|
-
|
|
1163
|
+
subscription,
|
|
1157
1164
|
ownerLookupError
|
|
1158
1165
|
};
|
|
1159
1166
|
}
|
|
@@ -1183,9 +1190,10 @@ async function claudeUsage() {
|
|
|
1183
1190
|
}
|
|
1184
1191
|
|
|
1185
1192
|
// src/commands/run.ts
|
|
1186
|
-
import {
|
|
1187
|
-
import {
|
|
1188
|
-
import
|
|
1193
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1194
|
+
import { homedir as homedir6 } from "node:os";
|
|
1195
|
+
import { isAbsolute as isAbsolute3, join as join10, parse, resolve as resolvePath2 } from "node:path";
|
|
1196
|
+
import chalk7 from "chalk";
|
|
1189
1197
|
|
|
1190
1198
|
// ../../packages/types/src/agents/index.ts
|
|
1191
1199
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
@@ -1206,6 +1214,7 @@ var TelemetryEventTypes = {
|
|
|
1206
1214
|
// ../../packages/types/src/tunnel/index.ts
|
|
1207
1215
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1208
1216
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1217
|
+
var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
|
|
1209
1218
|
|
|
1210
1219
|
// ../../packages/types/src/runner-files.ts
|
|
1211
1220
|
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
@@ -1243,7 +1252,7 @@ function stripQuery(url) {
|
|
|
1243
1252
|
|
|
1244
1253
|
// src/commands/run.ts
|
|
1245
1254
|
import ora3 from "ora";
|
|
1246
|
-
import { select as
|
|
1255
|
+
import { select as select4 } from "@inquirer/prompts";
|
|
1247
1256
|
|
|
1248
1257
|
// src/lib/telemetry.ts
|
|
1249
1258
|
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
@@ -1416,12 +1425,50 @@ var SEVERITY_BY_LEVEL = {
|
|
|
1416
1425
|
warn: "warning",
|
|
1417
1426
|
error: "error"
|
|
1418
1427
|
};
|
|
1428
|
+
function parseOpenCodeLogLine(line) {
|
|
1429
|
+
const normalisedLine = line.replace(/\r$/, "");
|
|
1430
|
+
const levelMatch = normalisedLine.match(/(?:^|\s)level=(\w+)/i);
|
|
1431
|
+
if (!levelMatch) return null;
|
|
1432
|
+
const level = levelMatch[1].toUpperCase();
|
|
1433
|
+
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1434
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1435
|
+
return { level: level === "WARN" ? "warn" : "error", sessionID: sessionMatch?.[1] };
|
|
1436
|
+
}
|
|
1437
|
+
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1438
|
+
function createOpenCodeActivityForwarder(getContext) {
|
|
1439
|
+
let buffer = Buffer.alloc(0);
|
|
1440
|
+
const flushLine = (line) => {
|
|
1441
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1442
|
+
if (!parsed) return;
|
|
1443
|
+
forwardRunnerActivity(
|
|
1444
|
+
{
|
|
1445
|
+
level: parsed.level,
|
|
1446
|
+
error: line,
|
|
1447
|
+
metadata: parsed.sessionID ? { sessionID: parsed.sessionID } : void 0,
|
|
1448
|
+
source: "opencode"
|
|
1449
|
+
},
|
|
1450
|
+
getContext()
|
|
1451
|
+
);
|
|
1452
|
+
};
|
|
1453
|
+
return (chunk) => {
|
|
1454
|
+
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1455
|
+
let newlineIndex;
|
|
1456
|
+
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
1457
|
+
flushLine(buffer.subarray(0, newlineIndex).toString("utf-8").replace(/\r$/, ""));
|
|
1458
|
+
buffer = buffer.subarray(newlineIndex + 1);
|
|
1459
|
+
}
|
|
1460
|
+
if (buffer.length > MAX_LINE_BUFFER_BYTES) {
|
|
1461
|
+
flushLine(buffer.toString("utf-8"));
|
|
1462
|
+
buffer = Buffer.alloc(0);
|
|
1463
|
+
}
|
|
1464
|
+
};
|
|
1465
|
+
}
|
|
1419
1466
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1420
1467
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1421
1468
|
var MAX_METADATA_ENTRIES = 20;
|
|
1422
1469
|
var TRUNCATION_MARKER = "\u2026";
|
|
1423
1470
|
function redact(message) {
|
|
1424
|
-
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1471
|
+
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1425
1472
|
}
|
|
1426
1473
|
function truncate(message) {
|
|
1427
1474
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
@@ -1447,43 +1494,47 @@ function sanitiseMetadata(metadata) {
|
|
|
1447
1494
|
}
|
|
1448
1495
|
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1449
1496
|
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1450
|
-
var
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1497
|
+
var rateWindows = /* @__PURE__ */ new Map();
|
|
1498
|
+
function admitUnderRateLimit(source, now) {
|
|
1499
|
+
let window = rateWindows.get(source);
|
|
1500
|
+
if (!window) {
|
|
1501
|
+
window = { windowStartedAt: 0, windowCount: 0, windowDroppedCount: 0 };
|
|
1502
|
+
rateWindows.set(source, window);
|
|
1503
|
+
}
|
|
1504
|
+
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1505
|
+
if (window.windowDroppedCount > 0) {
|
|
1456
1506
|
console.error(
|
|
1457
|
-
`[runner-activity-telemetry] rate cap reached: dropped ${windowDroppedCount} ${windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min)`
|
|
1507
|
+
`[runner-activity-telemetry] rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min) for source "${source}"`
|
|
1458
1508
|
);
|
|
1459
1509
|
}
|
|
1460
|
-
windowStartedAt = now;
|
|
1461
|
-
windowCount = 0;
|
|
1462
|
-
windowDroppedCount = 0;
|
|
1510
|
+
window.windowStartedAt = now;
|
|
1511
|
+
window.windowCount = 0;
|
|
1512
|
+
window.windowDroppedCount = 0;
|
|
1463
1513
|
}
|
|
1464
|
-
if (windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1465
|
-
windowDroppedCount++;
|
|
1466
|
-
if (windowDroppedCount === 1) {
|
|
1514
|
+
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1515
|
+
window.windowDroppedCount++;
|
|
1516
|
+
if (window.windowDroppedCount === 1) {
|
|
1467
1517
|
console.error(
|
|
1468
|
-
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window`
|
|
1518
|
+
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window for source "${source}"`
|
|
1469
1519
|
);
|
|
1470
1520
|
}
|
|
1471
1521
|
return false;
|
|
1472
1522
|
}
|
|
1473
|
-
windowCount++;
|
|
1523
|
+
window.windowCount++;
|
|
1474
1524
|
return true;
|
|
1475
1525
|
}
|
|
1476
1526
|
function forwardRunnerActivity(entry, context) {
|
|
1477
1527
|
try {
|
|
1478
1528
|
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1479
1529
|
if (!context.agentId || !context.authHeader) return;
|
|
1480
|
-
|
|
1530
|
+
const source = entry.source ?? "cli.run";
|
|
1531
|
+
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1481
1532
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1482
1533
|
const message = truncate(redact(rawMessage));
|
|
1483
1534
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1484
1535
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1485
1536
|
message,
|
|
1486
|
-
metadata: { ...sanitiseMetadata(entry.metadata), source
|
|
1537
|
+
metadata: { ...sanitiseMetadata(entry.metadata), source },
|
|
1487
1538
|
agentId: context.agentId
|
|
1488
1539
|
});
|
|
1489
1540
|
} catch (err) {
|
|
@@ -1494,8 +1545,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1494
1545
|
}
|
|
1495
1546
|
|
|
1496
1547
|
// src/lib/opencode/session-db-recovery-report.ts
|
|
1497
|
-
import { readFileSync as readFileSync2, unlinkSync } from "fs";
|
|
1498
|
-
import { join as join2 } from "path";
|
|
1548
|
+
import { readFileSync as readFileSync2, unlinkSync } from "node:fs";
|
|
1549
|
+
import { join as join2 } from "node:path";
|
|
1499
1550
|
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1500
1551
|
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1501
1552
|
return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
@@ -1524,7 +1575,14 @@ function drainSessionDbRecoveryReport({
|
|
|
1524
1575
|
skippedLines++;
|
|
1525
1576
|
return [];
|
|
1526
1577
|
}
|
|
1527
|
-
return [
|
|
1578
|
+
return [
|
|
1579
|
+
{
|
|
1580
|
+
...value,
|
|
1581
|
+
provenance_reason: value.provenance_reason ?? null,
|
|
1582
|
+
provenance_migration_delta: value.provenance_migration_delta ?? null,
|
|
1583
|
+
replication_suspended: value.replication_suspended ?? false
|
|
1584
|
+
}
|
|
1585
|
+
];
|
|
1528
1586
|
} catch (error2) {
|
|
1529
1587
|
skippedLines++;
|
|
1530
1588
|
console.error(
|
|
@@ -1549,12 +1607,39 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1549
1607
|
const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
|
|
1550
1608
|
if (!level) return null;
|
|
1551
1609
|
const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
|
|
1610
|
+
const replication = record.replication_suspended ? " This start is not backing up its new session history; restart after fixing the cause." : "";
|
|
1611
|
+
const giveupMessage = (() => {
|
|
1612
|
+
switch (record.reason) {
|
|
1613
|
+
case "restore_deadline_exceeded":
|
|
1614
|
+
return "The restore did not finish before this startup deadline. Earlier sessions are unavailable for this start; collect the SESSION-DB-RESTORE-TRUNCATED boot log before retrying.";
|
|
1615
|
+
case "restore_tool_unusable":
|
|
1616
|
+
case "classification_unrecognised":
|
|
1617
|
+
return "The restore or classification tool produced an unsupported result. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1618
|
+
case "synchroniser_config_unevaluable":
|
|
1619
|
+
case "synchroniser_config_incomplete":
|
|
1620
|
+
return "The installed hook and synchroniser bundle could not agree on configuration. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1621
|
+
case "synchroniser_config_unresolved":
|
|
1622
|
+
return "The synchroniser configuration could not be resolved. Inspect the preceding boot error and repair the identified runner image, bundle, or local environment before restarting.";
|
|
1623
|
+
case "litestream_config_unavailable":
|
|
1624
|
+
return "The local Litestream configuration could not be generated or written. Inspect the preceding boot error and repair the runner image, bundle, local environment, or write permissions before restarting.";
|
|
1625
|
+
case "classification_fatal":
|
|
1626
|
+
return "Session history could not be restored because backup setup could not be established. Inspect the named boot marker or log and repair the identified backup configuration or permissions.";
|
|
1627
|
+
default:
|
|
1628
|
+
return null;
|
|
1629
|
+
}
|
|
1630
|
+
})();
|
|
1631
|
+
if (giveupMessage)
|
|
1632
|
+
return {
|
|
1633
|
+
level,
|
|
1634
|
+
metadata: withoutContractFields(record),
|
|
1635
|
+
message: `${giveupMessage}${replication}`
|
|
1636
|
+
};
|
|
1552
1637
|
switch (record.outcome) {
|
|
1553
1638
|
case "fresh_session_db":
|
|
1554
1639
|
return {
|
|
1555
1640
|
level,
|
|
1556
1641
|
metadata: withoutContractFields(record),
|
|
1557
|
-
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history
|
|
1642
|
+
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.${replication}`
|
|
1558
1643
|
};
|
|
1559
1644
|
case "restore_retried":
|
|
1560
1645
|
return {
|
|
@@ -1592,7 +1677,7 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1592
1677
|
return {
|
|
1593
1678
|
level,
|
|
1594
1679
|
metadata: withoutContractFields(record),
|
|
1595
|
-
message:
|
|
1680
|
+
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1596
1681
|
};
|
|
1597
1682
|
case "session_db_boot_refused":
|
|
1598
1683
|
return {
|
|
@@ -1600,6 +1685,12 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1600
1685
|
metadata: withoutContractFields(record),
|
|
1601
1686
|
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1602
1687
|
};
|
|
1688
|
+
case "schema_provenance_mismatch":
|
|
1689
|
+
return {
|
|
1690
|
+
level,
|
|
1691
|
+
metadata: withoutContractFields(record),
|
|
1692
|
+
message: `Session database schema provenance mismatch for ${record.dbPath ?? record.db_path ?? "unknown"}: recorded version=${record.recorded_version ?? "unknown"}, current version=${record.current_version ?? "unknown"}, reason=${record.provenance_reason ?? "unknown"}, migration delta=${record.provenance_migration_delta ?? "unknown"}. Inspect the session database and runner backup before continuing.`
|
|
1693
|
+
};
|
|
1603
1694
|
default:
|
|
1604
1695
|
return null;
|
|
1605
1696
|
}
|
|
@@ -1614,7 +1705,8 @@ var OUTCOMES = /* @__PURE__ */ new Set([
|
|
|
1614
1705
|
"fresh_session_db",
|
|
1615
1706
|
"history_rolled_back",
|
|
1616
1707
|
"restore_misconfigured",
|
|
1617
|
-
"session_db_boot_refused"
|
|
1708
|
+
"session_db_boot_refused",
|
|
1709
|
+
"schema_provenance_mismatch"
|
|
1618
1710
|
]);
|
|
1619
1711
|
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1620
1712
|
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
@@ -1632,7 +1724,7 @@ var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"]
|
|
|
1632
1724
|
function isSessionDbRecoveryRecord(value) {
|
|
1633
1725
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1634
1726
|
const record = value;
|
|
1635
|
-
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1727
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && (record.provenance_reason === void 0 || record.provenance_reason === null || typeof record.provenance_reason === "string") && (record.provenance_migration_delta === void 0 || record.provenance_migration_delta === null || Number.isInteger(record.provenance_migration_delta)) && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1636
1728
|
(field) => record[field] === null || typeof record[field] === "string"
|
|
1637
1729
|
);
|
|
1638
1730
|
}
|
|
@@ -1661,11 +1753,667 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1661
1753
|
if (health.healthy) {
|
|
1662
1754
|
return health;
|
|
1663
1755
|
}
|
|
1664
|
-
await new Promise((
|
|
1756
|
+
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
1665
1757
|
}
|
|
1666
1758
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1667
1759
|
}
|
|
1668
1760
|
|
|
1761
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1762
|
+
import { spawn as spawn2 } from "node:child_process";
|
|
1763
|
+
import { mkdirSync, statSync as statSync2, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
1764
|
+
import { homedir as homedir2 } from "node:os";
|
|
1765
|
+
import { dirname as dirname2, resolve as resolvePath } from "node:path";
|
|
1766
|
+
|
|
1767
|
+
// src/lib/runner-synchroniser.ts
|
|
1768
|
+
import { spawn } from "node:child_process";
|
|
1769
|
+
function appendError(stderr, error2) {
|
|
1770
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1771
|
+
return stderr === "" ? message : `${stderr}
|
|
1772
|
+
${message}`;
|
|
1773
|
+
}
|
|
1774
|
+
function runSynchroniser(args, opts) {
|
|
1775
|
+
return new Promise((resolve4) => {
|
|
1776
|
+
let child;
|
|
1777
|
+
let stdout = "";
|
|
1778
|
+
let stderr = "";
|
|
1779
|
+
let settled = false;
|
|
1780
|
+
const timer = {};
|
|
1781
|
+
let abortListener;
|
|
1782
|
+
let spawnListener;
|
|
1783
|
+
const finish = (result) => {
|
|
1784
|
+
if (settled) return;
|
|
1785
|
+
settled = true;
|
|
1786
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
1787
|
+
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
1788
|
+
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
1789
|
+
resolve4(result);
|
|
1790
|
+
};
|
|
1791
|
+
try {
|
|
1792
|
+
child = spawn("runner-synchroniser", args, {
|
|
1793
|
+
env: opts.env ?? process.env,
|
|
1794
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1795
|
+
});
|
|
1796
|
+
} catch (error2) {
|
|
1797
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1798
|
+
return;
|
|
1799
|
+
}
|
|
1800
|
+
child.stdout?.setEncoding("utf8");
|
|
1801
|
+
child.stdout?.on("data", (chunk) => {
|
|
1802
|
+
stdout += chunk;
|
|
1803
|
+
});
|
|
1804
|
+
child.stderr?.setEncoding("utf8");
|
|
1805
|
+
child.stderr?.on("data", (chunk) => {
|
|
1806
|
+
stderr += chunk;
|
|
1807
|
+
});
|
|
1808
|
+
child.once("error", (error2) => {
|
|
1809
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1810
|
+
});
|
|
1811
|
+
child.once("close", (code) => {
|
|
1812
|
+
finish({ code, stdout, stderr, timedOut: false });
|
|
1813
|
+
});
|
|
1814
|
+
if (opts.signal) {
|
|
1815
|
+
const killChild = () => {
|
|
1816
|
+
if (child.pid === void 0) {
|
|
1817
|
+
if (!spawnListener) {
|
|
1818
|
+
spawnListener = killChild;
|
|
1819
|
+
child.once("spawn", spawnListener);
|
|
1820
|
+
}
|
|
1821
|
+
return;
|
|
1822
|
+
}
|
|
1823
|
+
child.kill("SIGKILL");
|
|
1824
|
+
};
|
|
1825
|
+
abortListener = killChild;
|
|
1826
|
+
if (opts.signal.aborted) {
|
|
1827
|
+
abortListener();
|
|
1828
|
+
} else {
|
|
1829
|
+
opts.signal.addEventListener("abort", abortListener, { once: true });
|
|
1830
|
+
if (opts.signal.aborted) abortListener();
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
timer.handle = setTimeout(
|
|
1834
|
+
() => {
|
|
1835
|
+
child.kill("SIGKILL");
|
|
1836
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1837
|
+
},
|
|
1838
|
+
Math.max(0, opts.timeoutMs)
|
|
1839
|
+
);
|
|
1840
|
+
});
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1844
|
+
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
1845
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
|
|
1846
|
+
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
1847
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
|
|
1848
|
+
function commandError(result) {
|
|
1849
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
1850
|
+
}
|
|
1851
|
+
function reportRecord(stage, outcome, reason, litestreamExitCode, options) {
|
|
1852
|
+
options.reportRecovery({
|
|
1853
|
+
v: 1,
|
|
1854
|
+
event: "session_db_recovery",
|
|
1855
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1856
|
+
stage,
|
|
1857
|
+
outcome,
|
|
1858
|
+
severity: "error",
|
|
1859
|
+
reason,
|
|
1860
|
+
litestream_exit_code: litestreamExitCode,
|
|
1861
|
+
attempt: null,
|
|
1862
|
+
replica_objects: null,
|
|
1863
|
+
replica_bytes: null,
|
|
1864
|
+
quarantine_destination: null,
|
|
1865
|
+
quarantined_objects: null,
|
|
1866
|
+
quarantine_failed_objects: null,
|
|
1867
|
+
quarantined_bytes: null,
|
|
1868
|
+
verified_restore_point: null,
|
|
1869
|
+
restore_points_tried: null,
|
|
1870
|
+
provenance_reason: null,
|
|
1871
|
+
provenance_migration_delta: null,
|
|
1872
|
+
replication_suspended: stage === "restore"
|
|
1873
|
+
});
|
|
1874
|
+
}
|
|
1875
|
+
function clearMarker(options) {
|
|
1876
|
+
if (!options.noReplicateMarker) return;
|
|
1877
|
+
try {
|
|
1878
|
+
unlinkSync2(options.noReplicateMarker);
|
|
1879
|
+
} catch (error2) {
|
|
1880
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1881
|
+
options.log(
|
|
1882
|
+
`Could not clear the previous session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1883
|
+
"warn"
|
|
1884
|
+
);
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
function markNoReplicate(options, message) {
|
|
1888
|
+
if (options.noReplicateMarker) {
|
|
1889
|
+
try {
|
|
1890
|
+
mkdirSync(dirname2(options.noReplicateMarker), { recursive: true });
|
|
1891
|
+
writeFileSync(options.noReplicateMarker, "");
|
|
1892
|
+
} catch (error2) {
|
|
1893
|
+
options.log(
|
|
1894
|
+
`Could not write session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1895
|
+
"error"
|
|
1896
|
+
);
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
options.log(`SESSION-DB-NO-REPLICATE: ${message}`, "warn");
|
|
1900
|
+
}
|
|
1901
|
+
function discardSessionDbDebris(options) {
|
|
1902
|
+
for (const path of [options.dbPath, `${options.dbPath}-wal`, `${options.dbPath}-shm`]) {
|
|
1903
|
+
try {
|
|
1904
|
+
unlinkSync2(path);
|
|
1905
|
+
} catch (error2) {
|
|
1906
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") continue;
|
|
1907
|
+
options.log(
|
|
1908
|
+
`Could not remove session-DB debris ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1909
|
+
"warn"
|
|
1910
|
+
);
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
function splitDiagnostics(text) {
|
|
1915
|
+
return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
1916
|
+
}
|
|
1917
|
+
function logSynchroniserDiagnostics(result, options) {
|
|
1918
|
+
for (const line of splitDiagnostics(result.stderr)) options.log(line, "warn");
|
|
1919
|
+
}
|
|
1920
|
+
function parseSingleQuotedAssignment(line) {
|
|
1921
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
1922
|
+
if (!match || !match[2].startsWith("'")) return null;
|
|
1923
|
+
const valueSource = match[2];
|
|
1924
|
+
let value = "";
|
|
1925
|
+
for (let index = 1; index < valueSource.length; index++) {
|
|
1926
|
+
const character = valueSource[index];
|
|
1927
|
+
if (character !== "'") {
|
|
1928
|
+
value += character;
|
|
1929
|
+
continue;
|
|
1930
|
+
}
|
|
1931
|
+
if (index === valueSource.length - 1) return [match[1], value];
|
|
1932
|
+
if (valueSource.slice(index + 1, index + 4) !== "\\''") return null;
|
|
1933
|
+
value += "'";
|
|
1934
|
+
index += 3;
|
|
1935
|
+
}
|
|
1936
|
+
return null;
|
|
1937
|
+
}
|
|
1938
|
+
function parseSynchroniserEnv(stdout) {
|
|
1939
|
+
const values = {};
|
|
1940
|
+
for (const line of stdout.split("\n")) {
|
|
1941
|
+
if (line.trim() === "") continue;
|
|
1942
|
+
const assignment = parseSingleQuotedAssignment(line);
|
|
1943
|
+
if (!assignment) return null;
|
|
1944
|
+
values[assignment[0]] = assignment[1];
|
|
1945
|
+
}
|
|
1946
|
+
return values;
|
|
1947
|
+
}
|
|
1948
|
+
function runCommand(command, args, options) {
|
|
1949
|
+
return new Promise((resolve4) => {
|
|
1950
|
+
let child;
|
|
1951
|
+
let stdout = "";
|
|
1952
|
+
let stderr = "";
|
|
1953
|
+
let settled = false;
|
|
1954
|
+
const finish = (result) => {
|
|
1955
|
+
if (settled) return;
|
|
1956
|
+
settled = true;
|
|
1957
|
+
if (timer) clearTimeout(timer);
|
|
1958
|
+
resolve4(result);
|
|
1959
|
+
};
|
|
1960
|
+
try {
|
|
1961
|
+
child = spawn2(command, args, {
|
|
1962
|
+
env: options.env,
|
|
1963
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1964
|
+
});
|
|
1965
|
+
} catch (error2) {
|
|
1966
|
+
resolve4({
|
|
1967
|
+
code: null,
|
|
1968
|
+
stdout,
|
|
1969
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
1970
|
+
timedOut: false
|
|
1971
|
+
});
|
|
1972
|
+
return;
|
|
1973
|
+
}
|
|
1974
|
+
child.stdout?.setEncoding("utf8");
|
|
1975
|
+
child.stdout?.on("data", (chunk) => {
|
|
1976
|
+
stdout += chunk;
|
|
1977
|
+
});
|
|
1978
|
+
child.stderr?.setEncoding("utf8");
|
|
1979
|
+
child.stderr?.on("data", (chunk) => {
|
|
1980
|
+
stderr += chunk;
|
|
1981
|
+
});
|
|
1982
|
+
child.once("error", (error2) => {
|
|
1983
|
+
finish({
|
|
1984
|
+
code: null,
|
|
1985
|
+
stdout,
|
|
1986
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
1987
|
+
${error2.message}`,
|
|
1988
|
+
timedOut: false
|
|
1989
|
+
});
|
|
1990
|
+
});
|
|
1991
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
1992
|
+
const timer = setTimeout(
|
|
1993
|
+
() => {
|
|
1994
|
+
child.kill("SIGKILL");
|
|
1995
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1996
|
+
},
|
|
1997
|
+
Math.max(0, options.timeoutMs)
|
|
1998
|
+
);
|
|
1999
|
+
});
|
|
2000
|
+
}
|
|
2001
|
+
async function ensureLitestreamConfig(options, env) {
|
|
2002
|
+
const configPath = options.litestreamConfig;
|
|
2003
|
+
if (!configPath) {
|
|
2004
|
+
markNoReplicate(options, "no Litestream configuration path was provided");
|
|
2005
|
+
reportRecord(
|
|
2006
|
+
"restore",
|
|
2007
|
+
"restore_misconfigured",
|
|
2008
|
+
"litestream_config_unavailable",
|
|
2009
|
+
null,
|
|
2010
|
+
options
|
|
2011
|
+
);
|
|
2012
|
+
return null;
|
|
2013
|
+
}
|
|
2014
|
+
try {
|
|
2015
|
+
if (statSync2(configPath).size > 0) return configPath;
|
|
2016
|
+
} catch (error2) {
|
|
2017
|
+
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
2018
|
+
options.log(
|
|
2019
|
+
`Could not inspect ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2020
|
+
"warn"
|
|
2021
|
+
);
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
const rendered = await runSynchroniser(["litestream-config"], {
|
|
2025
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2026
|
+
env
|
|
2027
|
+
});
|
|
2028
|
+
logSynchroniserDiagnostics(rendered, options);
|
|
2029
|
+
if (rendered.timedOut || rendered.code !== 0) {
|
|
2030
|
+
options.log(
|
|
2031
|
+
`Could not generate ${configPath}: runner-synchroniser litestream-config failed (${commandError(rendered)})`,
|
|
2032
|
+
"error"
|
|
2033
|
+
);
|
|
2034
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2035
|
+
reportRecord(
|
|
2036
|
+
"restore",
|
|
2037
|
+
"restore_misconfigured",
|
|
2038
|
+
"litestream_config_unavailable",
|
|
2039
|
+
null,
|
|
2040
|
+
options
|
|
2041
|
+
);
|
|
2042
|
+
return null;
|
|
2043
|
+
}
|
|
2044
|
+
try {
|
|
2045
|
+
mkdirSync(dirname2(configPath), { recursive: true });
|
|
2046
|
+
writeFileSync(configPath, rendered.stdout);
|
|
2047
|
+
} catch (error2) {
|
|
2048
|
+
options.log(
|
|
2049
|
+
`Could not write ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2050
|
+
"error"
|
|
2051
|
+
);
|
|
2052
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2053
|
+
reportRecord(
|
|
2054
|
+
"restore",
|
|
2055
|
+
"restore_misconfigured",
|
|
2056
|
+
"litestream_config_unavailable",
|
|
2057
|
+
null,
|
|
2058
|
+
options
|
|
2059
|
+
);
|
|
2060
|
+
return null;
|
|
2061
|
+
}
|
|
2062
|
+
const version2 = await runCommand("litestream", ["version"], {
|
|
2063
|
+
env,
|
|
2064
|
+
timeoutMs: 1e4
|
|
2065
|
+
});
|
|
2066
|
+
const litestreamVersion = version2.code === 0 ? version2.stdout.trim() || "unknown" : "unknown";
|
|
2067
|
+
const regionEmpty = !/^\s*region:\s*\S+/m.test(rendered.stdout);
|
|
2068
|
+
options.log(
|
|
2069
|
+
`litestream ${litestreamVersion}; AWS_REGION=${env.AWS_REGION ?? "<unset>"} AWS_DEFAULT_REGION=${env.AWS_DEFAULT_REGION ?? "<unset>"}; rendered litestream.yml region empty: ${regionEmpty ? "yes" : "no"}`
|
|
2070
|
+
);
|
|
2071
|
+
return configPath;
|
|
2072
|
+
}
|
|
2073
|
+
function restoreGiveUp(options, reason, message, litestreamExitCode, outcome = "fresh_session_db") {
|
|
2074
|
+
discardSessionDbDebris(options);
|
|
2075
|
+
markNoReplicate(options, message);
|
|
2076
|
+
reportRecord("restore", outcome, reason, litestreamExitCode, options);
|
|
2077
|
+
}
|
|
2078
|
+
async function restoreSessionDb(options, configPath, env) {
|
|
2079
|
+
const restored = await runCommand(
|
|
2080
|
+
"litestream",
|
|
2081
|
+
["restore", "-config", configPath, "-if-db-not-exists", "-if-replica-exists", options.dbPath],
|
|
2082
|
+
{ env, timeoutMs: SESSION_DB_RESTORE_TIMEOUT_MS }
|
|
2083
|
+
);
|
|
2084
|
+
for (const line of splitDiagnostics(restored.stderr)) options.log(line, "warn");
|
|
2085
|
+
if (restored.timedOut || restored.code === 124 || restored.code === 137) {
|
|
2086
|
+
restoreGiveUp(
|
|
2087
|
+
options,
|
|
2088
|
+
"restore_deadline_exceeded",
|
|
2089
|
+
`SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within ${SESSION_DB_RESTORE_TIMEOUT_MS}ms; opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2090
|
+
restored.code ?? 124
|
|
2091
|
+
);
|
|
2092
|
+
return;
|
|
2093
|
+
}
|
|
2094
|
+
if (restored.code === null || restored.code === 125 || restored.code === 126 || restored.code === 127) {
|
|
2095
|
+
options.log(`litestream restore could not run (${commandError(restored)})`, "error");
|
|
2096
|
+
restoreGiveUp(
|
|
2097
|
+
options,
|
|
2098
|
+
"restore_tool_unusable",
|
|
2099
|
+
`restore tool is broken (${commandError(restored)}); opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2100
|
+
restored.code
|
|
2101
|
+
);
|
|
2102
|
+
return;
|
|
2103
|
+
}
|
|
2104
|
+
const classified = await runSynchroniser(
|
|
2105
|
+
[
|
|
2106
|
+
"session-db-classify",
|
|
2107
|
+
String(restored.code ?? 1),
|
|
2108
|
+
"1",
|
|
2109
|
+
"--on-unusable-replica=leave",
|
|
2110
|
+
"--fresh-db-fallback"
|
|
2111
|
+
],
|
|
2112
|
+
{ timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS, env }
|
|
2113
|
+
);
|
|
2114
|
+
logSynchroniserDiagnostics(classified, options);
|
|
2115
|
+
const classifyCode = classified.code;
|
|
2116
|
+
switch (classifyCode) {
|
|
2117
|
+
case 0:
|
|
2118
|
+
return;
|
|
2119
|
+
case 31:
|
|
2120
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2121
|
+
options.log(
|
|
2122
|
+
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
2123
|
+
"warn"
|
|
2124
|
+
);
|
|
2125
|
+
return;
|
|
2126
|
+
case 32:
|
|
2127
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2128
|
+
discardSessionDbDebris(options);
|
|
2129
|
+
markNoReplicate(
|
|
2130
|
+
options,
|
|
2131
|
+
"session-db-classify asked for another restore attempt (32), but this boot has budget for only one; treating it as a give-up rather than retrying"
|
|
2132
|
+
);
|
|
2133
|
+
return;
|
|
2134
|
+
case 30:
|
|
2135
|
+
restoreGiveUp(
|
|
2136
|
+
options,
|
|
2137
|
+
"classification_fatal",
|
|
2138
|
+
"session-db-classify returned fatal (30); see the FATAL message above",
|
|
2139
|
+
restored.code,
|
|
2140
|
+
"restore_misconfigured"
|
|
2141
|
+
);
|
|
2142
|
+
return;
|
|
2143
|
+
default:
|
|
2144
|
+
restoreGiveUp(
|
|
2145
|
+
options,
|
|
2146
|
+
"classification_unrecognised",
|
|
2147
|
+
`session-db-classify exited ${classifyCode ?? "null"}, which is none of its documented answers`,
|
|
2148
|
+
restored.code
|
|
2149
|
+
);
|
|
2150
|
+
}
|
|
2151
|
+
}
|
|
2152
|
+
async function verifySessionDb(options, configPath, env) {
|
|
2153
|
+
if (!options.noReplicateMarker || !fileExists(options.noReplicateMarker)) {
|
|
2154
|
+
const result = await runSynchroniser(["session-db-verify", configPath], {
|
|
2155
|
+
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
2156
|
+
env: {
|
|
2157
|
+
...env,
|
|
2158
|
+
// The synchroniser reads this value in SECONDS. Keep this at 120, not
|
|
2159
|
+
// 120_000, so the walkback gives up before the outer process bound.
|
|
2160
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
2161
|
+
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
2162
|
+
)
|
|
2163
|
+
}
|
|
2164
|
+
});
|
|
2165
|
+
logSynchroniserDiagnostics(result, options);
|
|
2166
|
+
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
2167
|
+
options.log(
|
|
2168
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
2169
|
+
"warn"
|
|
2170
|
+
);
|
|
2171
|
+
return false;
|
|
2172
|
+
}
|
|
2173
|
+
if (result.code === 34) {
|
|
2174
|
+
reportRecord(
|
|
2175
|
+
"verify",
|
|
2176
|
+
"session_db_boot_refused",
|
|
2177
|
+
result.stderr.includes("SESSION-DB-LOCAL-DISCARD-FAILED") ? "local_discard_failed" : "replica_separation_unproven",
|
|
2178
|
+
null,
|
|
2179
|
+
options
|
|
2180
|
+
);
|
|
2181
|
+
return true;
|
|
2182
|
+
}
|
|
2183
|
+
if (result.code === 33) {
|
|
2184
|
+
options.log(
|
|
2185
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
2186
|
+
"warn"
|
|
2187
|
+
);
|
|
2188
|
+
return false;
|
|
2189
|
+
}
|
|
2190
|
+
if (result.code !== 0) {
|
|
2191
|
+
options.log(
|
|
2192
|
+
`SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${result.code ?? "null"}, which is none of its documented answers; continuing with the restored opencode.db as-is`,
|
|
2193
|
+
"warn"
|
|
2194
|
+
);
|
|
2195
|
+
}
|
|
2196
|
+
return false;
|
|
2197
|
+
}
|
|
2198
|
+
options.log(
|
|
2199
|
+
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
2200
|
+
"debug"
|
|
2201
|
+
);
|
|
2202
|
+
return false;
|
|
2203
|
+
}
|
|
2204
|
+
function fileExists(path) {
|
|
2205
|
+
try {
|
|
2206
|
+
statSync2(path);
|
|
2207
|
+
return true;
|
|
2208
|
+
} catch (error2) {
|
|
2209
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
2210
|
+
return true;
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
async function restoreAndVerifySessionDb(options) {
|
|
2214
|
+
const env = options.env ?? process.env;
|
|
2215
|
+
clearMarker(options);
|
|
2216
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2217
|
+
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
2218
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2219
|
+
env
|
|
2220
|
+
});
|
|
2221
|
+
logSynchroniserDiagnostics(synchroniserEnv, options);
|
|
2222
|
+
if (synchroniserEnv.timedOut || synchroniserEnv.code !== 0) {
|
|
2223
|
+
options.log(
|
|
2224
|
+
`runner-synchroniser env could not resolve the session-DB configuration (${commandError(synchroniserEnv)})`,
|
|
2225
|
+
"error"
|
|
2226
|
+
);
|
|
2227
|
+
markNoReplicate(
|
|
2228
|
+
options,
|
|
2229
|
+
"could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
2230
|
+
);
|
|
2231
|
+
reportRecord(
|
|
2232
|
+
"restore",
|
|
2233
|
+
"restore_misconfigured",
|
|
2234
|
+
"synchroniser_config_unresolved",
|
|
2235
|
+
null,
|
|
2236
|
+
options
|
|
2237
|
+
);
|
|
2238
|
+
return { verifyFatal: false };
|
|
2239
|
+
}
|
|
2240
|
+
const values = parseSynchroniserEnv(synchroniserEnv.stdout);
|
|
2241
|
+
if (!values) {
|
|
2242
|
+
markNoReplicate(
|
|
2243
|
+
options,
|
|
2244
|
+
"the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
2245
|
+
);
|
|
2246
|
+
reportRecord(
|
|
2247
|
+
"restore",
|
|
2248
|
+
"restore_misconfigured",
|
|
2249
|
+
"synchroniser_config_unevaluable",
|
|
2250
|
+
null,
|
|
2251
|
+
options
|
|
2252
|
+
);
|
|
2253
|
+
return { verifyFatal: false };
|
|
2254
|
+
}
|
|
2255
|
+
const synchroniserDbPath = values.OPENCODE_DB_PATH;
|
|
2256
|
+
if (!synchroniserDbPath) {
|
|
2257
|
+
markNoReplicate(
|
|
2258
|
+
options,
|
|
2259
|
+
"run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
2260
|
+
);
|
|
2261
|
+
reportRecord(
|
|
2262
|
+
"restore",
|
|
2263
|
+
"restore_misconfigured",
|
|
2264
|
+
"synchroniser_config_incomplete",
|
|
2265
|
+
null,
|
|
2266
|
+
options
|
|
2267
|
+
);
|
|
2268
|
+
return { verifyFatal: false };
|
|
2269
|
+
}
|
|
2270
|
+
if (resolvePath(synchroniserDbPath) !== resolvePath(options.dbPath)) {
|
|
2271
|
+
options.log(
|
|
2272
|
+
`runner-synchroniser reported OPENCODE_DB_PATH=${synchroniserDbPath}, but OpenCode uses ${options.dbPath}; continuing with OpenCode's session-DB path`,
|
|
2273
|
+
"warn"
|
|
2274
|
+
);
|
|
2275
|
+
}
|
|
2276
|
+
if (!values.PERSISTENCE_BUCKET) {
|
|
2277
|
+
options.log(
|
|
2278
|
+
"SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated.",
|
|
2279
|
+
"warn"
|
|
2280
|
+
);
|
|
2281
|
+
return { verifyFatal: false };
|
|
2282
|
+
}
|
|
2283
|
+
const configPath = await ensureLitestreamConfig(options, env);
|
|
2284
|
+
if (!configPath) return { verifyFatal: false };
|
|
2285
|
+
await restoreSessionDb(options, configPath, env);
|
|
2286
|
+
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
2287
|
+
return { verifyFatal: false };
|
|
2288
|
+
}
|
|
2289
|
+
return { verifyFatal: await verifySessionDb(options, configPath, env) };
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2292
|
+
// src/lib/opencode/session-db-provenance.ts
|
|
2293
|
+
import { createRequire } from "node:module";
|
|
2294
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
|
|
2295
|
+
import { dirname as dirname3, join as join3 } from "node:path";
|
|
2296
|
+
var require2 = createRequire(import.meta.url);
|
|
2297
|
+
function readSessionDbMigrationIds(dbPath) {
|
|
2298
|
+
let db;
|
|
2299
|
+
try {
|
|
2300
|
+
const { DatabaseSync } = require2("node:sqlite");
|
|
2301
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
2302
|
+
const columns = db.prepare("PRAGMA table_info(migration)").all();
|
|
2303
|
+
const hasExpectedShape = columns.length === 2 && columns.some(
|
|
2304
|
+
(column) => column.name === "id" && typeof column.type === "string" && column.type.toUpperCase() === "TEXT" && column.pk === 1
|
|
2305
|
+
) && columns.some(
|
|
2306
|
+
(column) => column.name === "time_completed" && typeof column.type === "string" && column.type.toUpperCase() === "INTEGER" && column.notnull === 1 && column.pk === 0
|
|
2307
|
+
);
|
|
2308
|
+
if (!hasExpectedShape) {
|
|
2309
|
+
console.warn(`[readSessionDbMigrationIds] unsupported migration table shape in ${dbPath}`);
|
|
2310
|
+
return null;
|
|
2311
|
+
}
|
|
2312
|
+
const rows = db.prepare("SELECT id FROM migration ORDER BY id").all();
|
|
2313
|
+
if (rows.some((row) => typeof row.id !== "string")) return null;
|
|
2314
|
+
return rows.map((row) => row.id);
|
|
2315
|
+
} catch (error2) {
|
|
2316
|
+
console.warn(
|
|
2317
|
+
`[readSessionDbMigrationIds] could not read migration history from ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2318
|
+
);
|
|
2319
|
+
return null;
|
|
2320
|
+
} finally {
|
|
2321
|
+
try {
|
|
2322
|
+
db?.close();
|
|
2323
|
+
} catch (error2) {
|
|
2324
|
+
console.warn(
|
|
2325
|
+
`[readSessionDbMigrationIds] could not close ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2326
|
+
);
|
|
2327
|
+
}
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
2331
|
+
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
2332
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
2333
|
+
}
|
|
2334
|
+
function loadSessionDbProvenanceState(path) {
|
|
2335
|
+
let value;
|
|
2336
|
+
try {
|
|
2337
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
2338
|
+
} catch (error2) {
|
|
2339
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return {};
|
|
2340
|
+
console.error(
|
|
2341
|
+
`[session-db-provenance] could not read ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2342
|
+
);
|
|
2343
|
+
return {};
|
|
2344
|
+
}
|
|
2345
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2346
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2347
|
+
return {};
|
|
2348
|
+
}
|
|
2349
|
+
const state = {};
|
|
2350
|
+
for (const [dbPath, record] of Object.entries(value)) {
|
|
2351
|
+
if (!isSessionDbProvenanceRecord(record)) {
|
|
2352
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2353
|
+
return {};
|
|
2354
|
+
}
|
|
2355
|
+
state[dbPath] = record;
|
|
2356
|
+
}
|
|
2357
|
+
return state;
|
|
2358
|
+
}
|
|
2359
|
+
function saveSessionDbProvenanceState(path, state) {
|
|
2360
|
+
try {
|
|
2361
|
+
mkdirSync2(dirname3(path), { recursive: true });
|
|
2362
|
+
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
2363
|
+
`, "utf8");
|
|
2364
|
+
} catch (error2) {
|
|
2365
|
+
console.error(
|
|
2366
|
+
`[session-db-provenance] could not write ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2367
|
+
);
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
function evaluateSessionDbProvenance(input) {
|
|
2371
|
+
const { currentVersion, currentIds, previous } = input;
|
|
2372
|
+
if (!previous) return { anomaly: false, reason: null };
|
|
2373
|
+
const current = new Set(currentIds);
|
|
2374
|
+
const prior = new Set(previous.migrationIds);
|
|
2375
|
+
for (const id of prior) {
|
|
2376
|
+
if (!current.has(id)) return { anomaly: true, reason: "migration-history-regressed" };
|
|
2377
|
+
}
|
|
2378
|
+
if (current.size > prior.size && previous.opencodeVersion === currentVersion) {
|
|
2379
|
+
return { anomaly: true, reason: "foreign-version-migrations" };
|
|
2380
|
+
}
|
|
2381
|
+
return { anomaly: false, reason: null };
|
|
2382
|
+
}
|
|
2383
|
+
function checkSessionDbProvenance(input) {
|
|
2384
|
+
const { dbPath, currentVersion, homeDir, env } = input;
|
|
2385
|
+
const path = sessionDbProvenanceStatePath(homeDir, env);
|
|
2386
|
+
const state = loadSessionDbProvenanceState(path);
|
|
2387
|
+
const previous = state[dbPath];
|
|
2388
|
+
const currentIds = readSessionDbMigrationIds(dbPath);
|
|
2389
|
+
if (currentIds === null) {
|
|
2390
|
+
return {
|
|
2391
|
+
anomaly: false,
|
|
2392
|
+
reason: null,
|
|
2393
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2394
|
+
migrationDelta: null
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
const decision = evaluateSessionDbProvenance({ currentVersion, currentIds, previous });
|
|
2398
|
+
const migrationDelta = previous ? new Set(currentIds).size - new Set(previous.migrationIds).size : null;
|
|
2399
|
+
state[dbPath] = {
|
|
2400
|
+
opencodeVersion: currentVersion,
|
|
2401
|
+
migrationIds: currentIds,
|
|
2402
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2403
|
+
};
|
|
2404
|
+
saveSessionDbProvenanceState(path, state);
|
|
2405
|
+
return {
|
|
2406
|
+
...decision,
|
|
2407
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2408
|
+
migrationDelta
|
|
2409
|
+
};
|
|
2410
|
+
}
|
|
2411
|
+
function isSessionDbProvenanceRecord(value) {
|
|
2412
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2413
|
+
const record = value;
|
|
2414
|
+
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
2415
|
+
}
|
|
2416
|
+
|
|
1669
2417
|
// src/lib/opencode/opencode-version-gate.ts
|
|
1670
2418
|
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
1671
2419
|
function isQueueValidatedVersion(version2) {
|
|
@@ -1680,7 +2428,7 @@ function buildOpenCodeVersionWarning(version2) {
|
|
|
1680
2428
|
}
|
|
1681
2429
|
|
|
1682
2430
|
// src/lib/opencode/process.ts
|
|
1683
|
-
import { execSync, spawn } from "child_process";
|
|
2431
|
+
import { execSync, spawn as spawn3 } from "child_process";
|
|
1684
2432
|
|
|
1685
2433
|
// src/lib/process-stop.ts
|
|
1686
2434
|
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
@@ -1690,7 +2438,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1690
2438
|
if (child.exitCode !== null || child.signalCode !== null) {
|
|
1691
2439
|
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
1692
2440
|
}
|
|
1693
|
-
return new Promise((
|
|
2441
|
+
return new Promise((resolve4, reject) => {
|
|
1694
2442
|
let forced = false;
|
|
1695
2443
|
let settled = false;
|
|
1696
2444
|
const timer = setTimeout(() => {
|
|
@@ -1710,7 +2458,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1710
2458
|
settled = true;
|
|
1711
2459
|
clearTimeout(timer);
|
|
1712
2460
|
child.removeListener("exit", onExit);
|
|
1713
|
-
|
|
2461
|
+
resolve4(result);
|
|
1714
2462
|
};
|
|
1715
2463
|
const fail = (error2) => {
|
|
1716
2464
|
if (settled) return;
|
|
@@ -1738,6 +2486,17 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1738
2486
|
|
|
1739
2487
|
// src/lib/opencode/process.ts
|
|
1740
2488
|
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
2489
|
+
var VALID_OPENCODE_LOG_LEVELS = /* @__PURE__ */ new Set(["DEBUG", "INFO", "WARN", "ERROR"]);
|
|
2490
|
+
function resolveOpenCodeLogLevel(env) {
|
|
2491
|
+
const raw = env.OPENCODE_LOG_LEVEL;
|
|
2492
|
+
if (!raw) return "INFO";
|
|
2493
|
+
const upper = raw.toUpperCase();
|
|
2494
|
+
if (VALID_OPENCODE_LOG_LEVELS.has(upper)) return upper;
|
|
2495
|
+
console.warn(
|
|
2496
|
+
`startOpenCode: ignoring invalid OPENCODE_LOG_LEVEL "${raw}" (expected DEBUG|INFO|WARN|ERROR) \u2014 using INFO`
|
|
2497
|
+
);
|
|
2498
|
+
return "INFO";
|
|
2499
|
+
}
|
|
1741
2500
|
function getProcessCwd(pid) {
|
|
1742
2501
|
const platform = process.platform;
|
|
1743
2502
|
try {
|
|
@@ -1786,14 +2545,14 @@ function findAvailablePort(startPort, maxAttempts = 10) {
|
|
|
1786
2545
|
}
|
|
1787
2546
|
return null;
|
|
1788
2547
|
}
|
|
1789
|
-
function
|
|
2548
|
+
function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
1790
2549
|
const instances = [];
|
|
1791
2550
|
try {
|
|
1792
2551
|
const platform = process.platform;
|
|
1793
2552
|
if (platform === "darwin" || platform === "linux") {
|
|
1794
2553
|
let pids = [];
|
|
1795
2554
|
try {
|
|
1796
|
-
const pgrepOutput = execSync(
|
|
2555
|
+
const pgrepOutput = execSync(`pgrep -f "${pgrepPattern}"`, {
|
|
1797
2556
|
encoding: "utf-8",
|
|
1798
2557
|
stdio: ["pipe", "pipe", "pipe"]
|
|
1799
2558
|
}).trim();
|
|
@@ -1802,7 +2561,7 @@ function findOpenCodeProcesses() {
|
|
|
1802
2561
|
}
|
|
1803
2562
|
} catch {
|
|
1804
2563
|
try {
|
|
1805
|
-
const psOutput = execSync(
|
|
2564
|
+
const psOutput = execSync(`ps aux | grep -E "${psPattern}" | grep -v grep`, {
|
|
1806
2565
|
encoding: "utf-8",
|
|
1807
2566
|
stdio: ["pipe", "pipe", "pipe"]
|
|
1808
2567
|
}).trim();
|
|
@@ -1848,6 +2607,9 @@ function findOpenCodeProcesses() {
|
|
|
1848
2607
|
}
|
|
1849
2608
|
return instances;
|
|
1850
2609
|
}
|
|
2610
|
+
function findOpenCodeProcesses() {
|
|
2611
|
+
return findProcessesByPattern("opencode serve|opencode-serve", "opencode (serve|--port)");
|
|
2612
|
+
}
|
|
1851
2613
|
async function scanPortsForOpenCode() {
|
|
1852
2614
|
const instances = [];
|
|
1853
2615
|
const checks = OPENCODE_PORT_RANGE.map(async (port) => {
|
|
@@ -1892,18 +2654,27 @@ async function findHealthyOpenCodeInstances() {
|
|
|
1892
2654
|
}
|
|
1893
2655
|
return healthy;
|
|
1894
2656
|
}
|
|
1895
|
-
async function startOpenCode(port) {
|
|
2657
|
+
async function startOpenCode(port, options = {}) {
|
|
1896
2658
|
let command = "opencode";
|
|
1897
|
-
|
|
2659
|
+
const printLogs = options.inheritStdio ? ["--print-logs", "--log-level", resolveOpenCodeLogLevel(process.env)] : [];
|
|
2660
|
+
let args = ["serve", "--port", port.toString(), "--hostname", "127.0.0.1", ...printLogs];
|
|
1898
2661
|
try {
|
|
1899
2662
|
execSync("which opencode", { stdio: "ignore" });
|
|
1900
2663
|
} catch {
|
|
1901
2664
|
command = "npx";
|
|
1902
|
-
args = [
|
|
1903
|
-
|
|
1904
|
-
|
|
2665
|
+
args = [
|
|
2666
|
+
"opencode",
|
|
2667
|
+
"serve",
|
|
2668
|
+
"--port",
|
|
2669
|
+
port.toString(),
|
|
2670
|
+
"--hostname",
|
|
2671
|
+
"127.0.0.1",
|
|
2672
|
+
...printLogs
|
|
2673
|
+
];
|
|
2674
|
+
}
|
|
2675
|
+
const child = spawn3(command, args, {
|
|
1905
2676
|
detached: true,
|
|
1906
|
-
stdio: "ignore",
|
|
2677
|
+
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
1907
2678
|
cwd: process.cwd()
|
|
1908
2679
|
});
|
|
1909
2680
|
return child;
|
|
@@ -1942,6 +2713,19 @@ function isOpenCodeInstalled() {
|
|
|
1942
2713
|
return false;
|
|
1943
2714
|
}
|
|
1944
2715
|
}
|
|
2716
|
+
function isOpenCode2Installed() {
|
|
2717
|
+
try {
|
|
2718
|
+
const platform = process.platform;
|
|
2719
|
+
if (platform === "win32") {
|
|
2720
|
+
execSync2("where opencode2", { stdio: "ignore" });
|
|
2721
|
+
} else {
|
|
2722
|
+
execSync2("which opencode2", { stdio: "ignore" });
|
|
2723
|
+
}
|
|
2724
|
+
return true;
|
|
2725
|
+
} catch {
|
|
2726
|
+
return false;
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
1945
2729
|
async function promptOpenCodeInstall(interactive) {
|
|
1946
2730
|
if (!interactive) {
|
|
1947
2731
|
console.log(
|
|
@@ -1951,7 +2735,11 @@ async function promptOpenCodeInstall(interactive) {
|
|
|
1951
2735
|
install_url: OPENCODE_INSTALL_URL,
|
|
1952
2736
|
install_commands: {
|
|
1953
2737
|
npm: "npm install -g opencode-ai",
|
|
1954
|
-
curl: "curl -fsSL https://opencode.ai/install.sh | sh"
|
|
2738
|
+
curl: "curl -fsSL https://opencode.ai/install.sh | sh",
|
|
2739
|
+
v2: {
|
|
2740
|
+
npm: "npm install -g @opencode-ai/cli@beta",
|
|
2741
|
+
curl: "curl -fsSL https://opencode.ai/v2/install | bash"
|
|
2742
|
+
}
|
|
1955
2743
|
}
|
|
1956
2744
|
})
|
|
1957
2745
|
);
|
|
@@ -2208,6 +2996,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
2208
2996
|
return data.id;
|
|
2209
2997
|
}
|
|
2210
2998
|
async function getModelAttachmentCapability(port, model) {
|
|
2999
|
+
const { model: baseModel } = splitModelVariant(model);
|
|
2211
3000
|
try {
|
|
2212
3001
|
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2213
3002
|
if (!res.ok) {
|
|
@@ -2224,9 +3013,9 @@ async function getModelAttachmentCapability(port, model) {
|
|
|
2224
3013
|
);
|
|
2225
3014
|
return null;
|
|
2226
3015
|
}
|
|
2227
|
-
const slash =
|
|
2228
|
-
const providerId = slash > 0 ?
|
|
2229
|
-
let modelId = slash > 0 ?
|
|
3016
|
+
const slash = baseModel ? baseModel.indexOf("/") : -1;
|
|
3017
|
+
const providerId = slash > 0 ? baseModel.slice(0, slash) : void 0;
|
|
3018
|
+
let modelId = slash > 0 ? baseModel.slice(slash + 1) : void 0;
|
|
2230
3019
|
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
2231
3020
|
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
2232
3021
|
if (!provider && !providerId) {
|
|
@@ -2306,6 +3095,29 @@ async function buildFileParts(attachments, capable) {
|
|
|
2306
3095
|
}
|
|
2307
3096
|
return { parts, outcomes, capabilityUnknown };
|
|
2308
3097
|
}
|
|
3098
|
+
function splitModelVariant(raw) {
|
|
3099
|
+
const value = raw?.trim();
|
|
3100
|
+
if (!value) return {};
|
|
3101
|
+
const hashIndex = value.indexOf("#");
|
|
3102
|
+
if (hashIndex === -1) return { model: value };
|
|
3103
|
+
const model = value.slice(0, hashIndex).trim() || void 0;
|
|
3104
|
+
const variant = value.slice(hashIndex + 1).trim() || void 0;
|
|
3105
|
+
return { model, variant };
|
|
3106
|
+
}
|
|
3107
|
+
function applyModelOptions(body, options) {
|
|
3108
|
+
if (options?.agent) body.agent = options.agent;
|
|
3109
|
+
const { model, variant } = splitModelVariant(options?.model);
|
|
3110
|
+
if (model) {
|
|
3111
|
+
const slashIndex = model.indexOf("/");
|
|
3112
|
+
if (slashIndex !== -1) {
|
|
3113
|
+
body.model = {
|
|
3114
|
+
providerID: model.substring(0, slashIndex),
|
|
3115
|
+
modelID: model.substring(slashIndex + 1)
|
|
3116
|
+
};
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
if (variant) body.variant = variant;
|
|
3120
|
+
}
|
|
2309
3121
|
function messageText(m) {
|
|
2310
3122
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2311
3123
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
@@ -2330,18 +3142,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2330
3142
|
const body = {
|
|
2331
3143
|
parts
|
|
2332
3144
|
};
|
|
2333
|
-
|
|
2334
|
-
body.agent = options.agent;
|
|
2335
|
-
}
|
|
2336
|
-
if (options?.model) {
|
|
2337
|
-
const slashIndex = options.model.indexOf("/");
|
|
2338
|
-
if (slashIndex !== -1) {
|
|
2339
|
-
body.model = {
|
|
2340
|
-
providerID: options.model.substring(0, slashIndex),
|
|
2341
|
-
modelID: options.model.substring(slashIndex + 1)
|
|
2342
|
-
};
|
|
2343
|
-
}
|
|
2344
|
-
}
|
|
3145
|
+
applyModelOptions(body, options);
|
|
2345
3146
|
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
2346
3147
|
method: "POST",
|
|
2347
3148
|
headers: { "Content-Type": "application/json" },
|
|
@@ -2349,7 +3150,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2349
3150
|
});
|
|
2350
3151
|
if (res.status < 200 || res.status >= 300) {
|
|
2351
3152
|
const text = await res.text().catch(() => "");
|
|
2352
|
-
|
|
3153
|
+
const { variant } = splitModelVariant(options?.model);
|
|
3154
|
+
throw new Error(
|
|
3155
|
+
`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}${variant ? ` (variant: ${variant})` : ""}`
|
|
3156
|
+
);
|
|
2353
3157
|
}
|
|
2354
3158
|
const READ_BACK_ATTEMPTS = 5;
|
|
2355
3159
|
const READ_BACK_DELAY_MS = 150;
|
|
@@ -2373,7 +3177,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2373
3177
|
}
|
|
2374
3178
|
}
|
|
2375
3179
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2376
|
-
await new Promise((
|
|
3180
|
+
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
2377
3181
|
}
|
|
2378
3182
|
}
|
|
2379
3183
|
return null;
|
|
@@ -2419,6 +3223,44 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2419
3223
|
}
|
|
2420
3224
|
return lastOk ?? last;
|
|
2421
3225
|
}
|
|
3226
|
+
function collectSubagentSessions(messages, userMessageId) {
|
|
3227
|
+
if (!messages || messages.length === 0) return [];
|
|
3228
|
+
const byParent = messages.filter(
|
|
3229
|
+
(message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
|
|
3230
|
+
);
|
|
3231
|
+
const assistants = byParent.length > 0 ? byParent : [];
|
|
3232
|
+
if (assistants.length === 0) {
|
|
3233
|
+
const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
|
|
3234
|
+
if (userIndex === -1) return [];
|
|
3235
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
3236
|
+
const message = messages[i];
|
|
3237
|
+
if (roleOf(message) === "user") break;
|
|
3238
|
+
if (roleOf(message) === "assistant") assistants.push(message);
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
const refs = [];
|
|
3242
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3243
|
+
for (const message of assistants) {
|
|
3244
|
+
const parts = Array.isArray(message.parts) ? message.parts : [];
|
|
3245
|
+
for (const part of parts) {
|
|
3246
|
+
if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
|
|
3247
|
+
continue;
|
|
3248
|
+
const state = part.state;
|
|
3249
|
+
if (!state || typeof state !== "object") continue;
|
|
3250
|
+
const metadata = state.metadata;
|
|
3251
|
+
if (!metadata || typeof metadata !== "object") continue;
|
|
3252
|
+
const sessionId = metadata.sessionId;
|
|
3253
|
+
if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
|
|
3254
|
+
seen.add(sessionId);
|
|
3255
|
+
const start = state.time?.start;
|
|
3256
|
+
refs.push({
|
|
3257
|
+
sessionId,
|
|
3258
|
+
startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
|
|
3259
|
+
});
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3262
|
+
return refs;
|
|
3263
|
+
}
|
|
2422
3264
|
function messageUsage(messages, userMessageId) {
|
|
2423
3265
|
if (!messages || messages.length === 0) return null;
|
|
2424
3266
|
const byParentAll = messages.filter(
|
|
@@ -2547,8 +3389,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2547
3389
|
}
|
|
2548
3390
|
return false;
|
|
2549
3391
|
}
|
|
2550
|
-
function
|
|
2551
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3392
|
+
function classifyReplyAuthError(reply) {
|
|
2552
3393
|
const error2 = errorOf(reply);
|
|
2553
3394
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2554
3395
|
const e = error2;
|
|
@@ -2573,6 +3414,32 @@ function messageFailure(messages, userMessageId) {
|
|
|
2573
3414
|
}
|
|
2574
3415
|
return null;
|
|
2575
3416
|
}
|
|
3417
|
+
function messageFailure(messages, userMessageId) {
|
|
3418
|
+
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
3419
|
+
}
|
|
3420
|
+
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
3421
|
+
if (!messages || messages.length === 0) return null;
|
|
3422
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3423
|
+
const message = messages[i];
|
|
3424
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3425
|
+
const created = createdOf(message);
|
|
3426
|
+
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
3427
|
+
const failure = classifyReplyAuthError(message);
|
|
3428
|
+
if (failure) {
|
|
3429
|
+
if (!failure.providerId) return null;
|
|
3430
|
+
return { providerId: failure.providerId, outcome: "failed", failure };
|
|
3431
|
+
}
|
|
3432
|
+
const providerId = message.info?.providerID;
|
|
3433
|
+
if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
|
|
3434
|
+
return { providerId, outcome: "succeeded" };
|
|
3435
|
+
}
|
|
3436
|
+
return null;
|
|
3437
|
+
}
|
|
3438
|
+
return null;
|
|
3439
|
+
}
|
|
3440
|
+
function findSubagentAuthOutcome(messages, sinceMs) {
|
|
3441
|
+
return findLatestSubagentAuthOutcome(messages, sinceMs);
|
|
3442
|
+
}
|
|
2576
3443
|
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2577
3444
|
if (classified != null) return classified;
|
|
2578
3445
|
if (hasConfiguredProvider !== false) return null;
|
|
@@ -2620,6 +3487,24 @@ async function hasAnyConfiguredProvider(port) {
|
|
|
2620
3487
|
return null;
|
|
2621
3488
|
}
|
|
2622
3489
|
}
|
|
3490
|
+
async function reloadProviderCache(port) {
|
|
3491
|
+
try {
|
|
3492
|
+
const res = await timedFetch(`${opencodeBase(port)}/config`, {
|
|
3493
|
+
method: "PATCH",
|
|
3494
|
+
headers: { "Content-Type": "application/json" },
|
|
3495
|
+
body: JSON.stringify({})
|
|
3496
|
+
});
|
|
3497
|
+
if (!res.ok) {
|
|
3498
|
+
console.error(
|
|
3499
|
+
`[reloadProviderCache] PATCH /config returned HTTP ${res.status} (port ${port})`
|
|
3500
|
+
);
|
|
3501
|
+
}
|
|
3502
|
+
} catch (err) {
|
|
3503
|
+
console.error(
|
|
3504
|
+
`[reloadProviderCache] PATCH /config failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
3505
|
+
);
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
2623
3508
|
|
|
2624
3509
|
// src/lib/opencode/session-cleanup.ts
|
|
2625
3510
|
var DURATION_UNIT_MS = {
|
|
@@ -2726,13 +3611,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2726
3611
|
}
|
|
2727
3612
|
|
|
2728
3613
|
// src/lib/opencode/session-db-size.ts
|
|
2729
|
-
import { statSync as
|
|
2730
|
-
import { join as
|
|
3614
|
+
import { statSync as statSync3 } from "node:fs";
|
|
3615
|
+
import { join as join4 } from "node:path";
|
|
2731
3616
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2732
3617
|
function statSessionDbBytes(homeDir) {
|
|
2733
|
-
const dbPath =
|
|
3618
|
+
const dbPath = join4(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2734
3619
|
try {
|
|
2735
|
-
return
|
|
3620
|
+
return statSync3(dbPath).size;
|
|
2736
3621
|
} catch (err) {
|
|
2737
3622
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2738
3623
|
if (!isMissingFile) {
|
|
@@ -2757,12 +3642,99 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
2757
3642
|
return null;
|
|
2758
3643
|
}
|
|
2759
3644
|
|
|
3645
|
+
// src/lib/opencode/log-tail.ts
|
|
3646
|
+
import { statSync as statSync4 } from "node:fs";
|
|
3647
|
+
import { homedir as homedir3 } from "node:os";
|
|
3648
|
+
import { join as join5 } from "node:path";
|
|
3649
|
+
import { open as open2, stat } from "node:fs/promises";
|
|
3650
|
+
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
3651
|
+
function resolveOpenCodeLogPath(homeDir = homedir3(), env = process.env) {
|
|
3652
|
+
const dataDir = env.XDG_DATA_HOME || join5(homeDir, ".local", "share");
|
|
3653
|
+
return join5(dataDir, "opencode", "log", "opencode.log");
|
|
3654
|
+
}
|
|
3655
|
+
function isEnoent(error2) {
|
|
3656
|
+
return error2?.code === "ENOENT";
|
|
3657
|
+
}
|
|
3658
|
+
function reportFailure(operation, logPath, error2) {
|
|
3659
|
+
console.error(
|
|
3660
|
+
`[opencode-log-tail] ${operation} failed for ${logPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3661
|
+
);
|
|
3662
|
+
}
|
|
3663
|
+
function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
3664
|
+
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
3665
|
+
let offset = 0;
|
|
3666
|
+
let inode = null;
|
|
3667
|
+
let baselineReady = true;
|
|
3668
|
+
try {
|
|
3669
|
+
const initial = statSync4(logPath);
|
|
3670
|
+
offset = initial.size;
|
|
3671
|
+
inode = initial.ino;
|
|
3672
|
+
} catch (error2) {
|
|
3673
|
+
if (!isEnoent(error2)) {
|
|
3674
|
+
reportFailure("initial stat", logPath, error2);
|
|
3675
|
+
baselineReady = false;
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
let polling = false;
|
|
3679
|
+
let stopped = false;
|
|
3680
|
+
const poll = async () => {
|
|
3681
|
+
if (polling || stopped) return;
|
|
3682
|
+
polling = true;
|
|
3683
|
+
try {
|
|
3684
|
+
let current;
|
|
3685
|
+
try {
|
|
3686
|
+
current = await stat(logPath);
|
|
3687
|
+
} catch (error2) {
|
|
3688
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
3689
|
+
return;
|
|
3690
|
+
}
|
|
3691
|
+
if (!baselineReady) {
|
|
3692
|
+
offset = current.size;
|
|
3693
|
+
inode = current.ino;
|
|
3694
|
+
baselineReady = true;
|
|
3695
|
+
return;
|
|
3696
|
+
}
|
|
3697
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
3698
|
+
offset = 0;
|
|
3699
|
+
}
|
|
3700
|
+
inode = current.ino;
|
|
3701
|
+
if (current.size === offset) return;
|
|
3702
|
+
const length = current.size - offset;
|
|
3703
|
+
const fh = await open2(logPath, "r");
|
|
3704
|
+
try {
|
|
3705
|
+
const buf = Buffer.alloc(length);
|
|
3706
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
3707
|
+
offset += bytesRead;
|
|
3708
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
3709
|
+
} finally {
|
|
3710
|
+
await fh.close();
|
|
3711
|
+
}
|
|
3712
|
+
} catch (error2) {
|
|
3713
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
3714
|
+
} finally {
|
|
3715
|
+
polling = false;
|
|
3716
|
+
}
|
|
3717
|
+
};
|
|
3718
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
3719
|
+
void poll();
|
|
3720
|
+
return {
|
|
3721
|
+
stop: () => {
|
|
3722
|
+
stopped = true;
|
|
3723
|
+
clearInterval(interval);
|
|
3724
|
+
}
|
|
3725
|
+
};
|
|
3726
|
+
}
|
|
3727
|
+
|
|
2760
3728
|
// src/lib/opencode/session-db-reclaim.ts
|
|
2761
|
-
import { statSync as
|
|
2762
|
-
import { dirname as
|
|
3729
|
+
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
3730
|
+
import { dirname as dirname4 } from "node:path";
|
|
3731
|
+
function errorMessage(error2) {
|
|
3732
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
3733
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
3734
|
+
}
|
|
2763
3735
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2764
3736
|
try {
|
|
2765
|
-
const fsStats = statfsSync(
|
|
3737
|
+
const fsStats = statfsSync(dirname4(dbPath));
|
|
2766
3738
|
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2767
3739
|
if (availableBytes < requiredBytes) {
|
|
2768
3740
|
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
@@ -2785,17 +3757,17 @@ async function probeReclaimAvailability(input) {
|
|
|
2785
3757
|
const { dbPath, requiredBytes } = input;
|
|
2786
3758
|
let sqlite;
|
|
2787
3759
|
try {
|
|
2788
|
-
sqlite = await import("sqlite");
|
|
3760
|
+
sqlite = await import("node:sqlite");
|
|
2789
3761
|
} catch (err) {
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
return "sqlite-unavailable";
|
|
3762
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
3763
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
3764
|
+
return { reason: "sqlite-unavailable", detail };
|
|
2794
3765
|
}
|
|
2795
3766
|
let autoVacuum = null;
|
|
2796
3767
|
try {
|
|
2797
3768
|
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
2798
3769
|
try {
|
|
3770
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
2799
3771
|
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2800
3772
|
} finally {
|
|
2801
3773
|
db.close();
|
|
@@ -2806,23 +3778,25 @@ async function probeReclaimAvailability(input) {
|
|
|
2806
3778
|
);
|
|
2807
3779
|
}
|
|
2808
3780
|
if (autoVacuum !== 0) return null;
|
|
2809
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? "insufficient-disk-space" : null;
|
|
3781
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
2810
3782
|
}
|
|
2811
3783
|
async function reclaimSessionDbSpace(input) {
|
|
2812
3784
|
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
2813
3785
|
let sqlite;
|
|
2814
3786
|
try {
|
|
2815
|
-
sqlite = await import("sqlite");
|
|
3787
|
+
sqlite = await import("node:sqlite");
|
|
2816
3788
|
} catch (err) {
|
|
3789
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
2817
3790
|
console.warn(
|
|
2818
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${
|
|
3791
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
2819
3792
|
);
|
|
2820
|
-
return { ok: false, skipped: "sqlite-unavailable" };
|
|
3793
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
2821
3794
|
}
|
|
2822
3795
|
const { DatabaseSync } = sqlite;
|
|
2823
3796
|
let db;
|
|
2824
3797
|
try {
|
|
2825
3798
|
db = new DatabaseSync(dbPath);
|
|
3799
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
2826
3800
|
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2827
3801
|
if (autoVacuum === 0) {
|
|
2828
3802
|
if (!allowFullVacuum) {
|
|
@@ -2831,7 +3805,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2831
3805
|
);
|
|
2832
3806
|
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2833
3807
|
}
|
|
2834
|
-
const fileBytesForGuard =
|
|
3808
|
+
const fileBytesForGuard = statSync5(dbPath).size;
|
|
2835
3809
|
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2836
3810
|
if (skipReason !== null) {
|
|
2837
3811
|
console.warn(
|
|
@@ -2859,10 +3833,12 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2859
3833
|
);
|
|
2860
3834
|
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
2861
3835
|
} catch (err) {
|
|
2862
|
-
console.error(
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
3836
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
3837
|
+
return {
|
|
3838
|
+
ok: false,
|
|
3839
|
+
skipped: "reclaim-error",
|
|
3840
|
+
detail: errorMessage(err)
|
|
3841
|
+
};
|
|
2866
3842
|
} finally {
|
|
2867
3843
|
db?.close();
|
|
2868
3844
|
}
|
|
@@ -2903,7 +3879,6 @@ var StreamForwarder = class {
|
|
|
2903
3879
|
handleFrame(frame) {
|
|
2904
3880
|
switch (frame.type) {
|
|
2905
3881
|
case "open":
|
|
2906
|
-
this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
|
|
2907
3882
|
void this.handleOpen(frame);
|
|
2908
3883
|
break;
|
|
2909
3884
|
case "req_data":
|
|
@@ -2939,12 +3914,21 @@ var StreamForwarder = class {
|
|
|
2939
3914
|
const { sid, method, path, headers, has_body } = frame;
|
|
2940
3915
|
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
2941
3916
|
const startedAt = Date.now();
|
|
3917
|
+
if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3918
|
+
this.callbacks.onOpen?.(sid, method, path);
|
|
3919
|
+
}
|
|
2942
3920
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
2943
3921
|
this.callbacks.onDrainPing?.();
|
|
2944
3922
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
2945
3923
|
this.send({ type: "res_end", sid });
|
|
2946
3924
|
return;
|
|
2947
3925
|
}
|
|
3926
|
+
if (path === TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3927
|
+
this.callbacks.onUsageRearmPing?.();
|
|
3928
|
+
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
3929
|
+
this.send({ type: "res_end", sid });
|
|
3930
|
+
return;
|
|
3931
|
+
}
|
|
2948
3932
|
if (process.env.DEBUG) {
|
|
2949
3933
|
log("debug", "agent_request", {
|
|
2950
3934
|
correlation_id: correlationId,
|
|
@@ -2959,12 +3943,12 @@ var StreamForwarder = class {
|
|
|
2959
3943
|
let endBody;
|
|
2960
3944
|
if (has_body) {
|
|
2961
3945
|
const chunks = [];
|
|
2962
|
-
bodyPromise = new Promise((
|
|
3946
|
+
bodyPromise = new Promise((resolve4) => {
|
|
2963
3947
|
pushBody = (buf) => {
|
|
2964
3948
|
chunks.push(buf);
|
|
2965
3949
|
};
|
|
2966
3950
|
endBody = () => {
|
|
2967
|
-
|
|
3951
|
+
resolve4(Buffer.concat(chunks));
|
|
2968
3952
|
};
|
|
2969
3953
|
});
|
|
2970
3954
|
}
|
|
@@ -3089,11 +4073,12 @@ function connectTunnel(options) {
|
|
|
3089
4073
|
onResponse,
|
|
3090
4074
|
onInfo,
|
|
3091
4075
|
onWarning,
|
|
3092
|
-
onDrainPing
|
|
4076
|
+
onDrainPing,
|
|
4077
|
+
onUsageRearmPing
|
|
3093
4078
|
} = options;
|
|
3094
4079
|
const tunnelUrl = getTunnelUrlConfig();
|
|
3095
4080
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
3096
|
-
return new Promise((
|
|
4081
|
+
return new Promise((resolve4, reject) => {
|
|
3097
4082
|
const ws = new WebSocket2(url, {
|
|
3098
4083
|
headers: {
|
|
3099
4084
|
Authorization: authHeader
|
|
@@ -3101,7 +4086,8 @@ function connectTunnel(options) {
|
|
|
3101
4086
|
});
|
|
3102
4087
|
const forwarder = new StreamForwarder(ws, port, {
|
|
3103
4088
|
onHead: () => onResponse?.(),
|
|
3104
|
-
onDrainPing: () => onDrainPing?.()
|
|
4089
|
+
onDrainPing: () => onDrainPing?.(),
|
|
4090
|
+
onUsageRearmPing: () => onUsageRearmPing?.()
|
|
3105
4091
|
});
|
|
3106
4092
|
const connectionTimeout = setTimeout(() => {
|
|
3107
4093
|
ws.close();
|
|
@@ -3144,8 +4130,8 @@ function connectTunnel(options) {
|
|
|
3144
4130
|
try {
|
|
3145
4131
|
message = JSON.parse(data.toString());
|
|
3146
4132
|
} catch (error2) {
|
|
3147
|
-
const
|
|
3148
|
-
onError?.(`Failed to handle message: ${
|
|
4133
|
+
const errorMessage3 = error2 instanceof Error ? error2.message : "Unknown error";
|
|
4134
|
+
onError?.(`Failed to handle message: ${errorMessage3}`);
|
|
3149
4135
|
return;
|
|
3150
4136
|
}
|
|
3151
4137
|
if (isStreamFrame(message)) {
|
|
@@ -3157,7 +4143,7 @@ function connectTunnel(options) {
|
|
|
3157
4143
|
clearTimeout(connectionTimeout);
|
|
3158
4144
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
3159
4145
|
onConnected?.(connectedAgentId);
|
|
3160
|
-
|
|
4146
|
+
resolve4({
|
|
3161
4147
|
ws,
|
|
3162
4148
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
3163
4149
|
});
|
|
@@ -3262,6 +4248,7 @@ var RunnerConnection = class {
|
|
|
3262
4248
|
onError: (error2) => events.onError?.(error2),
|
|
3263
4249
|
onResponse: () => events.onResponse?.(),
|
|
3264
4250
|
onDrainPing: () => events.onDrainPing?.(),
|
|
4251
|
+
onUsageRearmPing: () => events.onUsageRearmPing?.(),
|
|
3265
4252
|
onInfo: (message) => events.onInfo?.(message),
|
|
3266
4253
|
onWarning: (message) => events.onWarning?.(message)
|
|
3267
4254
|
});
|
|
@@ -3288,10 +4275,10 @@ var RunnerConnection = class {
|
|
|
3288
4275
|
};
|
|
3289
4276
|
|
|
3290
4277
|
// src/lib/tunnel/ready-marker.ts
|
|
3291
|
-
import { writeFileSync } from "fs";
|
|
4278
|
+
import { writeFileSync as writeFileSync3 } from "node:fs";
|
|
3292
4279
|
function writeTunnelReadyMarker(path, agentId) {
|
|
3293
4280
|
try {
|
|
3294
|
-
|
|
4281
|
+
writeFileSync3(path, `${agentId}
|
|
3295
4282
|
`);
|
|
3296
4283
|
return { ok: true };
|
|
3297
4284
|
} catch (error2) {
|
|
@@ -3300,9 +4287,9 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
3300
4287
|
}
|
|
3301
4288
|
|
|
3302
4289
|
// src/lib/replication.ts
|
|
3303
|
-
import { spawn as
|
|
4290
|
+
import { spawn as spawn4 } from "node:child_process";
|
|
3304
4291
|
function startSessionDbReplication(configPath) {
|
|
3305
|
-
return
|
|
4292
|
+
return spawn4("litestream", ["replicate", "-config", configPath], {
|
|
3306
4293
|
stdio: "inherit"
|
|
3307
4294
|
});
|
|
3308
4295
|
}
|
|
@@ -3315,10 +4302,36 @@ async function stopSessionDbReplication(child, timeoutMs) {
|
|
|
3315
4302
|
);
|
|
3316
4303
|
}
|
|
3317
4304
|
|
|
4305
|
+
// src/lib/process-liveness.ts
|
|
4306
|
+
import { readFileSync as readFileSync4 } from "node:fs";
|
|
4307
|
+
function isProcessAlive(pid) {
|
|
4308
|
+
try {
|
|
4309
|
+
process.kill(pid, 0);
|
|
4310
|
+
} catch (error2) {
|
|
4311
|
+
const code = error2.code;
|
|
4312
|
+
if (code === "ESRCH") return false;
|
|
4313
|
+
if (code === "EPERM") return true;
|
|
4314
|
+
console.error(
|
|
4315
|
+
`[process-liveness] could not probe process ${pid}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4316
|
+
);
|
|
4317
|
+
return false;
|
|
4318
|
+
}
|
|
4319
|
+
if (process.platform !== "linux") return true;
|
|
4320
|
+
try {
|
|
4321
|
+
const status2 = readFileSync4(`/proc/${pid}/status`, "utf8");
|
|
4322
|
+
return !/^State:\s+Z(?:\s|$)/m.test(status2);
|
|
4323
|
+
} catch (error2) {
|
|
4324
|
+
console.error(
|
|
4325
|
+
`[process-liveness] could not inspect /proc/${pid}/status: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4326
|
+
);
|
|
4327
|
+
return true;
|
|
4328
|
+
}
|
|
4329
|
+
}
|
|
4330
|
+
|
|
3318
4331
|
// src/lib/openai-usage.ts
|
|
3319
|
-
import { readFileSync as
|
|
3320
|
-
import { homedir as
|
|
3321
|
-
import { join as
|
|
4332
|
+
import { readFileSync as readFileSync5 } from "node:fs";
|
|
4333
|
+
import { homedir as homedir4 } from "node:os";
|
|
4334
|
+
import { join as join6 } from "node:path";
|
|
3322
4335
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3323
4336
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3324
4337
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3332,7 +4345,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3332
4345
|
}
|
|
3333
4346
|
function readOpenCodeChatGptCredentials() {
|
|
3334
4347
|
try {
|
|
3335
|
-
const raw =
|
|
4348
|
+
const raw = readFileSync5(join6(homedir4(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3336
4349
|
let parsed;
|
|
3337
4350
|
try {
|
|
3338
4351
|
parsed = JSON.parse(raw);
|
|
@@ -3354,6 +4367,23 @@ function readOpenCodeChatGptCredentials() {
|
|
|
3354
4367
|
return null;
|
|
3355
4368
|
}
|
|
3356
4369
|
}
|
|
4370
|
+
function parseChatGptIdentity(accessToken) {
|
|
4371
|
+
const segments = accessToken.split(".");
|
|
4372
|
+
if (segments.length !== 3) return null;
|
|
4373
|
+
let payload;
|
|
4374
|
+
try {
|
|
4375
|
+
const parsed = JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8"));
|
|
4376
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
4377
|
+
payload = parsed;
|
|
4378
|
+
} catch {
|
|
4379
|
+
return null;
|
|
4380
|
+
}
|
|
4381
|
+
const profile = payload["https://api.openai.com/profile"];
|
|
4382
|
+
const auth = payload["https://api.openai.com/auth"];
|
|
4383
|
+
const ownerEmail = typeof profile?.email === "string" ? profile.email.trim() || null : null;
|
|
4384
|
+
const planType = typeof auth?.chatgpt_plan_type === "string" ? auth.chatgpt_plan_type.trim() || null : null;
|
|
4385
|
+
return ownerEmail === null && planType === null ? null : { ownerEmail, planType, organizationName: null };
|
|
4386
|
+
}
|
|
3357
4387
|
function toWindow2(headers, name) {
|
|
3358
4388
|
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
3359
4389
|
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
@@ -3429,6 +4459,7 @@ async function getOpenAiUsage(port) {
|
|
|
3429
4459
|
"credentials_expired"
|
|
3430
4460
|
);
|
|
3431
4461
|
}
|
|
4462
|
+
const subscription = parseChatGptIdentity(credentials2.accessToken);
|
|
3432
4463
|
const models = await resolveProbeModels(port);
|
|
3433
4464
|
if (models.length === 0) {
|
|
3434
4465
|
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
@@ -3461,7 +4492,7 @@ async function getOpenAiUsage(port) {
|
|
|
3461
4492
|
"no_usable_window"
|
|
3462
4493
|
);
|
|
3463
4494
|
}
|
|
3464
|
-
return usage;
|
|
4495
|
+
return { ...usage, subscription };
|
|
3465
4496
|
}
|
|
3466
4497
|
if (res.status === 401) {
|
|
3467
4498
|
throw new OpenAiUsageError(
|
|
@@ -3576,8 +4607,8 @@ function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
|
3576
4607
|
}
|
|
3577
4608
|
|
|
3578
4609
|
// src/lib/resource-usage.ts
|
|
3579
|
-
import { cpus, totalmem, freemem } from "os";
|
|
3580
|
-
import { statfsSync as statfsSync2 } from "fs";
|
|
4610
|
+
import { cpus, totalmem, freemem } from "node:os";
|
|
4611
|
+
import { statfsSync as statfsSync2 } from "node:fs";
|
|
3581
4612
|
|
|
3582
4613
|
// src/lib/ecs-task-metadata.ts
|
|
3583
4614
|
var ECS_METADATA_TIMEOUT_MS = 2e3;
|
|
@@ -3662,58 +4693,97 @@ function readDisk(homeDir) {
|
|
|
3662
4693
|
};
|
|
3663
4694
|
}
|
|
3664
4695
|
}
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
4696
|
+
var CPU_PEAK_WINDOW_MS = 6e4;
|
|
4697
|
+
var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
|
|
4698
|
+
var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4699
|
+
function createCpuPeakSampler() {
|
|
4700
|
+
const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4701
|
+
sampleHistory[0] = readCpuSample();
|
|
4702
|
+
let nextSampleIndex = 1;
|
|
4703
|
+
let sampleCount = 1;
|
|
4704
|
+
let peak = null;
|
|
4705
|
+
const timer = setInterval(() => {
|
|
3668
4706
|
const current = readCpuSample();
|
|
3669
|
-
const
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
const warnings = [];
|
|
3676
|
-
if (disk.warning) warnings.push(disk.warning);
|
|
3677
|
-
if (ecsWarning) warnings.push(ecsWarning);
|
|
3678
|
-
let cpuPercent = hostCpuPercent;
|
|
3679
|
-
let cpuCount = hostCpuCount;
|
|
3680
|
-
let memoryTotalBytes = totalmem();
|
|
3681
|
-
let memoryAvailableBytes = freemem();
|
|
3682
|
-
if (limits !== null) {
|
|
3683
|
-
cpuCount = limits.cpuCount;
|
|
3684
|
-
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3685
|
-
memoryAvailableBytes = clamp(
|
|
3686
|
-
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3687
|
-
0,
|
|
3688
|
-
limits.memoryTotalBytes
|
|
3689
|
-
);
|
|
3690
|
-
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4707
|
+
const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
|
|
4708
|
+
if (sampleFromWindowAgo !== void 0) {
|
|
4709
|
+
const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
|
|
4710
|
+
if (percentage !== null) {
|
|
4711
|
+
peak = peak === null ? percentage : Math.max(peak, percentage);
|
|
4712
|
+
}
|
|
3691
4713
|
}
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
4714
|
+
sampleHistory[nextSampleIndex] = current;
|
|
4715
|
+
nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4716
|
+
sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4717
|
+
}, CPU_PEAK_SAMPLE_INTERVAL_MS);
|
|
4718
|
+
return {
|
|
4719
|
+
takeAndReset: () => {
|
|
4720
|
+
const currentPeak = peak;
|
|
4721
|
+
peak = null;
|
|
4722
|
+
return currentPeak;
|
|
4723
|
+
},
|
|
4724
|
+
stop: () => clearInterval(timer)
|
|
4725
|
+
};
|
|
4726
|
+
}
|
|
4727
|
+
function createResourceUsageCollector(homeDir) {
|
|
4728
|
+
let previous = readCpuSample();
|
|
4729
|
+
const cpuPeakSampler = createCpuPeakSampler();
|
|
4730
|
+
return {
|
|
4731
|
+
collect: async () => {
|
|
4732
|
+
const current = readCpuSample();
|
|
4733
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
4734
|
+
const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
|
|
4735
|
+
const hostCpuCount = cpus().length;
|
|
4736
|
+
previous = current;
|
|
4737
|
+
const disk = readDisk(homeDir);
|
|
4738
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
4739
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
4740
|
+
const warnings = [];
|
|
4741
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
4742
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
4743
|
+
let cpuPercent = hostCpuPercent;
|
|
4744
|
+
let cpuPeakPercent = hostCpuPeakPercent;
|
|
4745
|
+
let cpuCount = hostCpuCount;
|
|
4746
|
+
let memoryTotalBytes = totalmem();
|
|
4747
|
+
let memoryAvailableBytes = freemem();
|
|
4748
|
+
if (limits !== null) {
|
|
4749
|
+
cpuCount = limits.cpuCount;
|
|
4750
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4751
|
+
memoryAvailableBytes = clamp(
|
|
4752
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4753
|
+
0,
|
|
4754
|
+
limits.memoryTotalBytes
|
|
4755
|
+
);
|
|
4756
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4757
|
+
cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4758
|
+
}
|
|
4759
|
+
return {
|
|
4760
|
+
usage: {
|
|
4761
|
+
cpuPercent,
|
|
4762
|
+
cpuPeakPercent,
|
|
4763
|
+
cpuCount,
|
|
4764
|
+
memoryTotalBytes,
|
|
4765
|
+
memoryAvailableBytes,
|
|
4766
|
+
diskTotalBytes: disk.totalBytes,
|
|
4767
|
+
diskFreeBytes: disk.freeBytes,
|
|
4768
|
+
opencodeDbBytes
|
|
4769
|
+
},
|
|
4770
|
+
warnings
|
|
4771
|
+
};
|
|
4772
|
+
},
|
|
4773
|
+
stop: cpuPeakSampler.stop
|
|
3704
4774
|
};
|
|
3705
4775
|
}
|
|
3706
4776
|
|
|
3707
4777
|
// src/lib/channels/driver.ts
|
|
3708
|
-
import { homedir as
|
|
4778
|
+
import { homedir as homedir5 } from "node:os";
|
|
3709
4779
|
|
|
3710
4780
|
// src/lib/runner-file-sync.ts
|
|
3711
|
-
import { join as
|
|
4781
|
+
import { join as join8 } from "node:path";
|
|
3712
4782
|
|
|
3713
4783
|
// src/lib/file-push.ts
|
|
3714
|
-
import { randomUUID } from "crypto";
|
|
3715
|
-
import { chmod, mkdir, open as
|
|
3716
|
-
import { basename, dirname as
|
|
4784
|
+
import { randomUUID } from "node:crypto";
|
|
4785
|
+
import { chmod, mkdir, open as open3, realpath, rename, unlink } from "node:fs/promises";
|
|
4786
|
+
import { basename, dirname as dirname5, isAbsolute, join as join7, relative, resolve as resolve2, sep } from "node:path";
|
|
3717
4787
|
var FILE_MODE = 384;
|
|
3718
4788
|
var DIRECTORY_MODE = 448;
|
|
3719
4789
|
async function writePushedFile(request) {
|
|
@@ -3744,9 +4814,9 @@ async function writePushedFile(request) {
|
|
|
3744
4814
|
}
|
|
3745
4815
|
try {
|
|
3746
4816
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3747
|
-
|
|
4817
|
+
dirname5(candidate)
|
|
3748
4818
|
);
|
|
3749
|
-
const realTarget =
|
|
4819
|
+
const realTarget = join7(existingAncestor, ...missingSegments, basename(candidate));
|
|
3750
4820
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3751
4821
|
if (allowedDirectory === null) {
|
|
3752
4822
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3756,8 +4826,8 @@ async function writePushedFile(request) {
|
|
|
3756
4826
|
}
|
|
3757
4827
|
if (missingSegments.length > 0) {
|
|
3758
4828
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
3759
|
-
const realParent = await realpath(
|
|
3760
|
-
if (realParent !==
|
|
4829
|
+
const realParent = await realpath(dirname5(realTarget));
|
|
4830
|
+
if (realParent !== dirname5(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
3761
4831
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
3762
4832
|
path: realTarget,
|
|
3763
4833
|
bytes,
|
|
@@ -3782,7 +4852,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3782
4852
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3783
4853
|
return null;
|
|
3784
4854
|
}
|
|
3785
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4855
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3786
4856
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3787
4857
|
return null;
|
|
3788
4858
|
}
|
|
@@ -3800,7 +4870,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
3800
4870
|
try {
|
|
3801
4871
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
3802
4872
|
} catch (err) {
|
|
3803
|
-
const parent =
|
|
4873
|
+
const parent = dirname5(current);
|
|
3804
4874
|
if (err.code !== "ENOENT" || parent === current) {
|
|
3805
4875
|
throw err;
|
|
3806
4876
|
}
|
|
@@ -3855,16 +4925,16 @@ function contains(realDirectory, realTarget) {
|
|
|
3855
4925
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3856
4926
|
let current = existingAncestor;
|
|
3857
4927
|
for (const segment of missingSegments) {
|
|
3858
|
-
current =
|
|
4928
|
+
current = join7(current, segment);
|
|
3859
4929
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3860
4930
|
await chmod(current, DIRECTORY_MODE);
|
|
3861
4931
|
}
|
|
3862
4932
|
}
|
|
3863
4933
|
async function writeAtomically(realTarget, content) {
|
|
3864
|
-
const temporaryPath =
|
|
4934
|
+
const temporaryPath = join7(dirname5(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3865
4935
|
let handle;
|
|
3866
4936
|
try {
|
|
3867
|
-
handle = await
|
|
4937
|
+
handle = await open3(temporaryPath, "wx", FILE_MODE);
|
|
3868
4938
|
await handle.writeFile(content);
|
|
3869
4939
|
await handle.chmod(FILE_MODE);
|
|
3870
4940
|
await handle.close();
|
|
@@ -3991,12 +5061,12 @@ var NOT_APPLIED = {
|
|
|
3991
5061
|
opencodeAuthApplied: false
|
|
3992
5062
|
};
|
|
3993
5063
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3994
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3995
|
-
return expanded ===
|
|
5064
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join8(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
5065
|
+
return expanded === join8(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3996
5066
|
}
|
|
3997
5067
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
3998
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3999
|
-
return expanded ===
|
|
5068
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join8(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
5069
|
+
return expanded === join8(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
4000
5070
|
}
|
|
4001
5071
|
async function applyOne(options, file) {
|
|
4002
5072
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -4523,6 +5593,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4523
5593
|
* and stops opencode.
|
|
4524
5594
|
*/
|
|
4525
5595
|
stopped = false;
|
|
5596
|
+
recycleRequestedFlag = false;
|
|
4526
5597
|
constructor(config) {
|
|
4527
5598
|
this.agentId = config.agentId;
|
|
4528
5599
|
this.port = config.port;
|
|
@@ -4542,7 +5613,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4542
5613
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
4543
5614
|
this.now = config.now ?? (() => Date.now());
|
|
4544
5615
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
4545
|
-
this.homeDir = config.homeDir ??
|
|
5616
|
+
this.homeDir = config.homeDir ?? homedir5();
|
|
4546
5617
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4547
5618
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4548
5619
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
@@ -4628,6 +5699,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4628
5699
|
let dispatched = 0;
|
|
4629
5700
|
try {
|
|
4630
5701
|
const conversations = await this.getPendingConversations();
|
|
5702
|
+
if (this.recycleRequestedFlag) {
|
|
5703
|
+
this.stop();
|
|
5704
|
+
}
|
|
4631
5705
|
if (conversations.length > 0) {
|
|
4632
5706
|
const total = conversations.reduce((sum, c) => sum + (c.pending_message_count ?? 0), 0);
|
|
4633
5707
|
this.log({
|
|
@@ -4756,6 +5830,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4756
5830
|
stop() {
|
|
4757
5831
|
this.stopped = true;
|
|
4758
5832
|
}
|
|
5833
|
+
/**
|
|
5834
|
+
* The server clears this request when a new MicroVM identity is recorded, so a
|
|
5835
|
+
* same-VM tunnel reconnect does not consume it. This is a plain read rather
|
|
5836
|
+
* than a consume; `run.ts` guards the action once-only.
|
|
5837
|
+
*/
|
|
5838
|
+
get recycleRequested() {
|
|
5839
|
+
return this.recycleRequestedFlag;
|
|
5840
|
+
}
|
|
4759
5841
|
/**
|
|
4760
5842
|
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
4761
5843
|
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
@@ -4893,7 +5975,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4893
5975
|
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
4894
5976
|
break;
|
|
4895
5977
|
}
|
|
4896
|
-
const
|
|
5978
|
+
const errorMessage3 = err instanceof Error ? err.message : String(err);
|
|
4897
5979
|
this.sessions.delete(conv.id);
|
|
4898
5980
|
this.supersede(conv.id, sessionId);
|
|
4899
5981
|
this.log({
|
|
@@ -4902,7 +5984,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4902
5984
|
conversation_id: conv.id,
|
|
4903
5985
|
message_id: message.id
|
|
4904
5986
|
});
|
|
4905
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5987
|
+
await this.markFailed(conv.id, message.id, null, errorMessage3).catch((markErr) => {
|
|
4906
5988
|
this.log({
|
|
4907
5989
|
level: "warn",
|
|
4908
5990
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4913,7 +5995,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4913
5995
|
});
|
|
4914
5996
|
this.log({
|
|
4915
5997
|
level: "error",
|
|
4916
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
5998
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage3}`,
|
|
4917
5999
|
conversation_id: conv.id,
|
|
4918
6000
|
message_id: message.id
|
|
4919
6001
|
});
|
|
@@ -4934,14 +6016,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4934
6016
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4935
6017
|
this.sessions.delete(conv.id);
|
|
4936
6018
|
this.supersede(conv.id, sessionId);
|
|
4937
|
-
const
|
|
6019
|
+
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
4938
6020
|
this.log({
|
|
4939
6021
|
level: "error",
|
|
4940
|
-
message:
|
|
6022
|
+
message: errorMessage3,
|
|
4941
6023
|
conversation_id: conv.id,
|
|
4942
6024
|
message_id: message.id
|
|
4943
6025
|
});
|
|
4944
|
-
await this.markFailed(conv.id, message.id, null,
|
|
6026
|
+
await this.markFailed(conv.id, message.id, null, errorMessage3).catch((markErr) => {
|
|
4945
6027
|
this.log({
|
|
4946
6028
|
level: "warn",
|
|
4947
6029
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -5275,6 +6357,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5275
6357
|
});
|
|
5276
6358
|
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
5277
6359
|
}
|
|
6360
|
+
if (ocId !== null) {
|
|
6361
|
+
await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
|
|
6362
|
+
}
|
|
5278
6363
|
} catch (err) {
|
|
5279
6364
|
if (err instanceof ChannelAuthError) throw err;
|
|
5280
6365
|
this.log({
|
|
@@ -6239,6 +7324,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6239
7324
|
return;
|
|
6240
7325
|
}
|
|
6241
7326
|
inFlight.done = true;
|
|
7327
|
+
await this.reportSubagentAuthFailures(
|
|
7328
|
+
watcher.conv.id,
|
|
7329
|
+
inFlight.opencodeMessageId,
|
|
7330
|
+
inFlight.evidentMessageId,
|
|
7331
|
+
messages
|
|
7332
|
+
);
|
|
6242
7333
|
}
|
|
6243
7334
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6244
7335
|
return;
|
|
@@ -6481,6 +7572,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6481
7572
|
return;
|
|
6482
7573
|
}
|
|
6483
7574
|
inFlight.done = true;
|
|
7575
|
+
await this.reportSubagentAuthFailures(
|
|
7576
|
+
watcher.conv.id,
|
|
7577
|
+
inFlight.opencodeMessageId,
|
|
7578
|
+
inFlight.evidentMessageId,
|
|
7579
|
+
messages
|
|
7580
|
+
);
|
|
6484
7581
|
}
|
|
6485
7582
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6486
7583
|
}
|
|
@@ -6651,6 +7748,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6651
7748
|
});
|
|
6652
7749
|
return;
|
|
6653
7750
|
}
|
|
7751
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
6654
7752
|
this.dontRedispatch.delete(row.id);
|
|
6655
7753
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
6656
7754
|
return;
|
|
@@ -6812,6 +7910,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6812
7910
|
});
|
|
6813
7911
|
return;
|
|
6814
7912
|
}
|
|
7913
|
+
if (ocId !== null) {
|
|
7914
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
|
|
7915
|
+
}
|
|
6815
7916
|
this.dontRedispatch.delete(row.id);
|
|
6816
7917
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
6817
7918
|
}
|
|
@@ -6905,14 +8006,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6905
8006
|
this.unconfirmedDispatchFailures.delete(row.id);
|
|
6906
8007
|
this.sessions.delete(readoptConv.id);
|
|
6907
8008
|
this.supersede(readoptConv.id, sessionId);
|
|
6908
|
-
const
|
|
8009
|
+
const errorMessage3 = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
6909
8010
|
this.log({
|
|
6910
8011
|
level: "error",
|
|
6911
|
-
message:
|
|
8012
|
+
message: errorMessage3,
|
|
6912
8013
|
conversation_id: row.conversation_id,
|
|
6913
8014
|
message_id: row.id
|
|
6914
8015
|
});
|
|
6915
|
-
await this.markFailed(row.conversation_id, row.id, null,
|
|
8016
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
6916
8017
|
this.log({
|
|
6917
8018
|
level: "warn",
|
|
6918
8019
|
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -7527,6 +8628,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7527
8628
|
throw new Error(`Failed to get pending conversations: HTTP ${res.status}`);
|
|
7528
8629
|
}
|
|
7529
8630
|
const data = await res.json();
|
|
8631
|
+
this.recycleRequestedFlag = data.recycle_requested === true;
|
|
7530
8632
|
let conversations = data.conversations;
|
|
7531
8633
|
if (this.conversationFilter) {
|
|
7532
8634
|
conversations = conversations.filter((c) => c.id === this.conversationFilter);
|
|
@@ -7762,6 +8864,111 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7762
8864
|
reply?.info?.modelID ?? null
|
|
7763
8865
|
);
|
|
7764
8866
|
}
|
|
8867
|
+
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
8868
|
+
const providerId = failure.providerId ?? "(unknown)";
|
|
8869
|
+
try {
|
|
8870
|
+
const res = await this.fetchImpl(
|
|
8871
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
|
|
8872
|
+
{
|
|
8873
|
+
method: "POST",
|
|
8874
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
8875
|
+
body: JSON.stringify({
|
|
8876
|
+
provider_id: failure.providerId,
|
|
8877
|
+
model_id: failure.modelId,
|
|
8878
|
+
reason: failure.reason
|
|
8879
|
+
})
|
|
8880
|
+
}
|
|
8881
|
+
);
|
|
8882
|
+
if (!res.ok) {
|
|
8883
|
+
this.log({
|
|
8884
|
+
level: "warn",
|
|
8885
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8886
|
+
conversation_id: conversationId,
|
|
8887
|
+
message_id: messageId
|
|
8888
|
+
});
|
|
8889
|
+
}
|
|
8890
|
+
} catch (err) {
|
|
8891
|
+
this.log({
|
|
8892
|
+
level: "warn",
|
|
8893
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8894
|
+
conversation_id: conversationId,
|
|
8895
|
+
message_id: messageId
|
|
8896
|
+
});
|
|
8897
|
+
}
|
|
8898
|
+
}
|
|
8899
|
+
async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
|
|
8900
|
+
try {
|
|
8901
|
+
const res = await this.fetchImpl(
|
|
8902
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
|
|
8903
|
+
{
|
|
8904
|
+
method: "DELETE",
|
|
8905
|
+
headers: { Authorization: this.getAuthHeader() }
|
|
8906
|
+
}
|
|
8907
|
+
);
|
|
8908
|
+
if (!res.ok) {
|
|
8909
|
+
this.log({
|
|
8910
|
+
level: "warn",
|
|
8911
|
+
message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8912
|
+
conversation_id: conversationId,
|
|
8913
|
+
message_id: messageId
|
|
8914
|
+
});
|
|
8915
|
+
}
|
|
8916
|
+
} catch (err) {
|
|
8917
|
+
this.log({
|
|
8918
|
+
level: "warn",
|
|
8919
|
+
message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8920
|
+
conversation_id: conversationId,
|
|
8921
|
+
message_id: messageId
|
|
8922
|
+
});
|
|
8923
|
+
}
|
|
8924
|
+
}
|
|
8925
|
+
async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
|
|
8926
|
+
const refs = collectSubagentSessions(messages, opencodeMessageId);
|
|
8927
|
+
if (refs.length === 0) return;
|
|
8928
|
+
const failedProviders = /* @__PURE__ */ new Map();
|
|
8929
|
+
const succeededProviders = /* @__PURE__ */ new Set();
|
|
8930
|
+
for (const ref of refs) {
|
|
8931
|
+
try {
|
|
8932
|
+
const childMessages = await getSessionMessages(this.port, ref.sessionId);
|
|
8933
|
+
if (childMessages === null) {
|
|
8934
|
+
this.log({
|
|
8935
|
+
level: "debug",
|
|
8936
|
+
message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
|
|
8937
|
+
conversation_id: conversationId,
|
|
8938
|
+
message_id: evidentMessageId
|
|
8939
|
+
});
|
|
8940
|
+
continue;
|
|
8941
|
+
}
|
|
8942
|
+
const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
|
|
8943
|
+
if (!outcome) continue;
|
|
8944
|
+
if (outcome.outcome === "failed") {
|
|
8945
|
+
failedProviders.set(outcome.providerId, outcome.failure);
|
|
8946
|
+
} else {
|
|
8947
|
+
succeededProviders.add(outcome.providerId);
|
|
8948
|
+
}
|
|
8949
|
+
} catch (err) {
|
|
8950
|
+
this.log({
|
|
8951
|
+
level: "warn",
|
|
8952
|
+
message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8953
|
+
conversation_id: conversationId,
|
|
8954
|
+
message_id: evidentMessageId
|
|
8955
|
+
});
|
|
8956
|
+
}
|
|
8957
|
+
}
|
|
8958
|
+
for (const [providerId, failure] of failedProviders) {
|
|
8959
|
+
this.log({
|
|
8960
|
+
level: "warn",
|
|
8961
|
+
message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
|
|
8962
|
+
conversation_id: conversationId,
|
|
8963
|
+
message_id: evidentMessageId
|
|
8964
|
+
});
|
|
8965
|
+
await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
|
|
8966
|
+
}
|
|
8967
|
+
for (const providerId of succeededProviders) {
|
|
8968
|
+
if (failedProviders.has(providerId)) continue;
|
|
8969
|
+
await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
|
|
8970
|
+
}
|
|
8971
|
+
}
|
|
7765
8972
|
/**
|
|
7766
8973
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
7767
8974
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -7915,6 +9122,13 @@ import chalk5 from "chalk";
|
|
|
7915
9122
|
import ora2 from "ora";
|
|
7916
9123
|
import { select as select2 } from "@inquirer/prompts";
|
|
7917
9124
|
var INTERACTIVE_START_TIMEOUT_MS = 3e4;
|
|
9125
|
+
function checkNonInteractivePortConflict(port, isPortInUseFn) {
|
|
9126
|
+
if (isPortInUseFn(port)) {
|
|
9127
|
+
throw new Error(
|
|
9128
|
+
`Port ${port} is already in use by a non-OpenCode process. Free it or pass --port.`
|
|
9129
|
+
);
|
|
9130
|
+
}
|
|
9131
|
+
}
|
|
7918
9132
|
async function ensureOpenCodeRunning(ctx) {
|
|
7919
9133
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
7920
9134
|
if (healthCheck.healthy) {
|
|
@@ -7962,8 +9176,9 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
7962
9176
|
}
|
|
7963
9177
|
}
|
|
7964
9178
|
if (!ctx.interactive) {
|
|
9179
|
+
checkNonInteractivePortConflict(ctx.port, isPortInUse);
|
|
7965
9180
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
7966
|
-
const proc = await startOpenCode(ctx.port);
|
|
9181
|
+
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
7967
9182
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
7968
9183
|
if (!health.healthy) {
|
|
7969
9184
|
return {
|
|
@@ -7981,66 +9196,721 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
7981
9196
|
notReadyReason: null
|
|
7982
9197
|
};
|
|
7983
9198
|
}
|
|
7984
|
-
let port = ctx.port;
|
|
7985
|
-
if (isPortInUse(port)) {
|
|
7986
|
-
console.log(chalk5.yellow(`
|
|
7987
|
-
Port ${port} is already in use.`));
|
|
7988
|
-
const alternativePort = findAvailablePort(port + 1);
|
|
7989
|
-
if (alternativePort) {
|
|
7990
|
-
const useAlternative = await select2({
|
|
7991
|
-
message: `Use port ${alternativePort} instead?`,
|
|
7992
|
-
choices: [
|
|
7993
|
-
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
7994
|
-
{ name: "No, I will free the port manually", value: "no" }
|
|
7995
|
-
]
|
|
7996
|
-
});
|
|
7997
|
-
if (useAlternative === "yes") {
|
|
7998
|
-
port = alternativePort;
|
|
7999
|
-
} else {
|
|
8000
|
-
throw new Error(`Port ${ctx.port} is in use`);
|
|
9199
|
+
let port = ctx.port;
|
|
9200
|
+
if (isPortInUse(port)) {
|
|
9201
|
+
console.log(chalk5.yellow(`
|
|
9202
|
+
Port ${port} is already in use.`));
|
|
9203
|
+
const alternativePort = findAvailablePort(port + 1);
|
|
9204
|
+
if (alternativePort) {
|
|
9205
|
+
const useAlternative = await select2({
|
|
9206
|
+
message: `Use port ${alternativePort} instead?`,
|
|
9207
|
+
choices: [
|
|
9208
|
+
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
9209
|
+
{ name: "No, I will free the port manually", value: "no" }
|
|
9210
|
+
]
|
|
9211
|
+
});
|
|
9212
|
+
if (useAlternative === "yes") {
|
|
9213
|
+
port = alternativePort;
|
|
9214
|
+
} else {
|
|
9215
|
+
throw new Error(`Port ${ctx.port} is in use`);
|
|
9216
|
+
}
|
|
9217
|
+
}
|
|
9218
|
+
}
|
|
9219
|
+
const action = await select2({
|
|
9220
|
+
message: "OpenCode is not running. What would you like to do?",
|
|
9221
|
+
choices: [
|
|
9222
|
+
{
|
|
9223
|
+
name: "Start OpenCode for me",
|
|
9224
|
+
value: "start",
|
|
9225
|
+
description: `Run 'opencode serve --port ${port}'`
|
|
9226
|
+
},
|
|
9227
|
+
{
|
|
9228
|
+
name: "Show me the command",
|
|
9229
|
+
value: "manual",
|
|
9230
|
+
description: "Display the command to run manually"
|
|
9231
|
+
},
|
|
9232
|
+
{
|
|
9233
|
+
name: "Continue without OpenCode",
|
|
9234
|
+
value: "continue",
|
|
9235
|
+
description: "Requests will fail until OpenCode starts"
|
|
9236
|
+
}
|
|
9237
|
+
]
|
|
9238
|
+
});
|
|
9239
|
+
if (action === "manual") {
|
|
9240
|
+
blank();
|
|
9241
|
+
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
9242
|
+
blank();
|
|
9243
|
+
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
9244
|
+
blank();
|
|
9245
|
+
throw new Error("Please start OpenCode manually");
|
|
9246
|
+
}
|
|
9247
|
+
if (action === "start") {
|
|
9248
|
+
const spinner = ora2("Starting OpenCode...").start();
|
|
9249
|
+
const proc = await startOpenCode(port, { inheritStdio: ctx.inheritStdio });
|
|
9250
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
9251
|
+
if (!health.healthy) {
|
|
9252
|
+
spinner.fail("Failed to start OpenCode");
|
|
9253
|
+
throw new Error("OpenCode failed to start");
|
|
9254
|
+
}
|
|
9255
|
+
spinner.stop();
|
|
9256
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
9257
|
+
}
|
|
9258
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
9259
|
+
}
|
|
9260
|
+
|
|
9261
|
+
// src/commands/ensure-opencode-v2.ts
|
|
9262
|
+
import chalk6 from "chalk";
|
|
9263
|
+
import { select as select3 } from "@inquirer/prompts";
|
|
9264
|
+
async function probeOpenCode2WithoutPassword(port) {
|
|
9265
|
+
try {
|
|
9266
|
+
const response = await fetch(`http://127.0.0.1:${port}/global/health`, {
|
|
9267
|
+
signal: AbortSignal.timeout(2e3)
|
|
9268
|
+
});
|
|
9269
|
+
if (response.status === 401) {
|
|
9270
|
+
return { healthy: false, authFailed: true, error: "HTTP 401" };
|
|
9271
|
+
}
|
|
9272
|
+
if (!response.ok) {
|
|
9273
|
+
return { healthy: false, error: `HTTP ${response.status}` };
|
|
9274
|
+
}
|
|
9275
|
+
return { healthy: true };
|
|
9276
|
+
} catch (error2) {
|
|
9277
|
+
return {
|
|
9278
|
+
healthy: false,
|
|
9279
|
+
error: error2 instanceof Error ? error2.message : "Unknown error"
|
|
9280
|
+
};
|
|
9281
|
+
}
|
|
9282
|
+
}
|
|
9283
|
+
function unknownPasswordError(port) {
|
|
9284
|
+
return new Error(
|
|
9285
|
+
`OpenCode V2 (opencode2) is already running on port ${port} with a password this runner doesn't know. Free the port or pass --port.`
|
|
9286
|
+
);
|
|
9287
|
+
}
|
|
9288
|
+
function v2SessionSupportIncompleteError() {
|
|
9289
|
+
return new Error(
|
|
9290
|
+
"OpenCode V2 (opencode2) session support is incomplete \u2014 see https://github.com/sroze/evident/issues/2075. Not starting a new opencode2 process."
|
|
9291
|
+
);
|
|
9292
|
+
}
|
|
9293
|
+
async function ensureOpenCode2Running(ctx) {
|
|
9294
|
+
const initialHealth = await probeOpenCode2WithoutPassword(ctx.port);
|
|
9295
|
+
if (initialHealth.authFailed) {
|
|
9296
|
+
throw unknownPasswordError(ctx.port);
|
|
9297
|
+
}
|
|
9298
|
+
if (initialHealth.healthy) {
|
|
9299
|
+
return {
|
|
9300
|
+
port: ctx.port,
|
|
9301
|
+
process: null,
|
|
9302
|
+
version: null,
|
|
9303
|
+
notReadyReason: null,
|
|
9304
|
+
password: null
|
|
9305
|
+
};
|
|
9306
|
+
}
|
|
9307
|
+
if (!isOpenCode2Installed()) {
|
|
9308
|
+
throw new Error(
|
|
9309
|
+
"OpenCode V2 (opencode2) is not installed. Install it with: npm install -g @opencode-ai/cli@beta"
|
|
9310
|
+
);
|
|
9311
|
+
}
|
|
9312
|
+
let port = ctx.port;
|
|
9313
|
+
if (!ctx.interactive) {
|
|
9314
|
+
checkNonInteractivePortConflict(port, isPortInUse);
|
|
9315
|
+
} else if (isPortInUse(port)) {
|
|
9316
|
+
console.log(chalk6.yellow(`
|
|
9317
|
+
Port ${port} is already in use.`));
|
|
9318
|
+
const alternativePort = findAvailablePort(port + 1);
|
|
9319
|
+
if (alternativePort) {
|
|
9320
|
+
const useAlternative = await select3({
|
|
9321
|
+
message: `Use port ${alternativePort} instead?`,
|
|
9322
|
+
choices: [
|
|
9323
|
+
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
9324
|
+
{ name: "No, I will free the port manually", value: "no" }
|
|
9325
|
+
]
|
|
9326
|
+
});
|
|
9327
|
+
if (useAlternative === "yes") {
|
|
9328
|
+
port = alternativePort;
|
|
9329
|
+
} else {
|
|
9330
|
+
throw new Error(`Port ${ctx.port} is in use`);
|
|
9331
|
+
}
|
|
9332
|
+
}
|
|
9333
|
+
}
|
|
9334
|
+
if (!ctx.interactive) {
|
|
9335
|
+
throw v2SessionSupportIncompleteError();
|
|
9336
|
+
}
|
|
9337
|
+
console.log(chalk6.yellow(`
|
|
9338
|
+
${v2SessionSupportIncompleteError().message}`));
|
|
9339
|
+
const action = await select3({
|
|
9340
|
+
message: "OpenCode V2 is not running. What would you like to do?",
|
|
9341
|
+
choices: [
|
|
9342
|
+
{
|
|
9343
|
+
name: "Show me the command",
|
|
9344
|
+
value: "manual",
|
|
9345
|
+
description: "Display the command to run manually"
|
|
9346
|
+
},
|
|
9347
|
+
{
|
|
9348
|
+
name: "Continue without OpenCode V2",
|
|
9349
|
+
value: "continue",
|
|
9350
|
+
description: "Requests will fail until OpenCode V2 starts"
|
|
9351
|
+
}
|
|
9352
|
+
]
|
|
9353
|
+
});
|
|
9354
|
+
if (action === "manual") {
|
|
9355
|
+
blank();
|
|
9356
|
+
console.log(chalk6.bold("Run this command in another terminal:"));
|
|
9357
|
+
blank();
|
|
9358
|
+
console.log(` ${chalk6.cyan(`opencode2 serve --port ${port}`)}`);
|
|
9359
|
+
blank();
|
|
9360
|
+
throw new Error("Please start OpenCode V2 manually");
|
|
9361
|
+
}
|
|
9362
|
+
return {
|
|
9363
|
+
port,
|
|
9364
|
+
process: null,
|
|
9365
|
+
version: null,
|
|
9366
|
+
notReadyReason: "you chose to continue without OpenCode V2",
|
|
9367
|
+
password: null
|
|
9368
|
+
};
|
|
9369
|
+
}
|
|
9370
|
+
|
|
9371
|
+
// src/lib/runner-credentials.ts
|
|
9372
|
+
import { chmodSync as chmodSync2, writeFileSync as writeFileSync4 } from "node:fs";
|
|
9373
|
+
import { spawn as spawn5 } from "node:child_process";
|
|
9374
|
+
var RUNNER_SECRET_FETCH_TIMEOUT_MS = 6e4;
|
|
9375
|
+
var CREDENTIAL_RESTORE_TIMEOUT_MS = 6e4;
|
|
9376
|
+
var GITHUB_PROBE_TIMEOUT_MS = 1e4;
|
|
9377
|
+
var GIT_CONFIG_GLOBAL = "/tmp/gitconfig";
|
|
9378
|
+
var GIT_CREDENTIAL_HELPER = "/tmp/git-credential-helper.sh";
|
|
9379
|
+
function commandError2(result) {
|
|
9380
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
9381
|
+
}
|
|
9382
|
+
var runCommand2 = (command, args, opts) => {
|
|
9383
|
+
return new Promise((resolve4) => {
|
|
9384
|
+
let child;
|
|
9385
|
+
let stdout = "";
|
|
9386
|
+
let stderr = "";
|
|
9387
|
+
let settled = false;
|
|
9388
|
+
const timer = {};
|
|
9389
|
+
const finish = (result) => {
|
|
9390
|
+
if (settled) return;
|
|
9391
|
+
settled = true;
|
|
9392
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
9393
|
+
resolve4(result);
|
|
9394
|
+
};
|
|
9395
|
+
try {
|
|
9396
|
+
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
9397
|
+
} catch (error2) {
|
|
9398
|
+
finish({
|
|
9399
|
+
code: null,
|
|
9400
|
+
stdout,
|
|
9401
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
9402
|
+
timedOut: false
|
|
9403
|
+
});
|
|
9404
|
+
return;
|
|
9405
|
+
}
|
|
9406
|
+
child.stdout?.setEncoding("utf8");
|
|
9407
|
+
child.stdout?.on("data", (chunk) => {
|
|
9408
|
+
stdout += chunk;
|
|
9409
|
+
});
|
|
9410
|
+
child.stderr?.setEncoding("utf8");
|
|
9411
|
+
child.stderr?.on("data", (chunk) => {
|
|
9412
|
+
stderr += chunk;
|
|
9413
|
+
});
|
|
9414
|
+
child.once("error", (error2) => {
|
|
9415
|
+
finish({
|
|
9416
|
+
code: null,
|
|
9417
|
+
stdout,
|
|
9418
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
9419
|
+
${error2.message}`,
|
|
9420
|
+
timedOut: false
|
|
9421
|
+
});
|
|
9422
|
+
});
|
|
9423
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
9424
|
+
timer.handle = setTimeout(
|
|
9425
|
+
() => {
|
|
9426
|
+
child.kill("SIGKILL");
|
|
9427
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
9428
|
+
},
|
|
9429
|
+
Math.max(0, opts.timeoutMs)
|
|
9430
|
+
);
|
|
9431
|
+
});
|
|
9432
|
+
};
|
|
9433
|
+
function isEnvironmentObject(value) {
|
|
9434
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9435
|
+
}
|
|
9436
|
+
function secretFailure(marker, detail, log3) {
|
|
9437
|
+
const message = `${marker}: ${detail}`;
|
|
9438
|
+
log3(message, "error");
|
|
9439
|
+
return new Error(message);
|
|
9440
|
+
}
|
|
9441
|
+
async function installRunnerSecret({
|
|
9442
|
+
env,
|
|
9443
|
+
log: log3,
|
|
9444
|
+
commandRunner
|
|
9445
|
+
}) {
|
|
9446
|
+
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
9447
|
+
if (!arn) {
|
|
9448
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
9449
|
+
return false;
|
|
9450
|
+
}
|
|
9451
|
+
const result = await (commandRunner ?? runCommand2)(
|
|
9452
|
+
"aws",
|
|
9453
|
+
[
|
|
9454
|
+
"secretsmanager",
|
|
9455
|
+
"get-secret-value",
|
|
9456
|
+
"--secret-id",
|
|
9457
|
+
arn,
|
|
9458
|
+
"--query",
|
|
9459
|
+
"SecretString",
|
|
9460
|
+
"--output",
|
|
9461
|
+
"text"
|
|
9462
|
+
],
|
|
9463
|
+
{ env, timeoutMs: RUNNER_SECRET_FETCH_TIMEOUT_MS }
|
|
9464
|
+
);
|
|
9465
|
+
if (result.timedOut) {
|
|
9466
|
+
throw secretFailure(
|
|
9467
|
+
"CREDENTIAL-RESTORE-TIMEOUT",
|
|
9468
|
+
`runner-secret did not finish within ${RUNNER_SECRET_FETCH_TIMEOUT_MS}ms`,
|
|
9469
|
+
log3
|
|
9470
|
+
);
|
|
9471
|
+
}
|
|
9472
|
+
if (result.code !== 0) {
|
|
9473
|
+
throw secretFailure("RUNNER-SECRET-UNREADABLE", commandError2(result), log3);
|
|
9474
|
+
}
|
|
9475
|
+
let payload;
|
|
9476
|
+
try {
|
|
9477
|
+
payload = JSON.parse(result.stdout);
|
|
9478
|
+
} catch (error2) {
|
|
9479
|
+
log3(
|
|
9480
|
+
`RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9481
|
+
"warn"
|
|
9482
|
+
);
|
|
9483
|
+
return false;
|
|
9484
|
+
}
|
|
9485
|
+
if (!isEnvironmentObject(payload)) {
|
|
9486
|
+
log3("RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object", "warn");
|
|
9487
|
+
return false;
|
|
9488
|
+
}
|
|
9489
|
+
let populated = 0;
|
|
9490
|
+
let skipped = 0;
|
|
9491
|
+
let githubTokenPopulated = false;
|
|
9492
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
9493
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
9494
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
9495
|
+
log3(
|
|
9496
|
+
`RUNNER-SECRET-KEY-SKIPPED: ${JSON.stringify(key)} is not a valid environment variable name`,
|
|
9497
|
+
"warn"
|
|
9498
|
+
);
|
|
9499
|
+
skipped += 1;
|
|
9500
|
+
continue;
|
|
9501
|
+
}
|
|
9502
|
+
env[key] = value;
|
|
9503
|
+
populated += 1;
|
|
9504
|
+
if (key === "GH_TOKEN") githubTokenPopulated = true;
|
|
9505
|
+
}
|
|
9506
|
+
if (populated === 0) {
|
|
9507
|
+
log3(
|
|
9508
|
+
"RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md",
|
|
9509
|
+
"warn"
|
|
9510
|
+
);
|
|
9511
|
+
} else {
|
|
9512
|
+
log3(
|
|
9513
|
+
`RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names`
|
|
9514
|
+
);
|
|
9515
|
+
}
|
|
9516
|
+
return githubTokenPopulated;
|
|
9517
|
+
}
|
|
9518
|
+
function restoreFailure(operation, result, log3) {
|
|
9519
|
+
const message = `CREDENTIAL-RESTORE-FAILED: ${operation} failed: ${commandError2(result)}`;
|
|
9520
|
+
log3(message, "error");
|
|
9521
|
+
return new Error(message);
|
|
9522
|
+
}
|
|
9523
|
+
async function runCredentialRestore(args, operation, log3, synchroniserRunner) {
|
|
9524
|
+
const result = await synchroniserRunner(args, { timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS });
|
|
9525
|
+
if (result.timedOut) {
|
|
9526
|
+
log3(
|
|
9527
|
+
`CREDENTIAL-RESTORE-TIMEOUT: ${operation} did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9528
|
+
"warn"
|
|
9529
|
+
);
|
|
9530
|
+
return result;
|
|
9531
|
+
}
|
|
9532
|
+
if (result.code !== 0) throw restoreFailure(operation, result, log3);
|
|
9533
|
+
return result;
|
|
9534
|
+
}
|
|
9535
|
+
async function restoreCredentialStores({
|
|
9536
|
+
env,
|
|
9537
|
+
log: log3,
|
|
9538
|
+
synchroniserRunner = runSynchroniser
|
|
9539
|
+
}) {
|
|
9540
|
+
await runCredentialRestore(["restore", "claude"], "restore-claude", log3, synchroniserRunner);
|
|
9541
|
+
await runCredentialRestore(["restore", "opencode"], "restore-opencode", log3, synchroniserRunner);
|
|
9542
|
+
const result = await synchroniserRunner(["model-auth-ready"], {
|
|
9543
|
+
timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS
|
|
9544
|
+
});
|
|
9545
|
+
if (result.timedOut) {
|
|
9546
|
+
log3(
|
|
9547
|
+
`CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9548
|
+
"warn"
|
|
9549
|
+
);
|
|
9550
|
+
return;
|
|
9551
|
+
}
|
|
9552
|
+
switch (result.code) {
|
|
9553
|
+
case 0:
|
|
9554
|
+
return;
|
|
9555
|
+
case 10:
|
|
9556
|
+
log3(
|
|
9557
|
+
`no model credentials under s3://${env.LITESTREAM_BUCKET ?? ""}/${env.LITESTREAM_PREFIX ?? ""}/ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs a model provider fails until one is connected. See 'Seeding a credential store' in infrastructure/evident-runner/MICROVM.md.`,
|
|
9558
|
+
"warn"
|
|
9559
|
+
);
|
|
9560
|
+
return;
|
|
9561
|
+
default:
|
|
9562
|
+
log3("could not determine whether this VM has model credentials", "warn");
|
|
9563
|
+
}
|
|
9564
|
+
}
|
|
9565
|
+
var GIT_CREDENTIAL_HELPER_CONTENT = [
|
|
9566
|
+
"#!/usr/bin/env bash",
|
|
9567
|
+
'[ "$1" = get ] || exit 0',
|
|
9568
|
+
"echo username=x-access-token",
|
|
9569
|
+
'echo "password=${GH_TOKEN}"',
|
|
9570
|
+
""
|
|
9571
|
+
].join("\n");
|
|
9572
|
+
async function probeGitHubAccess({ env, log: log3 }) {
|
|
9573
|
+
const auth = await runCommand2("gh", ["api", "user", "--jq", ".login"], {
|
|
9574
|
+
env,
|
|
9575
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9576
|
+
});
|
|
9577
|
+
if (auth.timedOut) {
|
|
9578
|
+
log3(`GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`, "warn");
|
|
9579
|
+
return;
|
|
9580
|
+
}
|
|
9581
|
+
if (auth.code !== 0) {
|
|
9582
|
+
log3(`GITHUB-AUTH-REJECTED: ${commandError2(auth)}`, "warn");
|
|
9583
|
+
return;
|
|
9584
|
+
}
|
|
9585
|
+
log3(`GITHUB-AUTH-OK: ${auth.stdout.trim()}`, "debug");
|
|
9586
|
+
const remote = await runCommand2("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
|
9587
|
+
env,
|
|
9588
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9589
|
+
});
|
|
9590
|
+
if (remote.code !== 0 || remote.timedOut) return;
|
|
9591
|
+
const repo = remote.stdout.trim().replace(/^(?:https:\/\/github\.com\/|git@github\.com:)/, "").replace(/\.git$/, "");
|
|
9592
|
+
if (!repo) return;
|
|
9593
|
+
const repository = await runCommand2("gh", ["api", `repos/${repo}`, "--jq", ".full_name"], {
|
|
9594
|
+
env,
|
|
9595
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9596
|
+
});
|
|
9597
|
+
if (repository.timedOut) {
|
|
9598
|
+
log3(
|
|
9599
|
+
`GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`,
|
|
9600
|
+
"warn"
|
|
9601
|
+
);
|
|
9602
|
+
} else if (repository.code !== 0) {
|
|
9603
|
+
log3(`GITHUB-REPO-INACCESSIBLE: ${repo}: ${commandError2(repository)}`, "warn");
|
|
9604
|
+
}
|
|
9605
|
+
}
|
|
9606
|
+
async function configureGitHubAccess({ env, log: log3 }) {
|
|
9607
|
+
if (!env.GH_TOKEN) {
|
|
9608
|
+
log3("GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above", "warn");
|
|
9609
|
+
return;
|
|
9610
|
+
}
|
|
9611
|
+
try {
|
|
9612
|
+
env.GIT_CONFIG_GLOBAL = GIT_CONFIG_GLOBAL;
|
|
9613
|
+
writeFileSync4(GIT_CONFIG_GLOBAL, "");
|
|
9614
|
+
writeFileSync4(GIT_CREDENTIAL_HELPER, GIT_CREDENTIAL_HELPER_CONTENT, { mode: 448 });
|
|
9615
|
+
chmodSync2(GIT_CREDENTIAL_HELPER, 448);
|
|
9616
|
+
const config = [
|
|
9617
|
+
["user.name", env.GIT_USER_NAME ?? "evident-bot"],
|
|
9618
|
+
["user.email", env.GIT_USER_EMAIL ?? "evident-bot@users.noreply.github.com"],
|
|
9619
|
+
["init.defaultBranch", "main"],
|
|
9620
|
+
["credential.https://github.com.helper", GIT_CREDENTIAL_HELPER]
|
|
9621
|
+
];
|
|
9622
|
+
for (const [key, value] of config) {
|
|
9623
|
+
const result = await runCommand2("git", ["config", "--global", key, value], {
|
|
9624
|
+
env,
|
|
9625
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9626
|
+
});
|
|
9627
|
+
if (result.timedOut || result.code !== 0) throw new Error(commandError2(result));
|
|
9628
|
+
}
|
|
9629
|
+
} catch (error2) {
|
|
9630
|
+
log3(
|
|
9631
|
+
`GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9632
|
+
"warn"
|
|
9633
|
+
);
|
|
9634
|
+
return;
|
|
9635
|
+
}
|
|
9636
|
+
void probeGitHubAccess({ env, log: log3 }).catch((error2) => {
|
|
9637
|
+
log3(
|
|
9638
|
+
`GITHUB-SETUP-FAILED: GitHub access probe failed: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9639
|
+
"warn"
|
|
9640
|
+
);
|
|
9641
|
+
});
|
|
9642
|
+
}
|
|
9643
|
+
|
|
9644
|
+
// src/lib/opencode/config-overlay.ts
|
|
9645
|
+
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
9646
|
+
import { copyFileSync, existsSync as existsSync2, statSync as statSync6 } from "node:fs";
|
|
9647
|
+
import { isAbsolute as isAbsolute2, join as join9, resolve as resolve3 } from "node:path";
|
|
9648
|
+
function isFile(filePath) {
|
|
9649
|
+
return existsSync2(filePath) && statSync6(filePath).isFile();
|
|
9650
|
+
}
|
|
9651
|
+
function applyRunnerOpenCodeConfig({
|
|
9652
|
+
overlayPath,
|
|
9653
|
+
cwd = process.cwd(),
|
|
9654
|
+
log: log3
|
|
9655
|
+
}) {
|
|
9656
|
+
if (!overlayPath) {
|
|
9657
|
+
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
9658
|
+
return;
|
|
9659
|
+
}
|
|
9660
|
+
const source = isAbsolute2(overlayPath) ? overlayPath : resolve3(cwd, overlayPath);
|
|
9661
|
+
const target = isFile(join9(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
9662
|
+
if (!isFile(source)) {
|
|
9663
|
+
log3(
|
|
9664
|
+
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
9665
|
+
"error"
|
|
9666
|
+
);
|
|
9667
|
+
return;
|
|
9668
|
+
}
|
|
9669
|
+
copyFileSync(source, join9(cwd, target));
|
|
9670
|
+
try {
|
|
9671
|
+
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
9672
|
+
stdio: "ignore"
|
|
9673
|
+
});
|
|
9674
|
+
} catch (error2) {
|
|
9675
|
+
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
9676
|
+
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
9677
|
+
}
|
|
9678
|
+
log3(`Applied runner OpenCode config ${source} to ${join9(cwd, target)}`);
|
|
9679
|
+
}
|
|
9680
|
+
|
|
9681
|
+
// src/lib/credential-sync.ts
|
|
9682
|
+
import { renameSync, writeFileSync as writeFileSync5 } from "node:fs";
|
|
9683
|
+
var CREDENTIAL_SYNC_TICK_TIMEOUT_MS = 3e4;
|
|
9684
|
+
var CREDENTIAL_FLUSH_DEADLINE_MS = 8e3;
|
|
9685
|
+
var CREDENTIAL_FLUSH_ABORT_GRACE_MS = 500;
|
|
9686
|
+
var DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS = 60;
|
|
9687
|
+
var STORES = ["claude", "opencode"];
|
|
9688
|
+
var MAX_FLUSH_PASSES = 2;
|
|
9689
|
+
function outcomesWith(outcome) {
|
|
9690
|
+
return { claude: outcome, opencode: outcome };
|
|
9691
|
+
}
|
|
9692
|
+
function errorMessage2(error2) {
|
|
9693
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
9694
|
+
}
|
|
9695
|
+
function waitForSettlement(promise, timeoutMs) {
|
|
9696
|
+
return new Promise((resolve4) => {
|
|
9697
|
+
let settled = false;
|
|
9698
|
+
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
9699
|
+
const finish = (value) => {
|
|
9700
|
+
if (settled) return;
|
|
9701
|
+
settled = true;
|
|
9702
|
+
clearTimeout(timer);
|
|
9703
|
+
resolve4(value);
|
|
9704
|
+
};
|
|
9705
|
+
promise.then(
|
|
9706
|
+
() => finish(true),
|
|
9707
|
+
() => finish(true)
|
|
9708
|
+
);
|
|
9709
|
+
});
|
|
9710
|
+
}
|
|
9711
|
+
function writeMarker(markerPath, outcomes, log3) {
|
|
9712
|
+
const body = `${STORES.map((store) => `${store}=${outcomes[store]}`).join("\n")}
|
|
9713
|
+
`;
|
|
9714
|
+
const temporaryPath = `${markerPath}.tmp`;
|
|
9715
|
+
try {
|
|
9716
|
+
writeFileSync5(temporaryPath, body, { mode: 384 });
|
|
9717
|
+
renameSync(temporaryPath, markerPath);
|
|
9718
|
+
} catch (error2) {
|
|
9719
|
+
log3(`CREDENTIAL-FLUSH-MARKER-UNWRITABLE: ${markerPath}: ${errorMessage2(error2)}`, "warn");
|
|
9720
|
+
}
|
|
9721
|
+
}
|
|
9722
|
+
function intervalSeconds(env, log3) {
|
|
9723
|
+
const raw = env.CREDS_SYNC_INTERVAL;
|
|
9724
|
+
if (raw === void 0 || /^[1-9][0-9]*$/.test(raw) && Number.isSafeInteger(Number(raw))) {
|
|
9725
|
+
return raw === void 0 ? DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS : Number(raw);
|
|
9726
|
+
}
|
|
9727
|
+
log3(
|
|
9728
|
+
`CREDENTIAL-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${raw}' is not a positive integer; using ${DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS}s`,
|
|
9729
|
+
"warn"
|
|
9730
|
+
);
|
|
9731
|
+
return DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS;
|
|
9732
|
+
}
|
|
9733
|
+
async function runFlushStore(store, env, synchroniserRunner, log3, deadlineAt) {
|
|
9734
|
+
const remainingMs = deadlineAt - Date.now();
|
|
9735
|
+
if (remainingMs <= 0) return { outcome: "timeout", orphaned: false };
|
|
9736
|
+
const controller = new AbortController();
|
|
9737
|
+
let result;
|
|
9738
|
+
let failed = false;
|
|
9739
|
+
const completion = Promise.resolve().then(
|
|
9740
|
+
() => synchroniserRunner(["sync-once", store], {
|
|
9741
|
+
timeoutMs: remainingMs,
|
|
9742
|
+
env,
|
|
9743
|
+
signal: controller.signal
|
|
9744
|
+
})
|
|
9745
|
+
).then(
|
|
9746
|
+
(value) => {
|
|
9747
|
+
result = value;
|
|
9748
|
+
},
|
|
9749
|
+
(error2) => {
|
|
9750
|
+
failed = true;
|
|
9751
|
+
log3(`CREDENTIAL-FLUSH-ERROR: ${store}: ${errorMessage2(error2)}`, "warn");
|
|
9752
|
+
}
|
|
9753
|
+
);
|
|
9754
|
+
const abortTimer = setTimeout(() => controller.abort(), remainingMs);
|
|
9755
|
+
const settledBeforeDeadline = await waitForSettlement(completion, remainingMs);
|
|
9756
|
+
clearTimeout(abortTimer);
|
|
9757
|
+
if (!settledBeforeDeadline) {
|
|
9758
|
+
controller.abort();
|
|
9759
|
+
const settledAfterAbort = await waitForSettlement(completion, CREDENTIAL_FLUSH_ABORT_GRACE_MS);
|
|
9760
|
+
if (!settledAfterAbort) return { outcome: "timeout", orphaned: true };
|
|
9761
|
+
return { outcome: "timeout", orphaned: false };
|
|
9762
|
+
}
|
|
9763
|
+
if (failed || !result) return { outcome: "failed", orphaned: false };
|
|
9764
|
+
if (result.timedOut || Date.now() >= deadlineAt) {
|
|
9765
|
+
return { outcome: "timeout", orphaned: false };
|
|
9766
|
+
}
|
|
9767
|
+
return { outcome: result.code === 0 ? "ok" : "failed", orphaned: false };
|
|
9768
|
+
}
|
|
9769
|
+
function createCredentialSync({
|
|
9770
|
+
markerPath,
|
|
9771
|
+
env,
|
|
9772
|
+
log: log3,
|
|
9773
|
+
synchroniserRunner = runSynchroniser
|
|
9774
|
+
}) {
|
|
9775
|
+
const persistenceDisabled = !env.PERSISTENCE_BUCKET;
|
|
9776
|
+
let disabled = persistenceDisabled;
|
|
9777
|
+
let armed = false;
|
|
9778
|
+
let stopped = false;
|
|
9779
|
+
let timer;
|
|
9780
|
+
let inFlight;
|
|
9781
|
+
let activeTickAbort;
|
|
9782
|
+
let lastTickFailed;
|
|
9783
|
+
let flushPromise;
|
|
9784
|
+
const scheduleTick = (intervalMs, startTick2) => {
|
|
9785
|
+
if (stopped) return;
|
|
9786
|
+
timer = setTimeout(() => {
|
|
9787
|
+
timer = void 0;
|
|
9788
|
+
startTick2();
|
|
9789
|
+
}, intervalMs);
|
|
9790
|
+
};
|
|
9791
|
+
const startTick = (intervalMs) => {
|
|
9792
|
+
if (stopped) return;
|
|
9793
|
+
const controller = new AbortController();
|
|
9794
|
+
activeTickAbort = controller;
|
|
9795
|
+
const tick = (async () => {
|
|
9796
|
+
const outcomes = {
|
|
9797
|
+
claude: "failed",
|
|
9798
|
+
opencode: "failed"
|
|
9799
|
+
};
|
|
9800
|
+
for (const store of STORES) {
|
|
9801
|
+
if (controller.signal.aborted) break;
|
|
9802
|
+
try {
|
|
9803
|
+
const result = await synchroniserRunner(["sync-once", store], {
|
|
9804
|
+
timeoutMs: CREDENTIAL_SYNC_TICK_TIMEOUT_MS,
|
|
9805
|
+
env,
|
|
9806
|
+
signal: controller.signal
|
|
9807
|
+
});
|
|
9808
|
+
outcomes[store] = !result.timedOut && result.code === 0 ? "ok" : "failed";
|
|
9809
|
+
} catch (error2) {
|
|
9810
|
+
outcomes[store] = "failed";
|
|
9811
|
+
log3(`CREDENTIAL-SYNC-TICK-ERROR: ${store}: ${errorMessage2(error2)}`, "debug");
|
|
9812
|
+
}
|
|
9813
|
+
}
|
|
9814
|
+
const failed = STORES.some((store) => outcomes[store] === "failed");
|
|
9815
|
+
log3(
|
|
9816
|
+
`CREDENTIAL-SYNC-TICK: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
9817
|
+
"debug"
|
|
9818
|
+
);
|
|
9819
|
+
if (failed && lastTickFailed !== true) {
|
|
9820
|
+
log3(
|
|
9821
|
+
"CREDENTIAL-SYNC-FAILED: an interval credential sync failed; retrying on the next tick",
|
|
9822
|
+
"warn"
|
|
9823
|
+
);
|
|
9824
|
+
} else if (!failed && lastTickFailed === true) {
|
|
9825
|
+
log3("CREDENTIAL-SYNC-RECOVERED: interval credential sync succeeded again", "warn");
|
|
9826
|
+
}
|
|
9827
|
+
lastTickFailed = failed;
|
|
9828
|
+
})().finally(() => {
|
|
9829
|
+
if (activeTickAbort === controller) activeTickAbort = void 0;
|
|
9830
|
+
if (inFlight === tick) inFlight = void 0;
|
|
9831
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9832
|
+
});
|
|
9833
|
+
inFlight = tick;
|
|
9834
|
+
};
|
|
9835
|
+
const performFlush = async () => {
|
|
9836
|
+
stopped = true;
|
|
9837
|
+
if (timer) {
|
|
9838
|
+
clearTimeout(timer);
|
|
9839
|
+
timer = void 0;
|
|
9840
|
+
}
|
|
9841
|
+
const deadlineAt = Date.now() + CREDENTIAL_FLUSH_DEADLINE_MS;
|
|
9842
|
+
if (inFlight) {
|
|
9843
|
+
const settled = await waitForSettlement(inFlight, CREDENTIAL_FLUSH_DEADLINE_MS);
|
|
9844
|
+
if (!settled) {
|
|
9845
|
+
activeTickAbort?.abort();
|
|
9846
|
+
const settledAfterAbort = await waitForSettlement(
|
|
9847
|
+
inFlight,
|
|
9848
|
+
CREDENTIAL_FLUSH_ABORT_GRACE_MS
|
|
9849
|
+
);
|
|
9850
|
+
if (!settledAfterAbort) {
|
|
9851
|
+
log3(
|
|
9852
|
+
"CREDENTIAL-FLUSH-ORPHANED-TICK: a sync-once child could not be proven settled; skipping the boundary flush rather than becoming a second writer",
|
|
9853
|
+
"warn"
|
|
9854
|
+
);
|
|
9855
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9856
|
+
}
|
|
8001
9857
|
}
|
|
8002
9858
|
}
|
|
8003
|
-
|
|
8004
|
-
|
|
8005
|
-
|
|
8006
|
-
|
|
8007
|
-
{
|
|
8008
|
-
|
|
8009
|
-
|
|
8010
|
-
|
|
8011
|
-
|
|
8012
|
-
|
|
8013
|
-
name: "Show me the command",
|
|
8014
|
-
value: "manual",
|
|
8015
|
-
description: "Display the command to run manually"
|
|
8016
|
-
},
|
|
8017
|
-
{
|
|
8018
|
-
name: "Continue without OpenCode",
|
|
8019
|
-
value: "continue",
|
|
8020
|
-
description: "Requests will fail until OpenCode starts"
|
|
9859
|
+
if (disabled) return { outcomes: outcomesWith("skipped"), orphaned: false };
|
|
9860
|
+
const outcomes = outcomesWith("timeout");
|
|
9861
|
+
for (const store of STORES) {
|
|
9862
|
+
const result = await runFlushStore(store, env, synchroniserRunner, log3, deadlineAt);
|
|
9863
|
+
if (result.orphaned) {
|
|
9864
|
+
log3(
|
|
9865
|
+
"CREDENTIAL-FLUSH-ORPHANED-CHILD: a flush sync-once child could not be proven settled; refusing another flush pass rather than becoming a second writer",
|
|
9866
|
+
"warn"
|
|
9867
|
+
);
|
|
9868
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
8021
9869
|
}
|
|
8022
|
-
|
|
8023
|
-
});
|
|
8024
|
-
if (action === "manual") {
|
|
8025
|
-
blank();
|
|
8026
|
-
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
8027
|
-
blank();
|
|
8028
|
-
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
8029
|
-
blank();
|
|
8030
|
-
throw new Error("Please start OpenCode manually");
|
|
8031
|
-
}
|
|
8032
|
-
if (action === "start") {
|
|
8033
|
-
const spinner = ora2("Starting OpenCode...").start();
|
|
8034
|
-
const proc = await startOpenCode(port);
|
|
8035
|
-
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
8036
|
-
if (!health.healthy) {
|
|
8037
|
-
spinner.fail("Failed to start OpenCode");
|
|
8038
|
-
throw new Error("OpenCode failed to start");
|
|
9870
|
+
outcomes[store] = result.outcome;
|
|
8039
9871
|
}
|
|
8040
|
-
|
|
8041
|
-
|
|
8042
|
-
|
|
8043
|
-
|
|
9872
|
+
return { outcomes, orphaned: false };
|
|
9873
|
+
};
|
|
9874
|
+
let flushPasses = 0;
|
|
9875
|
+
let lastFlush;
|
|
9876
|
+
return {
|
|
9877
|
+
arm() {
|
|
9878
|
+
if (stopped || armed) return;
|
|
9879
|
+
armed = true;
|
|
9880
|
+
if (persistenceDisabled) {
|
|
9881
|
+
disabled = true;
|
|
9882
|
+
log3(
|
|
9883
|
+
"CREDENTIAL-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot",
|
|
9884
|
+
"warn"
|
|
9885
|
+
);
|
|
9886
|
+
return;
|
|
9887
|
+
}
|
|
9888
|
+
disabled = false;
|
|
9889
|
+
const intervalMs = intervalSeconds(env, log3) * 1e3;
|
|
9890
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9891
|
+
},
|
|
9892
|
+
async stopAndFlush(publish) {
|
|
9893
|
+
let result;
|
|
9894
|
+
const runningFlush = flushPromise;
|
|
9895
|
+
if (runningFlush) {
|
|
9896
|
+
result = await runningFlush;
|
|
9897
|
+
} else if (flushPasses >= MAX_FLUSH_PASSES || lastFlush?.orphaned) {
|
|
9898
|
+
result = lastFlush ?? { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9899
|
+
} else {
|
|
9900
|
+
flushPasses++;
|
|
9901
|
+
const currentFlush = performFlush();
|
|
9902
|
+
flushPromise = currentFlush;
|
|
9903
|
+
try {
|
|
9904
|
+
result = await currentFlush;
|
|
9905
|
+
lastFlush = result;
|
|
9906
|
+
} finally {
|
|
9907
|
+
if (flushPromise === currentFlush) flushPromise = void 0;
|
|
9908
|
+
}
|
|
9909
|
+
}
|
|
9910
|
+
if (publish) writeMarker(markerPath, result.outcomes, log3);
|
|
9911
|
+
return result.outcomes;
|
|
9912
|
+
}
|
|
9913
|
+
};
|
|
8044
9914
|
}
|
|
8045
9915
|
|
|
8046
9916
|
// src/commands/run.ts
|
|
@@ -8080,11 +9950,11 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
8080
9950
|
if (trimmed === "") {
|
|
8081
9951
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
8082
9952
|
}
|
|
8083
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
8084
|
-
if (!
|
|
9953
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join10(homeDir, trimmed.slice(2)) : trimmed;
|
|
9954
|
+
if (!isAbsolute3(expanded)) {
|
|
8085
9955
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
8086
9956
|
}
|
|
8087
|
-
const normalized =
|
|
9957
|
+
const normalized = resolvePath2(expanded);
|
|
8088
9958
|
if (parse(normalized).root === normalized) {
|
|
8089
9959
|
throw new Error(
|
|
8090
9960
|
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
@@ -8104,6 +9974,28 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
8104
9974
|
var DEFAULT_OPENCODE_START_TIMEOUT_SECONDS = 180;
|
|
8105
9975
|
var MAX_OPENCODE_START_TIMEOUT_SECONDS = 3600;
|
|
8106
9976
|
var OPENCODE_START_TIMEOUT_ENV = "EVIDENT_OPENCODE_START_TIMEOUT";
|
|
9977
|
+
var OPENCODE_VERSION_ENV = "EVIDENT_OPENCODE_VERSION";
|
|
9978
|
+
function resolveOpenCodeVersion(options, env = process.env) {
|
|
9979
|
+
let raw;
|
|
9980
|
+
let source;
|
|
9981
|
+
if (options.opencodeVersion !== void 0) {
|
|
9982
|
+
raw = options.opencodeVersion;
|
|
9983
|
+
source = "--opencode-version";
|
|
9984
|
+
} else if (env[OPENCODE_VERSION_ENV] !== void 0 && env[OPENCODE_VERSION_ENV] !== "") {
|
|
9985
|
+
raw = env[OPENCODE_VERSION_ENV];
|
|
9986
|
+
source = OPENCODE_VERSION_ENV;
|
|
9987
|
+
} else {
|
|
9988
|
+
return { version: "v1", warnings: [] };
|
|
9989
|
+
}
|
|
9990
|
+
const normalized = raw.trim().toLowerCase();
|
|
9991
|
+
if (normalized !== "v1" && normalized !== "v2") {
|
|
9992
|
+
return {
|
|
9993
|
+
version: "v1",
|
|
9994
|
+
warnings: [`Ignoring invalid ${source} "${raw}": expected v1 or v2; using the default v1`]
|
|
9995
|
+
};
|
|
9996
|
+
}
|
|
9997
|
+
return { version: normalized, warnings: [] };
|
|
9998
|
+
}
|
|
8107
9999
|
function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
8108
10000
|
const defaultMs = DEFAULT_OPENCODE_START_TIMEOUT_SECONDS * 1e3;
|
|
8109
10001
|
let raw;
|
|
@@ -8170,7 +10062,7 @@ function log2(state, message, level = "info") {
|
|
|
8170
10062
|
})
|
|
8171
10063
|
);
|
|
8172
10064
|
} else if (!state.interactive) {
|
|
8173
|
-
const prefix = level === "error" ?
|
|
10065
|
+
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
8174
10066
|
console.log(`${prefix} ${message}`);
|
|
8175
10067
|
}
|
|
8176
10068
|
}
|
|
@@ -8200,7 +10092,7 @@ function logActivity(state, entry) {
|
|
|
8200
10092
|
}
|
|
8201
10093
|
function reportSessionDbRecovery(state) {
|
|
8202
10094
|
try {
|
|
8203
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
10095
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir6(), env: process.env });
|
|
8204
10096
|
for (const record of report.records) {
|
|
8205
10097
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
8206
10098
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -8218,21 +10110,31 @@ function reportSessionDbRecovery(state) {
|
|
|
8218
10110
|
);
|
|
8219
10111
|
}
|
|
8220
10112
|
}
|
|
10113
|
+
function reportSessionDbRecoveryRecord(state, record) {
|
|
10114
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10115
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
10116
|
+
logActivity(state, {
|
|
10117
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10118
|
+
level: activity.level,
|
|
10119
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10120
|
+
metadata: activity.metadata
|
|
10121
|
+
});
|
|
10122
|
+
}
|
|
8221
10123
|
function displayStatus(state) {
|
|
8222
10124
|
if (!state.interactive) return;
|
|
8223
10125
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
8224
|
-
const tunnel = state.connected ?
|
|
8225
|
-
const opencode = state.opencodeConnected ?
|
|
8226
|
-
const messages = state.messageCount > 0 ?
|
|
10126
|
+
const tunnel = state.connected ? chalk7.green("tunnel: connected") : attempt > 0 ? chalk7.yellow(`tunnel: reconnecting (#${attempt})`) : chalk7.yellow("tunnel: connecting");
|
|
10127
|
+
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
10128
|
+
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
8227
10129
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
8228
|
-
const detail = last ?
|
|
10130
|
+
const detail = last ? chalk7.dim(` \xB7 ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`) : "";
|
|
8229
10131
|
const agent = state.agentName ?? state.agentId;
|
|
8230
10132
|
console.log(
|
|
8231
|
-
`${
|
|
10133
|
+
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
8232
10134
|
);
|
|
8233
10135
|
}
|
|
8234
10136
|
async function promptForLogin(promptMessage, successMessage) {
|
|
8235
|
-
const action = await
|
|
10137
|
+
const action = await select4({
|
|
8236
10138
|
message: promptMessage,
|
|
8237
10139
|
choices: [
|
|
8238
10140
|
{
|
|
@@ -8248,7 +10150,7 @@ async function promptForLogin(promptMessage, successMessage) {
|
|
|
8248
10150
|
]
|
|
8249
10151
|
});
|
|
8250
10152
|
if (action === "exit") {
|
|
8251
|
-
console.log(
|
|
10153
|
+
console.log(chalk7.dim(`
|
|
8252
10154
|
You can log in later by running: ${getCliName()} login`));
|
|
8253
10155
|
process.exit(0);
|
|
8254
10156
|
}
|
|
@@ -8259,7 +10161,7 @@ You can log in later by running: ${getCliName()} login`));
|
|
|
8259
10161
|
process.exit(1);
|
|
8260
10162
|
}
|
|
8261
10163
|
blank();
|
|
8262
|
-
console.log(
|
|
10164
|
+
console.log(chalk7.green(successMessage));
|
|
8263
10165
|
blank();
|
|
8264
10166
|
return { token: credentials2.token, authType: "bearer", user: credentials2.user };
|
|
8265
10167
|
}
|
|
@@ -8272,12 +10174,12 @@ async function handleAuthError(state, error2) {
|
|
|
8272
10174
|
if (state.interactive) displayStatus(state);
|
|
8273
10175
|
if (!state.interactive) {
|
|
8274
10176
|
blank();
|
|
8275
|
-
console.log(
|
|
8276
|
-
console.log(
|
|
10177
|
+
console.log(chalk7.red("Authentication expired"));
|
|
10178
|
+
console.log(chalk7.dim("Your authentication token is no longer valid."));
|
|
8277
10179
|
blank();
|
|
8278
|
-
console.log(
|
|
8279
|
-
console.log(
|
|
8280
|
-
console.log(
|
|
10180
|
+
console.log(chalk7.dim("To fix this:"));
|
|
10181
|
+
console.log(chalk7.dim(` 1. Run '${getCliName()} login' to re-authenticate`));
|
|
10182
|
+
console.log(chalk7.dim(" 2. Restart this command"));
|
|
8281
10183
|
blank();
|
|
8282
10184
|
await cleanup(state);
|
|
8283
10185
|
await shutdownTelemetry();
|
|
@@ -8285,7 +10187,7 @@ async function handleAuthError(state, error2) {
|
|
|
8285
10187
|
return { success: false };
|
|
8286
10188
|
}
|
|
8287
10189
|
blank();
|
|
8288
|
-
console.log(
|
|
10190
|
+
console.log(chalk7.yellow("Your authentication has expired."));
|
|
8289
10191
|
blank();
|
|
8290
10192
|
try {
|
|
8291
10193
|
const credentials2 = await promptForLogin(
|
|
@@ -8330,6 +10232,10 @@ async function driveChannels(state, driver) {
|
|
|
8330
10232
|
consecutiveDrainFailures = 0;
|
|
8331
10233
|
unreachableMs = 0;
|
|
8332
10234
|
state.messageCount += processed;
|
|
10235
|
+
if (driver.recycleRequested) {
|
|
10236
|
+
await beginGracefulShutdown(state, "recycle");
|
|
10237
|
+
return;
|
|
10238
|
+
}
|
|
8333
10239
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
8334
10240
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
8335
10241
|
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
@@ -8345,6 +10251,14 @@ async function driveChannels(state, driver) {
|
|
|
8345
10251
|
const opencodeAuthApplied = opencodeAuthApplies !== lastSeenOpencodeAuthApplies;
|
|
8346
10252
|
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
8347
10253
|
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
10254
|
+
if (claudeCredentialApplied || opencodeAuthApplied) {
|
|
10255
|
+
void reloadProviderCache(state.port).catch(
|
|
10256
|
+
(error2) => logActivity(state, {
|
|
10257
|
+
type: "error",
|
|
10258
|
+
error: `OpenCode provider cache reload failed on port ${state.port}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10259
|
+
})
|
|
10260
|
+
);
|
|
10261
|
+
}
|
|
8348
10262
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
8349
10263
|
idlePolls = 0;
|
|
8350
10264
|
idleMs = 0;
|
|
@@ -8372,8 +10286,8 @@ async function driveChannels(state, driver) {
|
|
|
8372
10286
|
state.running = false;
|
|
8373
10287
|
break;
|
|
8374
10288
|
}
|
|
8375
|
-
const
|
|
8376
|
-
logActivity(state, { type: "error", error: `Channel processing error: ${
|
|
10289
|
+
const errorMessage3 = error2 instanceof Error ? error2.message : String(error2);
|
|
10290
|
+
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage3}` });
|
|
8377
10291
|
if (state.interactive) displayStatus(state);
|
|
8378
10292
|
if (driver.hasInFlightWatchers()) {
|
|
8379
10293
|
consecutiveDrainFailures = 0;
|
|
@@ -8390,7 +10304,7 @@ async function driveChannels(state, driver) {
|
|
|
8390
10304
|
}
|
|
8391
10305
|
}
|
|
8392
10306
|
}
|
|
8393
|
-
await new Promise((
|
|
10307
|
+
await new Promise((resolve4) => setTimeout(resolve4, CHANNEL_POLL_INTERVAL_MS));
|
|
8394
10308
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
8395
10309
|
if (idleThisCycle) idleMs += cycleMs;
|
|
8396
10310
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -8411,9 +10325,54 @@ async function driveChannels(state, driver) {
|
|
|
8411
10325
|
}
|
|
8412
10326
|
}
|
|
8413
10327
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
8414
|
-
var SESSION_DB_RECLAIM_MAX_PAGES =
|
|
10328
|
+
var SESSION_DB_RECLAIM_MAX_PAGES = 2e4;
|
|
10329
|
+
function shouldWarnForReclaimSkip(reason) {
|
|
10330
|
+
if (reason !== "sqlite-unavailable") return false;
|
|
10331
|
+
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
10332
|
+
if (!version2) return false;
|
|
10333
|
+
const major = Number(version2[1]);
|
|
10334
|
+
const minor = Number(version2[2]);
|
|
10335
|
+
const patch = Number(version2[3]);
|
|
10336
|
+
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
10337
|
+
}
|
|
8415
10338
|
function sessionDbPath() {
|
|
8416
|
-
return
|
|
10339
|
+
return join10(homedir6(), ".local", "share", "opencode", "opencode.db");
|
|
10340
|
+
}
|
|
10341
|
+
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
10342
|
+
const record = {
|
|
10343
|
+
v: 1,
|
|
10344
|
+
event: "session_db_recovery",
|
|
10345
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10346
|
+
stage: "verify",
|
|
10347
|
+
outcome: "schema_provenance_mismatch",
|
|
10348
|
+
severity: "error",
|
|
10349
|
+
reason: provenance.reason ?? "schema-provenance-mismatch",
|
|
10350
|
+
litestream_exit_code: null,
|
|
10351
|
+
attempt: null,
|
|
10352
|
+
replica_objects: null,
|
|
10353
|
+
replica_bytes: null,
|
|
10354
|
+
quarantine_destination: null,
|
|
10355
|
+
quarantined_objects: null,
|
|
10356
|
+
quarantine_failed_objects: null,
|
|
10357
|
+
quarantined_bytes: null,
|
|
10358
|
+
verified_restore_point: null,
|
|
10359
|
+
restore_points_tried: null,
|
|
10360
|
+
provenance_reason: provenance.reason,
|
|
10361
|
+
provenance_migration_delta: provenance.migrationDelta,
|
|
10362
|
+
replication_suspended: false,
|
|
10363
|
+
dbPath: sessionDbPath(),
|
|
10364
|
+
recorded_version: provenance.recordedVersion,
|
|
10365
|
+
current_version: currentVersion,
|
|
10366
|
+
provenance_pre_boot_migration_count: preBootMigrationCount
|
|
10367
|
+
};
|
|
10368
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10369
|
+
if (!activity) throw new Error("could not map session-DB provenance activity");
|
|
10370
|
+
logActivity(state, {
|
|
10371
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10372
|
+
level: activity.level,
|
|
10373
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10374
|
+
metadata: activity.metadata
|
|
10375
|
+
});
|
|
8417
10376
|
}
|
|
8418
10377
|
async function runSweep(state, driver, config) {
|
|
8419
10378
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -8460,7 +10419,7 @@ async function runSweep(state, driver, config) {
|
|
|
8460
10419
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
8461
10420
|
dbPath: sessionDbPath(),
|
|
8462
10421
|
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
8463
|
-
allowFullVacuum: protectedNow.size === 0
|
|
10422
|
+
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
8464
10423
|
});
|
|
8465
10424
|
if (reclaimResult.ok) {
|
|
8466
10425
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
@@ -8473,7 +10432,7 @@ async function runSweep(state, driver, config) {
|
|
|
8473
10432
|
} else {
|
|
8474
10433
|
logActivity(state, {
|
|
8475
10434
|
type: "info",
|
|
8476
|
-
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})`
|
|
10435
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : "")
|
|
8477
10436
|
});
|
|
8478
10437
|
}
|
|
8479
10438
|
} catch (error2) {
|
|
@@ -8496,13 +10455,20 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
8496
10455
|
for (const warning2 of config.warnings) {
|
|
8497
10456
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
8498
10457
|
}
|
|
8499
|
-
const dbBytes = statSessionDbBytes(
|
|
10458
|
+
const dbBytes = statSessionDbBytes(homedir6());
|
|
8500
10459
|
void (async () => {
|
|
8501
|
-
const
|
|
10460
|
+
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
10461
|
+
if (reclaimAvailability !== null) {
|
|
10462
|
+
logActivity(state, {
|
|
10463
|
+
type: "info",
|
|
10464
|
+
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
10465
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
10466
|
+
});
|
|
10467
|
+
}
|
|
8502
10468
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
8503
10469
|
dbBytes,
|
|
8504
10470
|
cleanupEnabled: config.enabled,
|
|
8505
|
-
reclaimSkipReason
|
|
10471
|
+
reclaimSkipReason: reclaimAvailability?.reason ?? null
|
|
8506
10472
|
});
|
|
8507
10473
|
if (sizeWarning !== null) {
|
|
8508
10474
|
logActivity(state, { type: "info", level: "warn", message: sizeWarning });
|
|
@@ -8697,7 +10663,8 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
8697
10663
|
});
|
|
8698
10664
|
return;
|
|
8699
10665
|
}
|
|
8700
|
-
const collect = createResourceUsageCollector(
|
|
10666
|
+
const { collect, stop } = createResourceUsageCollector(homedir6());
|
|
10667
|
+
state.stopResourceUsageSampling = stop;
|
|
8701
10668
|
let consecutiveFailures = 0;
|
|
8702
10669
|
const tick = async () => {
|
|
8703
10670
|
try {
|
|
@@ -8793,6 +10760,8 @@ async function cleanup(state, opts = {}) {
|
|
|
8793
10760
|
clearTimeout(timer);
|
|
8794
10761
|
}
|
|
8795
10762
|
state.sessionCleanupTimers = [];
|
|
10763
|
+
state.stopOpenCodeLogTail?.();
|
|
10764
|
+
state.stopOpenCodeLogTail = null;
|
|
8796
10765
|
if (state.claudeUsageTimer) {
|
|
8797
10766
|
clearTimeout(state.claudeUsageTimer);
|
|
8798
10767
|
state.claudeUsageTimer = null;
|
|
@@ -8807,21 +10776,41 @@ async function cleanup(state, opts = {}) {
|
|
|
8807
10776
|
clearTimeout(state.resourceUsageTimer);
|
|
8808
10777
|
state.resourceUsageTimer = null;
|
|
8809
10778
|
}
|
|
10779
|
+
state.stopResourceUsageSampling?.();
|
|
10780
|
+
state.stopResourceUsageSampling = null;
|
|
10781
|
+
const credentialSync = state.credentialSync;
|
|
10782
|
+
const flushCredentials = credentialSync ? async (phase, publish) => {
|
|
10783
|
+
await timeShutdownPhase(state, durations, phase, async () => {
|
|
10784
|
+
const outcomes = await credentialSync.stopAndFlush(publish);
|
|
10785
|
+
const level = Object.values(outcomes).every((outcome) => outcome === "ok") ? "info" : "warn";
|
|
10786
|
+
log2(
|
|
10787
|
+
state,
|
|
10788
|
+
`Credential flush: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
10789
|
+
level
|
|
10790
|
+
);
|
|
10791
|
+
});
|
|
10792
|
+
} : void 0;
|
|
10793
|
+
let drainSettled = true;
|
|
8810
10794
|
if (opts.graceful && state.channelDriver) {
|
|
8811
10795
|
state.channelDriver.stop();
|
|
10796
|
+
}
|
|
10797
|
+
if (flushCredentials) {
|
|
10798
|
+
await flushCredentials("credential_flush", !(opts.graceful && state.channelDriver));
|
|
10799
|
+
}
|
|
10800
|
+
if (opts.graceful && state.channelDriver) {
|
|
8812
10801
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
8813
10802
|
if (state.interactive) {
|
|
8814
10803
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
8815
10804
|
displayStatus(state);
|
|
8816
10805
|
}
|
|
8817
10806
|
const driver = state.channelDriver;
|
|
8818
|
-
|
|
10807
|
+
drainSettled = await timeShutdownPhase(
|
|
8819
10808
|
state,
|
|
8820
10809
|
durations,
|
|
8821
10810
|
"drain",
|
|
8822
10811
|
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
8823
10812
|
);
|
|
8824
|
-
if (!
|
|
10813
|
+
if (!drainSettled) {
|
|
8825
10814
|
logActivity(state, {
|
|
8826
10815
|
type: "info",
|
|
8827
10816
|
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
@@ -8829,6 +10818,9 @@ async function cleanup(state, opts = {}) {
|
|
|
8829
10818
|
if (state.interactive) displayStatus(state);
|
|
8830
10819
|
}
|
|
8831
10820
|
}
|
|
10821
|
+
if (opts.graceful && state.channelDriver && flushCredentials && drainSettled) {
|
|
10822
|
+
await flushCredentials("credential_flush_final", true);
|
|
10823
|
+
}
|
|
8832
10824
|
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
8833
10825
|
if (state.connection) {
|
|
8834
10826
|
const connection = state.connection;
|
|
@@ -8864,13 +10856,56 @@ async function cleanup(state, opts = {}) {
|
|
|
8864
10856
|
}
|
|
8865
10857
|
return durations;
|
|
8866
10858
|
}
|
|
10859
|
+
async function beginGracefulShutdown(state, trigger) {
|
|
10860
|
+
if (state.shuttingDown) return;
|
|
10861
|
+
state.shuttingDown = true;
|
|
10862
|
+
const shutdownStartedAt = Date.now();
|
|
10863
|
+
const shutdownMessage = trigger === "recycle" ? "Recycle requested \u2014 finishing in-flight work, then exiting" : "Shutting down...";
|
|
10864
|
+
if (state.interactive) {
|
|
10865
|
+
logActivity(state, { type: "info", message: shutdownMessage });
|
|
10866
|
+
displayStatus(state);
|
|
10867
|
+
} else {
|
|
10868
|
+
log2(state, shutdownMessage);
|
|
10869
|
+
}
|
|
10870
|
+
const durations = await cleanup(state, { graceful: true });
|
|
10871
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
10872
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
10873
|
+
let timer;
|
|
10874
|
+
const flushed = shutdownTelemetry().then(
|
|
10875
|
+
() => true,
|
|
10876
|
+
(error2) => {
|
|
10877
|
+
log2(
|
|
10878
|
+
state,
|
|
10879
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10880
|
+
"warn"
|
|
10881
|
+
);
|
|
10882
|
+
return true;
|
|
10883
|
+
}
|
|
10884
|
+
);
|
|
10885
|
+
const timedOut = new Promise((resolve4) => {
|
|
10886
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
10887
|
+
});
|
|
10888
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
10889
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
10890
|
+
}
|
|
10891
|
+
clearTimeout(timer);
|
|
10892
|
+
});
|
|
10893
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
10894
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
10895
|
+
process.exit(0);
|
|
10896
|
+
}
|
|
8867
10897
|
async function run(options) {
|
|
8868
10898
|
const interactive = isInteractive(options.json);
|
|
8869
10899
|
let logLevel;
|
|
8870
10900
|
let fileSyncDirectories;
|
|
8871
10901
|
try {
|
|
8872
10902
|
logLevel = resolveLogLevel(options);
|
|
8873
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
10903
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir6());
|
|
10904
|
+
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
10905
|
+
throw new Error(
|
|
10906
|
+
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
10907
|
+
);
|
|
10908
|
+
}
|
|
8874
10909
|
} catch (error2) {
|
|
8875
10910
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8876
10911
|
if (options.json) {
|
|
@@ -8894,7 +10929,9 @@ async function run(options) {
|
|
|
8894
10929
|
connected: false,
|
|
8895
10930
|
opencodeConnected: false,
|
|
8896
10931
|
opencodeVersion: null,
|
|
10932
|
+
sessionDbProvenanceAnomaly: false,
|
|
8897
10933
|
opencodeProcess: null,
|
|
10934
|
+
stopOpenCodeLogTail: null,
|
|
8898
10935
|
litestreamProcess: null,
|
|
8899
10936
|
connection: null,
|
|
8900
10937
|
channelDriver: null,
|
|
@@ -8909,9 +10946,24 @@ async function run(options) {
|
|
|
8909
10946
|
openaiUsageTimer: null,
|
|
8910
10947
|
openaiUsageRearm: null,
|
|
8911
10948
|
resourceUsageTimer: null,
|
|
10949
|
+
stopResourceUsageSampling: null,
|
|
10950
|
+
credentialSync: null,
|
|
8912
10951
|
authHeader: ""
|
|
8913
10952
|
};
|
|
8914
10953
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
10954
|
+
if (options.credentialSyncMarker) {
|
|
10955
|
+
state.credentialSync = createCredentialSync({
|
|
10956
|
+
markerPath: options.credentialSyncMarker,
|
|
10957
|
+
env: process.env,
|
|
10958
|
+
log: (message, level = "info") => {
|
|
10959
|
+
if (level === "error") {
|
|
10960
|
+
logActivity(state, { type: "error", error: message });
|
|
10961
|
+
} else {
|
|
10962
|
+
logActivity(state, { type: "info", level, message });
|
|
10963
|
+
}
|
|
10964
|
+
}
|
|
10965
|
+
});
|
|
10966
|
+
}
|
|
8915
10967
|
if (fileSyncDirectories.length > 0) {
|
|
8916
10968
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
8917
10969
|
} else {
|
|
@@ -8937,43 +10989,7 @@ async function run(options) {
|
|
|
8937
10989
|
"warn"
|
|
8938
10990
|
);
|
|
8939
10991
|
}
|
|
8940
|
-
const handleSignal =
|
|
8941
|
-
if (state.shuttingDown) return;
|
|
8942
|
-
state.shuttingDown = true;
|
|
8943
|
-
const shutdownStartedAt = Date.now();
|
|
8944
|
-
if (state.interactive) {
|
|
8945
|
-
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
8946
|
-
displayStatus(state);
|
|
8947
|
-
} else {
|
|
8948
|
-
log2(state, "Shutting down...");
|
|
8949
|
-
}
|
|
8950
|
-
const durations = await cleanup(state, { graceful: true });
|
|
8951
|
-
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
8952
|
-
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
8953
|
-
let timer;
|
|
8954
|
-
const flushed = shutdownTelemetry().then(
|
|
8955
|
-
() => true,
|
|
8956
|
-
(error2) => {
|
|
8957
|
-
log2(
|
|
8958
|
-
state,
|
|
8959
|
-
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8960
|
-
"warn"
|
|
8961
|
-
);
|
|
8962
|
-
return true;
|
|
8963
|
-
}
|
|
8964
|
-
);
|
|
8965
|
-
const timedOut = new Promise((resolve3) => {
|
|
8966
|
-
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
8967
|
-
});
|
|
8968
|
-
if (!await Promise.race([flushed, timedOut])) {
|
|
8969
|
-
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
8970
|
-
}
|
|
8971
|
-
clearTimeout(timer);
|
|
8972
|
-
});
|
|
8973
|
-
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
8974
|
-
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
8975
|
-
process.exit(0);
|
|
8976
|
-
};
|
|
10992
|
+
const handleSignal = () => beginGracefulShutdown(state, "signal");
|
|
8977
10993
|
process.on("SIGINT", handleSignal);
|
|
8978
10994
|
process.on("SIGTERM", handleSignal);
|
|
8979
10995
|
try {
|
|
@@ -8983,15 +10999,15 @@ async function run(options) {
|
|
|
8983
10999
|
printError("Authentication required");
|
|
8984
11000
|
blank();
|
|
8985
11001
|
console.log(
|
|
8986
|
-
|
|
11002
|
+
chalk7.dim("Set EVIDENT_RUNNER_KEY (or EVIDENT_AGENT_KEY) environment variable for CI")
|
|
8987
11003
|
);
|
|
8988
|
-
console.log(
|
|
11004
|
+
console.log(chalk7.dim("Or run `evident login` for interactive authentication"));
|
|
8989
11005
|
blank();
|
|
8990
11006
|
process.exit(1);
|
|
8991
11007
|
return;
|
|
8992
11008
|
}
|
|
8993
11009
|
blank();
|
|
8994
|
-
console.log(
|
|
11010
|
+
console.log(chalk7.yellow("You are not logged in to Evident."));
|
|
8995
11011
|
blank();
|
|
8996
11012
|
credentials2 = await promptForLogin(
|
|
8997
11013
|
"Would you like to log in now?",
|
|
@@ -9041,7 +11057,7 @@ async function run(options) {
|
|
|
9041
11057
|
);
|
|
9042
11058
|
blank();
|
|
9043
11059
|
console.log(
|
|
9044
|
-
|
|
11060
|
+
chalk7.dim(
|
|
9045
11061
|
"Either provide --runner/--agent <id> or set EVIDENT_RUNNER_KEY/EVIDENT_AGENT_KEY"
|
|
9046
11062
|
)
|
|
9047
11063
|
);
|
|
@@ -9064,15 +11080,15 @@ async function run(options) {
|
|
|
9064
11080
|
);
|
|
9065
11081
|
if (interactive && !state.json) {
|
|
9066
11082
|
blank();
|
|
9067
|
-
console.log(
|
|
9068
|
-
console.log(
|
|
11083
|
+
console.log(chalk7.bold("Evident Run"));
|
|
11084
|
+
console.log(chalk7.dim("-".repeat(40)));
|
|
9069
11085
|
}
|
|
9070
11086
|
const spinner = interactive && !state.json ? ora3("Validating runner...").start() : null;
|
|
9071
11087
|
let validation = await getAgentInfo(state.agentId, state.authHeader);
|
|
9072
11088
|
if (!validation.valid && validation.authFailed && interactive) {
|
|
9073
11089
|
spinner?.fail("Authentication failed");
|
|
9074
11090
|
blank();
|
|
9075
|
-
console.log(
|
|
11091
|
+
console.log(chalk7.yellow("Your authentication token is invalid or expired."));
|
|
9076
11092
|
blank();
|
|
9077
11093
|
credentials2 = await promptForLogin(
|
|
9078
11094
|
"Would you like to log in again?",
|
|
@@ -9103,27 +11119,140 @@ async function run(options) {
|
|
|
9103
11119
|
} else {
|
|
9104
11120
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
9105
11121
|
}
|
|
11122
|
+
if (options.restoreRunnerCredentials) {
|
|
11123
|
+
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
11124
|
+
const credentialContext = {
|
|
11125
|
+
env: process.env,
|
|
11126
|
+
log: (message, level = "info") => {
|
|
11127
|
+
if (level === "error") {
|
|
11128
|
+
logActivity(state, { type: "error", error: message });
|
|
11129
|
+
} else {
|
|
11130
|
+
logActivity(state, { type: "info", level, message });
|
|
11131
|
+
}
|
|
11132
|
+
}
|
|
11133
|
+
};
|
|
11134
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
11135
|
+
await restoreCredentialStores(credentialContext);
|
|
11136
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
11137
|
+
}
|
|
11138
|
+
state.credentialSync?.arm();
|
|
11139
|
+
state.stopOpenCodeLogTail = tailOpenCodeLogFile(
|
|
11140
|
+
resolveOpenCodeLogPath(homedir6(), process.env),
|
|
11141
|
+
createOpenCodeActivityForwarder(() => ({
|
|
11142
|
+
agentId: state.agentId,
|
|
11143
|
+
authHeader: state.authHeader
|
|
11144
|
+
}))
|
|
11145
|
+
).stop;
|
|
11146
|
+
let sessionDbVerifyFatal = false;
|
|
11147
|
+
if (!options.restoreSessionDb) {
|
|
11148
|
+
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
11149
|
+
} else {
|
|
11150
|
+
const health = await checkOpenCodeHealth(state.port);
|
|
11151
|
+
if (health.healthy) {
|
|
11152
|
+
log2(
|
|
11153
|
+
state,
|
|
11154
|
+
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
11155
|
+
"debug"
|
|
11156
|
+
);
|
|
11157
|
+
} else {
|
|
11158
|
+
const result = await restoreAndVerifySessionDb({
|
|
11159
|
+
dbPath: sessionDbPath(),
|
|
11160
|
+
litestreamConfig: options.litestreamConfig,
|
|
11161
|
+
noReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
11162
|
+
env: process.env,
|
|
11163
|
+
log: (message, level = "info") => {
|
|
11164
|
+
if (level === "error") {
|
|
11165
|
+
logActivity(state, { type: "error", error: message });
|
|
11166
|
+
} else {
|
|
11167
|
+
logActivity(state, { type: "info", level, message });
|
|
11168
|
+
}
|
|
11169
|
+
},
|
|
11170
|
+
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
11171
|
+
});
|
|
11172
|
+
sessionDbVerifyFatal = result.verifyFatal;
|
|
11173
|
+
}
|
|
11174
|
+
}
|
|
9106
11175
|
reportSessionDbRecovery(state);
|
|
11176
|
+
if (sessionDbVerifyFatal) {
|
|
11177
|
+
throw new Error(
|
|
11178
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
11179
|
+
);
|
|
11180
|
+
}
|
|
11181
|
+
applyRunnerOpenCodeConfig({
|
|
11182
|
+
overlayPath: options.opencodeConfigOverlay,
|
|
11183
|
+
log: (message, level = "info") => {
|
|
11184
|
+
if (level === "error") {
|
|
11185
|
+
logActivity(state, { type: "error", error: message });
|
|
11186
|
+
} else {
|
|
11187
|
+
logActivity(state, { type: "info", level, message });
|
|
11188
|
+
}
|
|
11189
|
+
}
|
|
11190
|
+
});
|
|
9107
11191
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
9108
11192
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
9109
11193
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
9110
11194
|
}
|
|
11195
|
+
const { version: opencodeVersion, warnings: opencodeVersionWarnings } = resolveOpenCodeVersion(
|
|
11196
|
+
options,
|
|
11197
|
+
process.env
|
|
11198
|
+
);
|
|
11199
|
+
for (const warning2 of opencodeVersionWarnings) {
|
|
11200
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
11201
|
+
}
|
|
9111
11202
|
const { value: maxActiveSessions, warnings: maxActiveSessionsWarnings } = resolveMaxActiveSessions(options, process.env);
|
|
9112
11203
|
for (const warning2 of maxActiveSessionsWarnings) {
|
|
9113
11204
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
9114
11205
|
}
|
|
11206
|
+
const preBootMigrationIds = readSessionDbMigrationIds(sessionDbPath());
|
|
9115
11207
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
9116
11208
|
try {
|
|
9117
|
-
const oc = await
|
|
11209
|
+
const oc = opencodeVersion === "v2" ? await ensureOpenCode2Running({
|
|
11210
|
+
port: state.port,
|
|
11211
|
+
interactive: state.interactive,
|
|
11212
|
+
agentId: state.agentId,
|
|
11213
|
+
log: (message) => log2(state, message),
|
|
11214
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
11215
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
11216
|
+
}) : await ensureOpenCodeRunning({
|
|
9118
11217
|
port: state.port,
|
|
9119
11218
|
interactive: state.interactive,
|
|
9120
11219
|
agentId: state.agentId,
|
|
9121
11220
|
log: (message) => log2(state, message),
|
|
9122
|
-
startTimeoutMs: opencodeStartTimeoutMs
|
|
11221
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
11222
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
9123
11223
|
});
|
|
9124
11224
|
state.port = oc.port;
|
|
9125
|
-
state.opencodeProcess = oc.process;
|
|
11225
|
+
state.opencodeProcess = options.opencodePidFile ? null : oc.process;
|
|
9126
11226
|
state.opencodeVersion = oc.version;
|
|
11227
|
+
if (options.opencodePidFile && oc.process?.pid !== void 0) {
|
|
11228
|
+
try {
|
|
11229
|
+
writeFileSync6(options.opencodePidFile, `${oc.process.pid}
|
|
11230
|
+
`, { mode: 384 });
|
|
11231
|
+
chmodSync3(options.opencodePidFile, 384);
|
|
11232
|
+
} catch (error2) {
|
|
11233
|
+
logActivity(state, {
|
|
11234
|
+
type: "error",
|
|
11235
|
+
error: `Failed to write OpenCode pid file ${options.opencodePidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11236
|
+
});
|
|
11237
|
+
}
|
|
11238
|
+
}
|
|
11239
|
+
if (state.opencodeVersion !== null) {
|
|
11240
|
+
const provenance = checkSessionDbProvenance({
|
|
11241
|
+
dbPath: sessionDbPath(),
|
|
11242
|
+
currentVersion: state.opencodeVersion,
|
|
11243
|
+
homeDir: homedir6(),
|
|
11244
|
+
env: process.env
|
|
11245
|
+
});
|
|
11246
|
+
if (provenance.anomaly) {
|
|
11247
|
+
state.sessionDbProvenanceAnomaly = true;
|
|
11248
|
+
logSessionDbProvenanceMismatch(
|
|
11249
|
+
state,
|
|
11250
|
+
provenance,
|
|
11251
|
+
state.opencodeVersion,
|
|
11252
|
+
preBootMigrationIds?.length ?? null
|
|
11253
|
+
);
|
|
11254
|
+
}
|
|
11255
|
+
}
|
|
9127
11256
|
state.opencodeConnected = oc.notReadyReason === null;
|
|
9128
11257
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
9129
11258
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
@@ -9146,10 +11275,10 @@ async function run(options) {
|
|
|
9146
11275
|
if (state.interactive && !state.json) {
|
|
9147
11276
|
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
9148
11277
|
blank();
|
|
9149
|
-
console.log(
|
|
11278
|
+
console.log(chalk7.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
9150
11279
|
console.log(
|
|
9151
|
-
|
|
9152
|
-
`Run ${
|
|
11280
|
+
chalk7.dim(
|
|
11281
|
+
`Run ${chalk7.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
9153
11282
|
)
|
|
9154
11283
|
);
|
|
9155
11284
|
blank();
|
|
@@ -9160,7 +11289,75 @@ async function run(options) {
|
|
|
9160
11289
|
ocSpinner?.fail(error2.message);
|
|
9161
11290
|
throw error2;
|
|
9162
11291
|
}
|
|
9163
|
-
if (options.
|
|
11292
|
+
if (options.litestreamPidFile) {
|
|
11293
|
+
if (options.sessionDbNoReplicateMarker && existsSync3(options.sessionDbNoReplicateMarker)) {
|
|
11294
|
+
log2(
|
|
11295
|
+
state,
|
|
11296
|
+
`Skipping Litestream replication because ${options.sessionDbNoReplicateMarker} marks this session database unsafe to replicate`
|
|
11297
|
+
);
|
|
11298
|
+
} else if (!options.litestreamConfig) {
|
|
11299
|
+
logActivity(state, {
|
|
11300
|
+
type: "info",
|
|
11301
|
+
level: "warn",
|
|
11302
|
+
message: "Skipping Litestream replication because no configuration file was provided"
|
|
11303
|
+
});
|
|
11304
|
+
} else {
|
|
11305
|
+
let existingPid;
|
|
11306
|
+
if (existsSync3(options.litestreamPidFile)) {
|
|
11307
|
+
try {
|
|
11308
|
+
const rawPid = readFileSync6(options.litestreamPidFile, "utf8").trim();
|
|
11309
|
+
const parsedPid = Number(rawPid);
|
|
11310
|
+
if (/^\d+$/.test(rawPid) && Number.isSafeInteger(parsedPid) && parsedPid > 0) {
|
|
11311
|
+
existingPid = parsedPid;
|
|
11312
|
+
}
|
|
11313
|
+
} catch (error2) {
|
|
11314
|
+
logActivity(state, {
|
|
11315
|
+
type: "info",
|
|
11316
|
+
level: "warn",
|
|
11317
|
+
message: `Could not read Litestream pid file ${options.litestreamPidFile}; checking for a new process: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11318
|
+
});
|
|
11319
|
+
}
|
|
11320
|
+
}
|
|
11321
|
+
if (existingPid !== void 0 && isProcessAlive(existingPid)) {
|
|
11322
|
+
log2(state, `Litestream replication is already running with pid ${existingPid}`);
|
|
11323
|
+
} else {
|
|
11324
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
11325
|
+
state.litestreamProcess = null;
|
|
11326
|
+
let failureHandled = false;
|
|
11327
|
+
const reportImageOwnedReplicationFailure = (message) => {
|
|
11328
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
11329
|
+
failureHandled = true;
|
|
11330
|
+
logActivity(state, { type: "error", error: message });
|
|
11331
|
+
if (state.interactive) displayStatus(state);
|
|
11332
|
+
};
|
|
11333
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
11334
|
+
reportImageOwnedReplicationFailure(
|
|
11335
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
11336
|
+
);
|
|
11337
|
+
});
|
|
11338
|
+
litestreamProcess.on("error", (error2) => {
|
|
11339
|
+
reportImageOwnedReplicationFailure(
|
|
11340
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11341
|
+
);
|
|
11342
|
+
});
|
|
11343
|
+
try {
|
|
11344
|
+
if (litestreamProcess.pid !== void 0) {
|
|
11345
|
+
writeFileSync6(options.litestreamPidFile, `${litestreamProcess.pid}
|
|
11346
|
+
`, {
|
|
11347
|
+
mode: 384
|
|
11348
|
+
});
|
|
11349
|
+
chmodSync3(options.litestreamPidFile, 384);
|
|
11350
|
+
}
|
|
11351
|
+
} catch (error2) {
|
|
11352
|
+
logActivity(state, {
|
|
11353
|
+
type: "error",
|
|
11354
|
+
error: `Failed to write Litestream pid file ${options.litestreamPidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11355
|
+
});
|
|
11356
|
+
}
|
|
11357
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
11358
|
+
}
|
|
11359
|
+
}
|
|
11360
|
+
} else if (options.litestreamConfig) {
|
|
9164
11361
|
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
9165
11362
|
state.litestreamProcess = litestreamProcess;
|
|
9166
11363
|
let failureHandled = false;
|
|
@@ -9205,7 +11402,7 @@ async function run(options) {
|
|
|
9205
11402
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
9206
11403
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
9207
11404
|
fileSyncDirectories,
|
|
9208
|
-
homeDir:
|
|
11405
|
+
homeDir: homedir6(),
|
|
9209
11406
|
maxActiveSessions,
|
|
9210
11407
|
log: (entry) => (
|
|
9211
11408
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -9331,6 +11528,18 @@ async function run(options) {
|
|
|
9331
11528
|
if (state.interactive) displayStatus(state);
|
|
9332
11529
|
});
|
|
9333
11530
|
},
|
|
11531
|
+
// Both loops are rearmed because `rearm()` is idempotent for the
|
|
11532
|
+
// provider that did not just connect, and is a no-op when reporting is off.
|
|
11533
|
+
onUsageRearmPing: () => {
|
|
11534
|
+
if (!state.running) return;
|
|
11535
|
+
logActivity(state, {
|
|
11536
|
+
type: "info",
|
|
11537
|
+
level: "debug",
|
|
11538
|
+
message: "Usage rearm ping received"
|
|
11539
|
+
});
|
|
11540
|
+
state.claudeUsageRearm?.();
|
|
11541
|
+
state.openaiUsageRearm?.();
|
|
11542
|
+
},
|
|
9334
11543
|
onInfo: (message) => logActivity(state, { type: "info", message })
|
|
9335
11544
|
}
|
|
9336
11545
|
});
|
|
@@ -9351,7 +11560,17 @@ async function run(options) {
|
|
|
9351
11560
|
setTimer: (timer) => {
|
|
9352
11561
|
state.openaiUsageTimer = timer;
|
|
9353
11562
|
},
|
|
9354
|
-
fetchUsage: () =>
|
|
11563
|
+
fetchUsage: async () => {
|
|
11564
|
+
const usage = await getOpenAiUsage(state.port);
|
|
11565
|
+
if (usage.subscription === null) {
|
|
11566
|
+
logActivity(state, {
|
|
11567
|
+
type: "info",
|
|
11568
|
+
level: "debug",
|
|
11569
|
+
message: "OpenAI usage subscription could not be identified from the local credential"
|
|
11570
|
+
});
|
|
11571
|
+
}
|
|
11572
|
+
return usage;
|
|
11573
|
+
},
|
|
9355
11574
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
9356
11575
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
9357
11576
|
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
@@ -9397,7 +11616,7 @@ async function run(options) {
|
|
|
9397
11616
|
}
|
|
9398
11617
|
|
|
9399
11618
|
// src/index.ts
|
|
9400
|
-
var { version } =
|
|
11619
|
+
var { version } = createRequire2(import.meta.url)("../package.json");
|
|
9401
11620
|
var program = new Command();
|
|
9402
11621
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
9403
11622
|
"--endpoint <url>",
|
|
@@ -9425,6 +11644,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9425
11644
|
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
9426
11645
|
"--opencode-start-timeout <seconds>",
|
|
9427
11646
|
"Seconds to wait for OpenCode to become healthy when the runner starts it (default: 180). Env: EVIDENT_OPENCODE_START_TIMEOUT"
|
|
11647
|
+
).option(
|
|
11648
|
+
"--opencode-version <v1|v2>",
|
|
11649
|
+
"Which OpenCode major version to launch: v1 or v2 (default: v1). Env: EVIDENT_OPENCODE_VERSION"
|
|
9428
11650
|
).option("--json", "Output in JSON format").option(
|
|
9429
11651
|
"--session-cleanup-max-age <duration>",
|
|
9430
11652
|
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
@@ -9457,6 +11679,27 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9457
11679
|
).option(
|
|
9458
11680
|
"--litestream-config <path>",
|
|
9459
11681
|
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
11682
|
+
).option(
|
|
11683
|
+
"--opencode-pid-file <path>",
|
|
11684
|
+
"Record the OpenCode pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11685
|
+
).option(
|
|
11686
|
+
"--litestream-pid-file <path>",
|
|
11687
|
+
"Record the Litestream pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11688
|
+
).option(
|
|
11689
|
+
"--session-db-no-replicate-marker <path>",
|
|
11690
|
+
"Read this marker as a gate before starting Litestream; the session-DB restore writes it when this boot's database is not safe to replicate."
|
|
11691
|
+
).option(
|
|
11692
|
+
"--restore-session-db",
|
|
11693
|
+
"Restore and verify the OpenCode session database before starting OpenCode; requires --litestream-config for the replica and --session-db-no-replicate-marker for give-up records."
|
|
11694
|
+
).option(
|
|
11695
|
+
"--restore-runner-credentials",
|
|
11696
|
+
"Restore the hosted runner secret and persisted credential stores before starting OpenCode."
|
|
11697
|
+
).option(
|
|
11698
|
+
"--opencode-config-overlay <path>",
|
|
11699
|
+
"Apply this runner-provided OpenCode config before starting OpenCode."
|
|
11700
|
+
).option(
|
|
11701
|
+
"--credential-sync-marker <path>",
|
|
11702
|
+
"Own the interval credential sync and write this marker once the shutdown flush has finished, so the runner image's lifecycle hooks can wait on it."
|
|
9460
11703
|
).action(
|
|
9461
11704
|
(options) => {
|
|
9462
11705
|
run({
|
|
@@ -9472,6 +11715,7 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9472
11715
|
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
9473
11716
|
// resolveOpenCodeStartTimeoutMs (flag > EVIDENT_OPENCODE_START_TIMEOUT > 180s default).
|
|
9474
11717
|
opencodeStartTimeout: options.opencodeStartTimeout,
|
|
11718
|
+
opencodeVersion: options.opencodeVersion,
|
|
9475
11719
|
json: options.json,
|
|
9476
11720
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
9477
11721
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
@@ -9489,7 +11733,14 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9489
11733
|
// resolveFileSyncDirectories.
|
|
9490
11734
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
9491
11735
|
tunnelReadyFile: options.tunnelReadyFile,
|
|
9492
|
-
litestreamConfig: options.litestreamConfig
|
|
11736
|
+
litestreamConfig: options.litestreamConfig,
|
|
11737
|
+
opencodePidFile: options.opencodePidFile,
|
|
11738
|
+
litestreamPidFile: options.litestreamPidFile,
|
|
11739
|
+
sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
11740
|
+
restoreSessionDb: options.restoreSessionDb,
|
|
11741
|
+
restoreRunnerCredentials: options.restoreRunnerCredentials,
|
|
11742
|
+
opencodeConfigOverlay: options.opencodeConfigOverlay,
|
|
11743
|
+
credentialSyncMarker: options.credentialSyncMarker
|
|
9493
11744
|
});
|
|
9494
11745
|
}
|
|
9495
11746
|
);
|