@evident-ai/cli 3.4.1-dev.1856549 → 3.4.1-dev.2fec2c1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/index.js +3218 -411
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire } from "module";
|
|
4
|
+
import { createRequire as createRequire2 } from "module";
|
|
5
5
|
import { Command } from "commander";
|
|
6
6
|
|
|
7
7
|
// src/commands/login.ts
|
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
-
import { dirname } from "path";
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "node:fs";
|
|
15
|
+
import { dirname } from "node:path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -371,14 +371,14 @@ function blank() {
|
|
|
371
371
|
console.log();
|
|
372
372
|
}
|
|
373
373
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
374
|
-
return new Promise((
|
|
374
|
+
return new Promise((resolve4) => {
|
|
375
375
|
process.stdout.write(chalk.dim(prompt));
|
|
376
376
|
const handler = () => {
|
|
377
377
|
process.stdin.removeListener("data", handler);
|
|
378
378
|
process.stdin.setRawMode?.(false);
|
|
379
379
|
process.stdin.pause();
|
|
380
380
|
console.log();
|
|
381
|
-
|
|
381
|
+
resolve4();
|
|
382
382
|
};
|
|
383
383
|
if (process.stdin.isTTY) {
|
|
384
384
|
process.stdin.setRawMode?.(true);
|
|
@@ -388,7 +388,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
388
388
|
});
|
|
389
389
|
}
|
|
390
390
|
function sleep(ms) {
|
|
391
|
-
return new Promise((
|
|
391
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
392
392
|
}
|
|
393
393
|
|
|
394
394
|
// src/commands/login.ts
|
|
@@ -466,19 +466,19 @@ async function tokenLogin() {
|
|
|
466
466
|
);
|
|
467
467
|
blank();
|
|
468
468
|
process.stdout.write("Paste token: ");
|
|
469
|
-
const token = await new Promise((
|
|
469
|
+
const token = await new Promise((resolve4) => {
|
|
470
470
|
let data = "";
|
|
471
471
|
process.stdin.setEncoding("utf8");
|
|
472
472
|
process.stdin.on("data", (chunk) => {
|
|
473
473
|
data += chunk;
|
|
474
474
|
});
|
|
475
475
|
process.stdin.on("end", () => {
|
|
476
|
-
|
|
476
|
+
resolve4(data.trim());
|
|
477
477
|
});
|
|
478
478
|
if (process.stdin.isTTY) {
|
|
479
479
|
process.stdin.once("data", (chunk) => {
|
|
480
480
|
process.stdin.pause();
|
|
481
|
-
|
|
481
|
+
resolve4(chunk.toString().trim());
|
|
482
482
|
});
|
|
483
483
|
process.stdin.resume();
|
|
484
484
|
}
|
|
@@ -623,6 +623,19 @@ function isInteractive(jsonOutput) {
|
|
|
623
623
|
return true;
|
|
624
624
|
}
|
|
625
625
|
|
|
626
|
+
// src/lib/subscription-usage-report.ts
|
|
627
|
+
function toReportedSubscription(collected) {
|
|
628
|
+
if (!collected) return null;
|
|
629
|
+
if (collected.ownerEmail === null && collected.planType === null && collected.organizationName === null) {
|
|
630
|
+
return null;
|
|
631
|
+
}
|
|
632
|
+
return {
|
|
633
|
+
owner_email: collected.ownerEmail,
|
|
634
|
+
plan_type: collected.planType,
|
|
635
|
+
organization_name: collected.organizationName
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
|
|
626
639
|
// src/commands/agent-lookup.ts
|
|
627
640
|
async function readErrorMessage(response) {
|
|
628
641
|
const text = await response.text().catch(() => "");
|
|
@@ -722,14 +735,6 @@ function toReportedWindow(window) {
|
|
|
722
735
|
if (!window) return null;
|
|
723
736
|
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
724
737
|
}
|
|
725
|
-
function toReportedOwner(snapshot) {
|
|
726
|
-
if (!snapshot.owner) return null;
|
|
727
|
-
return {
|
|
728
|
-
email: snapshot.owner.email,
|
|
729
|
-
organization_name: snapshot.owner.organizationName,
|
|
730
|
-
rate_limit_tier: snapshot.owner.rateLimitTier
|
|
731
|
-
};
|
|
732
|
-
}
|
|
733
738
|
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
734
739
|
try {
|
|
735
740
|
const apiUrl = getApiUrlConfig();
|
|
@@ -739,7 +744,7 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
739
744
|
body: JSON.stringify({
|
|
740
745
|
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
741
746
|
seven_day: toReportedWindow(snapshot.sevenDay),
|
|
742
|
-
|
|
747
|
+
subscription: toReportedSubscription(snapshot.subscription)
|
|
743
748
|
}),
|
|
744
749
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
745
750
|
});
|
|
@@ -773,7 +778,8 @@ async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
|
773
778
|
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
774
779
|
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
775
780
|
has_credits: snapshot.hasCredits,
|
|
776
|
-
credits_unlimited: snapshot.creditsUnlimited
|
|
781
|
+
credits_unlimited: snapshot.creditsUnlimited,
|
|
782
|
+
subscription: toReportedSubscription(snapshot.subscription)
|
|
777
783
|
}),
|
|
778
784
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
779
785
|
});
|
|
@@ -797,6 +803,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
797
803
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
798
804
|
body: JSON.stringify({
|
|
799
805
|
cpu_percent: usage.cpuPercent,
|
|
806
|
+
cpu_peak_percent: usage.cpuPeakPercent,
|
|
800
807
|
cpu_count: usage.cpuCount,
|
|
801
808
|
memory_total_bytes: usage.memoryTotalBytes,
|
|
802
809
|
memory_available_bytes: usage.memoryAvailableBytes,
|
|
@@ -1000,10 +1007,10 @@ async function status(options = {}) {
|
|
|
1000
1007
|
}
|
|
1001
1008
|
|
|
1002
1009
|
// src/lib/claude-usage.ts
|
|
1003
|
-
import { execFileSync } from "child_process";
|
|
1004
|
-
import { readFileSync } from "fs";
|
|
1005
|
-
import { homedir } from "os";
|
|
1006
|
-
import { join } from "path";
|
|
1010
|
+
import { execFileSync } from "node:child_process";
|
|
1011
|
+
import { readFileSync } from "node:fs";
|
|
1012
|
+
import { homedir } from "node:os";
|
|
1013
|
+
import { join } from "node:path";
|
|
1007
1014
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
1008
1015
|
var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
|
|
1009
1016
|
var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
|
|
@@ -1088,7 +1095,7 @@ function ownerLookupFailure(error2) {
|
|
|
1088
1095
|
}
|
|
1089
1096
|
async function getClaudeUsageOwner(accessToken) {
|
|
1090
1097
|
if (cachedOwner?.accessToken === accessToken) {
|
|
1091
|
-
return {
|
|
1098
|
+
return { subscription: cachedOwner.owner, ownerLookupError: null };
|
|
1092
1099
|
}
|
|
1093
1100
|
try {
|
|
1094
1101
|
const response = await fetch(CLAUDE_PROFILE_URL, {
|
|
@@ -1100,27 +1107,27 @@ async function getClaudeUsageOwner(accessToken) {
|
|
|
1100
1107
|
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1101
1108
|
});
|
|
1102
1109
|
if (!response.ok) {
|
|
1103
|
-
return {
|
|
1110
|
+
return { subscription: null, ownerLookupError: `HTTP ${response.status}` };
|
|
1104
1111
|
}
|
|
1105
1112
|
let body;
|
|
1106
1113
|
try {
|
|
1107
1114
|
body = await response.json();
|
|
1108
1115
|
} catch (error2) {
|
|
1109
|
-
return {
|
|
1116
|
+
return { subscription: null, ownerLookupError: "malformed response" };
|
|
1110
1117
|
}
|
|
1111
1118
|
const profile = body;
|
|
1112
1119
|
if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
|
|
1113
|
-
return {
|
|
1120
|
+
return { subscription: null, ownerLookupError: "malformed response" };
|
|
1114
1121
|
}
|
|
1115
|
-
const
|
|
1116
|
-
|
|
1122
|
+
const subscription = {
|
|
1123
|
+
ownerEmail: profile.account.email,
|
|
1117
1124
|
organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
|
|
1118
|
-
|
|
1125
|
+
planType: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
|
|
1119
1126
|
};
|
|
1120
|
-
cachedOwner = { accessToken, owner };
|
|
1121
|
-
return {
|
|
1127
|
+
cachedOwner = { accessToken, owner: subscription };
|
|
1128
|
+
return { subscription, ownerLookupError: null };
|
|
1122
1129
|
} catch (error2) {
|
|
1123
|
-
return {
|
|
1130
|
+
return { subscription: null, ownerLookupError: ownerLookupFailure(error2) };
|
|
1124
1131
|
}
|
|
1125
1132
|
}
|
|
1126
1133
|
async function getClaudeUsage() {
|
|
@@ -1149,11 +1156,11 @@ async function getClaudeUsage() {
|
|
|
1149
1156
|
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
1150
1157
|
}
|
|
1151
1158
|
const body = await res.json();
|
|
1152
|
-
const {
|
|
1159
|
+
const { subscription, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
|
|
1153
1160
|
return {
|
|
1154
1161
|
fiveHour: toWindow(body.five_hour),
|
|
1155
1162
|
sevenDay: toWindow(body.seven_day),
|
|
1156
|
-
|
|
1163
|
+
subscription,
|
|
1157
1164
|
ownerLookupError
|
|
1158
1165
|
};
|
|
1159
1166
|
}
|
|
@@ -1183,9 +1190,10 @@ async function claudeUsage() {
|
|
|
1183
1190
|
}
|
|
1184
1191
|
|
|
1185
1192
|
// src/commands/run.ts
|
|
1186
|
-
import {
|
|
1187
|
-
import {
|
|
1188
|
-
import
|
|
1193
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1194
|
+
import { homedir as homedir6 } from "node:os";
|
|
1195
|
+
import { isAbsolute as isAbsolute3, join as join10, parse, resolve as resolvePath2 } from "node:path";
|
|
1196
|
+
import chalk7 from "chalk";
|
|
1189
1197
|
|
|
1190
1198
|
// ../../packages/types/src/agents/index.ts
|
|
1191
1199
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
@@ -1206,6 +1214,7 @@ var TelemetryEventTypes = {
|
|
|
1206
1214
|
// ../../packages/types/src/tunnel/index.ts
|
|
1207
1215
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1208
1216
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1217
|
+
var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
|
|
1209
1218
|
|
|
1210
1219
|
// ../../packages/types/src/runner-files.ts
|
|
1211
1220
|
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
@@ -1243,7 +1252,7 @@ function stripQuery(url) {
|
|
|
1243
1252
|
|
|
1244
1253
|
// src/commands/run.ts
|
|
1245
1254
|
import ora3 from "ora";
|
|
1246
|
-
import { select as
|
|
1255
|
+
import { select as select4 } from "@inquirer/prompts";
|
|
1247
1256
|
|
|
1248
1257
|
// src/lib/telemetry.ts
|
|
1249
1258
|
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
@@ -1416,12 +1425,50 @@ var SEVERITY_BY_LEVEL = {
|
|
|
1416
1425
|
warn: "warning",
|
|
1417
1426
|
error: "error"
|
|
1418
1427
|
};
|
|
1428
|
+
function parseOpenCodeLogLine(line) {
|
|
1429
|
+
const normalisedLine = line.replace(/\r$/, "");
|
|
1430
|
+
const levelMatch = normalisedLine.match(/(?:^|\s)level=(\w+)/i);
|
|
1431
|
+
if (!levelMatch) return null;
|
|
1432
|
+
const level = levelMatch[1].toUpperCase();
|
|
1433
|
+
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1434
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1435
|
+
return { level: level === "WARN" ? "warn" : "error", sessionID: sessionMatch?.[1] };
|
|
1436
|
+
}
|
|
1437
|
+
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1438
|
+
function createOpenCodeActivityForwarder(getContext) {
|
|
1439
|
+
let buffer = Buffer.alloc(0);
|
|
1440
|
+
const flushLine = (line) => {
|
|
1441
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1442
|
+
if (!parsed) return;
|
|
1443
|
+
forwardRunnerActivity(
|
|
1444
|
+
{
|
|
1445
|
+
level: parsed.level,
|
|
1446
|
+
error: line,
|
|
1447
|
+
metadata: parsed.sessionID ? { sessionID: parsed.sessionID } : void 0,
|
|
1448
|
+
source: "opencode"
|
|
1449
|
+
},
|
|
1450
|
+
getContext()
|
|
1451
|
+
);
|
|
1452
|
+
};
|
|
1453
|
+
return (chunk) => {
|
|
1454
|
+
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1455
|
+
let newlineIndex;
|
|
1456
|
+
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
1457
|
+
flushLine(buffer.subarray(0, newlineIndex).toString("utf-8").replace(/\r$/, ""));
|
|
1458
|
+
buffer = buffer.subarray(newlineIndex + 1);
|
|
1459
|
+
}
|
|
1460
|
+
if (buffer.length > MAX_LINE_BUFFER_BYTES) {
|
|
1461
|
+
flushLine(buffer.toString("utf-8"));
|
|
1462
|
+
buffer = Buffer.alloc(0);
|
|
1463
|
+
}
|
|
1464
|
+
};
|
|
1465
|
+
}
|
|
1419
1466
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1420
1467
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1421
1468
|
var MAX_METADATA_ENTRIES = 20;
|
|
1422
1469
|
var TRUNCATION_MARKER = "\u2026";
|
|
1423
1470
|
function redact(message) {
|
|
1424
|
-
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1471
|
+
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1425
1472
|
}
|
|
1426
1473
|
function truncate(message) {
|
|
1427
1474
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
@@ -1447,43 +1494,47 @@ function sanitiseMetadata(metadata) {
|
|
|
1447
1494
|
}
|
|
1448
1495
|
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1449
1496
|
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1450
|
-
var
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1497
|
+
var rateWindows = /* @__PURE__ */ new Map();
|
|
1498
|
+
function admitUnderRateLimit(source, now) {
|
|
1499
|
+
let window = rateWindows.get(source);
|
|
1500
|
+
if (!window) {
|
|
1501
|
+
window = { windowStartedAt: 0, windowCount: 0, windowDroppedCount: 0 };
|
|
1502
|
+
rateWindows.set(source, window);
|
|
1503
|
+
}
|
|
1504
|
+
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1505
|
+
if (window.windowDroppedCount > 0) {
|
|
1456
1506
|
console.error(
|
|
1457
|
-
`[runner-activity-telemetry] rate cap reached: dropped ${windowDroppedCount} ${windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min)`
|
|
1507
|
+
`[runner-activity-telemetry] rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min) for source "${source}"`
|
|
1458
1508
|
);
|
|
1459
1509
|
}
|
|
1460
|
-
windowStartedAt = now;
|
|
1461
|
-
windowCount = 0;
|
|
1462
|
-
windowDroppedCount = 0;
|
|
1510
|
+
window.windowStartedAt = now;
|
|
1511
|
+
window.windowCount = 0;
|
|
1512
|
+
window.windowDroppedCount = 0;
|
|
1463
1513
|
}
|
|
1464
|
-
if (windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1465
|
-
windowDroppedCount++;
|
|
1466
|
-
if (windowDroppedCount === 1) {
|
|
1514
|
+
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1515
|
+
window.windowDroppedCount++;
|
|
1516
|
+
if (window.windowDroppedCount === 1) {
|
|
1467
1517
|
console.error(
|
|
1468
|
-
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window`
|
|
1518
|
+
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window for source "${source}"`
|
|
1469
1519
|
);
|
|
1470
1520
|
}
|
|
1471
1521
|
return false;
|
|
1472
1522
|
}
|
|
1473
|
-
windowCount++;
|
|
1523
|
+
window.windowCount++;
|
|
1474
1524
|
return true;
|
|
1475
1525
|
}
|
|
1476
1526
|
function forwardRunnerActivity(entry, context) {
|
|
1477
1527
|
try {
|
|
1478
1528
|
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1479
1529
|
if (!context.agentId || !context.authHeader) return;
|
|
1480
|
-
|
|
1530
|
+
const source = entry.source ?? "cli.run";
|
|
1531
|
+
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1481
1532
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1482
1533
|
const message = truncate(redact(rawMessage));
|
|
1483
1534
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1484
1535
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1485
1536
|
message,
|
|
1486
|
-
metadata: { ...sanitiseMetadata(entry.metadata), source
|
|
1537
|
+
metadata: { ...sanitiseMetadata(entry.metadata), source },
|
|
1487
1538
|
agentId: context.agentId
|
|
1488
1539
|
});
|
|
1489
1540
|
} catch (err) {
|
|
@@ -1494,8 +1545,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1494
1545
|
}
|
|
1495
1546
|
|
|
1496
1547
|
// src/lib/opencode/session-db-recovery-report.ts
|
|
1497
|
-
import { readFileSync as readFileSync2, unlinkSync } from "fs";
|
|
1498
|
-
import { join as join2 } from "path";
|
|
1548
|
+
import { readFileSync as readFileSync2, unlinkSync } from "node:fs";
|
|
1549
|
+
import { join as join2 } from "node:path";
|
|
1499
1550
|
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1500
1551
|
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1501
1552
|
return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
@@ -1524,7 +1575,14 @@ function drainSessionDbRecoveryReport({
|
|
|
1524
1575
|
skippedLines++;
|
|
1525
1576
|
return [];
|
|
1526
1577
|
}
|
|
1527
|
-
return [
|
|
1578
|
+
return [
|
|
1579
|
+
{
|
|
1580
|
+
...value,
|
|
1581
|
+
provenance_reason: value.provenance_reason ?? null,
|
|
1582
|
+
provenance_migration_delta: value.provenance_migration_delta ?? null,
|
|
1583
|
+
replication_suspended: value.replication_suspended ?? false
|
|
1584
|
+
}
|
|
1585
|
+
];
|
|
1528
1586
|
} catch (error2) {
|
|
1529
1587
|
skippedLines++;
|
|
1530
1588
|
console.error(
|
|
@@ -1627,6 +1685,12 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1627
1685
|
metadata: withoutContractFields(record),
|
|
1628
1686
|
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1629
1687
|
};
|
|
1688
|
+
case "schema_provenance_mismatch":
|
|
1689
|
+
return {
|
|
1690
|
+
level,
|
|
1691
|
+
metadata: withoutContractFields(record),
|
|
1692
|
+
message: `Session database schema provenance mismatch for ${record.dbPath ?? record.db_path ?? "unknown"}: recorded version=${record.recorded_version ?? "unknown"}, current version=${record.current_version ?? "unknown"}, reason=${record.provenance_reason ?? "unknown"}, migration delta=${record.provenance_migration_delta ?? "unknown"}. Inspect the session database and runner backup before continuing.`
|
|
1693
|
+
};
|
|
1630
1694
|
default:
|
|
1631
1695
|
return null;
|
|
1632
1696
|
}
|
|
@@ -1641,7 +1705,8 @@ var OUTCOMES = /* @__PURE__ */ new Set([
|
|
|
1641
1705
|
"fresh_session_db",
|
|
1642
1706
|
"history_rolled_back",
|
|
1643
1707
|
"restore_misconfigured",
|
|
1644
|
-
"session_db_boot_refused"
|
|
1708
|
+
"session_db_boot_refused",
|
|
1709
|
+
"schema_provenance_mismatch"
|
|
1645
1710
|
]);
|
|
1646
1711
|
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1647
1712
|
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
@@ -1659,7 +1724,7 @@ var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"]
|
|
|
1659
1724
|
function isSessionDbRecoveryRecord(value) {
|
|
1660
1725
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1661
1726
|
const record = value;
|
|
1662
|
-
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1727
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && (record.provenance_reason === void 0 || record.provenance_reason === null || typeof record.provenance_reason === "string") && (record.provenance_migration_delta === void 0 || record.provenance_migration_delta === null || Number.isInteger(record.provenance_migration_delta)) && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1663
1728
|
(field) => record[field] === null || typeof record[field] === "string"
|
|
1664
1729
|
);
|
|
1665
1730
|
}
|
|
@@ -1688,11 +1753,667 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1688
1753
|
if (health.healthy) {
|
|
1689
1754
|
return health;
|
|
1690
1755
|
}
|
|
1691
|
-
await new Promise((
|
|
1756
|
+
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
1692
1757
|
}
|
|
1693
1758
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1694
1759
|
}
|
|
1695
1760
|
|
|
1761
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1762
|
+
import { spawn as spawn2 } from "node:child_process";
|
|
1763
|
+
import { mkdirSync, statSync as statSync2, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
1764
|
+
import { homedir as homedir2 } from "node:os";
|
|
1765
|
+
import { dirname as dirname2, resolve as resolvePath } from "node:path";
|
|
1766
|
+
|
|
1767
|
+
// src/lib/runner-synchroniser.ts
|
|
1768
|
+
import { spawn } from "node:child_process";
|
|
1769
|
+
function appendError(stderr, error2) {
|
|
1770
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1771
|
+
return stderr === "" ? message : `${stderr}
|
|
1772
|
+
${message}`;
|
|
1773
|
+
}
|
|
1774
|
+
function runSynchroniser(args, opts) {
|
|
1775
|
+
return new Promise((resolve4) => {
|
|
1776
|
+
let child;
|
|
1777
|
+
let stdout = "";
|
|
1778
|
+
let stderr = "";
|
|
1779
|
+
let settled = false;
|
|
1780
|
+
const timer = {};
|
|
1781
|
+
let abortListener;
|
|
1782
|
+
let spawnListener;
|
|
1783
|
+
const finish = (result) => {
|
|
1784
|
+
if (settled) return;
|
|
1785
|
+
settled = true;
|
|
1786
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
1787
|
+
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
1788
|
+
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
1789
|
+
resolve4(result);
|
|
1790
|
+
};
|
|
1791
|
+
try {
|
|
1792
|
+
child = spawn("runner-synchroniser", args, {
|
|
1793
|
+
env: opts.env ?? process.env,
|
|
1794
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1795
|
+
});
|
|
1796
|
+
} catch (error2) {
|
|
1797
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1798
|
+
return;
|
|
1799
|
+
}
|
|
1800
|
+
child.stdout?.setEncoding("utf8");
|
|
1801
|
+
child.stdout?.on("data", (chunk) => {
|
|
1802
|
+
stdout += chunk;
|
|
1803
|
+
});
|
|
1804
|
+
child.stderr?.setEncoding("utf8");
|
|
1805
|
+
child.stderr?.on("data", (chunk) => {
|
|
1806
|
+
stderr += chunk;
|
|
1807
|
+
});
|
|
1808
|
+
child.once("error", (error2) => {
|
|
1809
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1810
|
+
});
|
|
1811
|
+
child.once("close", (code) => {
|
|
1812
|
+
finish({ code, stdout, stderr, timedOut: false });
|
|
1813
|
+
});
|
|
1814
|
+
if (opts.signal) {
|
|
1815
|
+
const killChild = () => {
|
|
1816
|
+
if (child.pid === void 0) {
|
|
1817
|
+
if (!spawnListener) {
|
|
1818
|
+
spawnListener = killChild;
|
|
1819
|
+
child.once("spawn", spawnListener);
|
|
1820
|
+
}
|
|
1821
|
+
return;
|
|
1822
|
+
}
|
|
1823
|
+
child.kill("SIGKILL");
|
|
1824
|
+
};
|
|
1825
|
+
abortListener = killChild;
|
|
1826
|
+
if (opts.signal.aborted) {
|
|
1827
|
+
abortListener();
|
|
1828
|
+
} else {
|
|
1829
|
+
opts.signal.addEventListener("abort", abortListener, { once: true });
|
|
1830
|
+
if (opts.signal.aborted) abortListener();
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
timer.handle = setTimeout(
|
|
1834
|
+
() => {
|
|
1835
|
+
child.kill("SIGKILL");
|
|
1836
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1837
|
+
},
|
|
1838
|
+
Math.max(0, opts.timeoutMs)
|
|
1839
|
+
);
|
|
1840
|
+
});
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1844
|
+
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
1845
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
|
|
1846
|
+
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
1847
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
|
|
1848
|
+
function commandError(result) {
|
|
1849
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
1850
|
+
}
|
|
1851
|
+
function reportRecord(stage, outcome, reason, litestreamExitCode, options) {
|
|
1852
|
+
options.reportRecovery({
|
|
1853
|
+
v: 1,
|
|
1854
|
+
event: "session_db_recovery",
|
|
1855
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1856
|
+
stage,
|
|
1857
|
+
outcome,
|
|
1858
|
+
severity: "error",
|
|
1859
|
+
reason,
|
|
1860
|
+
litestream_exit_code: litestreamExitCode,
|
|
1861
|
+
attempt: null,
|
|
1862
|
+
replica_objects: null,
|
|
1863
|
+
replica_bytes: null,
|
|
1864
|
+
quarantine_destination: null,
|
|
1865
|
+
quarantined_objects: null,
|
|
1866
|
+
quarantine_failed_objects: null,
|
|
1867
|
+
quarantined_bytes: null,
|
|
1868
|
+
verified_restore_point: null,
|
|
1869
|
+
restore_points_tried: null,
|
|
1870
|
+
provenance_reason: null,
|
|
1871
|
+
provenance_migration_delta: null,
|
|
1872
|
+
replication_suspended: stage === "restore"
|
|
1873
|
+
});
|
|
1874
|
+
}
|
|
1875
|
+
function clearMarker(options) {
|
|
1876
|
+
if (!options.noReplicateMarker) return;
|
|
1877
|
+
try {
|
|
1878
|
+
unlinkSync2(options.noReplicateMarker);
|
|
1879
|
+
} catch (error2) {
|
|
1880
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1881
|
+
options.log(
|
|
1882
|
+
`Could not clear the previous session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1883
|
+
"warn"
|
|
1884
|
+
);
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
function markNoReplicate(options, message) {
|
|
1888
|
+
if (options.noReplicateMarker) {
|
|
1889
|
+
try {
|
|
1890
|
+
mkdirSync(dirname2(options.noReplicateMarker), { recursive: true });
|
|
1891
|
+
writeFileSync(options.noReplicateMarker, "");
|
|
1892
|
+
} catch (error2) {
|
|
1893
|
+
options.log(
|
|
1894
|
+
`Could not write session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1895
|
+
"error"
|
|
1896
|
+
);
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
options.log(`SESSION-DB-NO-REPLICATE: ${message}`, "warn");
|
|
1900
|
+
}
|
|
1901
|
+
function discardSessionDbDebris(options) {
|
|
1902
|
+
for (const path of [options.dbPath, `${options.dbPath}-wal`, `${options.dbPath}-shm`]) {
|
|
1903
|
+
try {
|
|
1904
|
+
unlinkSync2(path);
|
|
1905
|
+
} catch (error2) {
|
|
1906
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") continue;
|
|
1907
|
+
options.log(
|
|
1908
|
+
`Could not remove session-DB debris ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1909
|
+
"warn"
|
|
1910
|
+
);
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
function splitDiagnostics(text) {
|
|
1915
|
+
return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
1916
|
+
}
|
|
1917
|
+
function logSynchroniserDiagnostics(result, options) {
|
|
1918
|
+
for (const line of splitDiagnostics(result.stderr)) options.log(line, "warn");
|
|
1919
|
+
}
|
|
1920
|
+
function parseSingleQuotedAssignment(line) {
|
|
1921
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
1922
|
+
if (!match || !match[2].startsWith("'")) return null;
|
|
1923
|
+
const valueSource = match[2];
|
|
1924
|
+
let value = "";
|
|
1925
|
+
for (let index = 1; index < valueSource.length; index++) {
|
|
1926
|
+
const character = valueSource[index];
|
|
1927
|
+
if (character !== "'") {
|
|
1928
|
+
value += character;
|
|
1929
|
+
continue;
|
|
1930
|
+
}
|
|
1931
|
+
if (index === valueSource.length - 1) return [match[1], value];
|
|
1932
|
+
if (valueSource.slice(index + 1, index + 4) !== "\\''") return null;
|
|
1933
|
+
value += "'";
|
|
1934
|
+
index += 3;
|
|
1935
|
+
}
|
|
1936
|
+
return null;
|
|
1937
|
+
}
|
|
1938
|
+
function parseSynchroniserEnv(stdout) {
|
|
1939
|
+
const values = {};
|
|
1940
|
+
for (const line of stdout.split("\n")) {
|
|
1941
|
+
if (line.trim() === "") continue;
|
|
1942
|
+
const assignment = parseSingleQuotedAssignment(line);
|
|
1943
|
+
if (!assignment) return null;
|
|
1944
|
+
values[assignment[0]] = assignment[1];
|
|
1945
|
+
}
|
|
1946
|
+
return values;
|
|
1947
|
+
}
|
|
1948
|
+
function runCommand(command, args, options) {
|
|
1949
|
+
return new Promise((resolve4) => {
|
|
1950
|
+
let child;
|
|
1951
|
+
let stdout = "";
|
|
1952
|
+
let stderr = "";
|
|
1953
|
+
let settled = false;
|
|
1954
|
+
const finish = (result) => {
|
|
1955
|
+
if (settled) return;
|
|
1956
|
+
settled = true;
|
|
1957
|
+
if (timer) clearTimeout(timer);
|
|
1958
|
+
resolve4(result);
|
|
1959
|
+
};
|
|
1960
|
+
try {
|
|
1961
|
+
child = spawn2(command, args, {
|
|
1962
|
+
env: options.env,
|
|
1963
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1964
|
+
});
|
|
1965
|
+
} catch (error2) {
|
|
1966
|
+
resolve4({
|
|
1967
|
+
code: null,
|
|
1968
|
+
stdout,
|
|
1969
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
1970
|
+
timedOut: false
|
|
1971
|
+
});
|
|
1972
|
+
return;
|
|
1973
|
+
}
|
|
1974
|
+
child.stdout?.setEncoding("utf8");
|
|
1975
|
+
child.stdout?.on("data", (chunk) => {
|
|
1976
|
+
stdout += chunk;
|
|
1977
|
+
});
|
|
1978
|
+
child.stderr?.setEncoding("utf8");
|
|
1979
|
+
child.stderr?.on("data", (chunk) => {
|
|
1980
|
+
stderr += chunk;
|
|
1981
|
+
});
|
|
1982
|
+
child.once("error", (error2) => {
|
|
1983
|
+
finish({
|
|
1984
|
+
code: null,
|
|
1985
|
+
stdout,
|
|
1986
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
1987
|
+
${error2.message}`,
|
|
1988
|
+
timedOut: false
|
|
1989
|
+
});
|
|
1990
|
+
});
|
|
1991
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
1992
|
+
const timer = setTimeout(
|
|
1993
|
+
() => {
|
|
1994
|
+
child.kill("SIGKILL");
|
|
1995
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1996
|
+
},
|
|
1997
|
+
Math.max(0, options.timeoutMs)
|
|
1998
|
+
);
|
|
1999
|
+
});
|
|
2000
|
+
}
|
|
2001
|
+
async function ensureLitestreamConfig(options, env) {
|
|
2002
|
+
const configPath = options.litestreamConfig;
|
|
2003
|
+
if (!configPath) {
|
|
2004
|
+
markNoReplicate(options, "no Litestream configuration path was provided");
|
|
2005
|
+
reportRecord(
|
|
2006
|
+
"restore",
|
|
2007
|
+
"restore_misconfigured",
|
|
2008
|
+
"litestream_config_unavailable",
|
|
2009
|
+
null,
|
|
2010
|
+
options
|
|
2011
|
+
);
|
|
2012
|
+
return null;
|
|
2013
|
+
}
|
|
2014
|
+
try {
|
|
2015
|
+
if (statSync2(configPath).size > 0) return configPath;
|
|
2016
|
+
} catch (error2) {
|
|
2017
|
+
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
2018
|
+
options.log(
|
|
2019
|
+
`Could not inspect ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2020
|
+
"warn"
|
|
2021
|
+
);
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
const rendered = await runSynchroniser(["litestream-config"], {
|
|
2025
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2026
|
+
env
|
|
2027
|
+
});
|
|
2028
|
+
logSynchroniserDiagnostics(rendered, options);
|
|
2029
|
+
if (rendered.timedOut || rendered.code !== 0) {
|
|
2030
|
+
options.log(
|
|
2031
|
+
`Could not generate ${configPath}: runner-synchroniser litestream-config failed (${commandError(rendered)})`,
|
|
2032
|
+
"error"
|
|
2033
|
+
);
|
|
2034
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2035
|
+
reportRecord(
|
|
2036
|
+
"restore",
|
|
2037
|
+
"restore_misconfigured",
|
|
2038
|
+
"litestream_config_unavailable",
|
|
2039
|
+
null,
|
|
2040
|
+
options
|
|
2041
|
+
);
|
|
2042
|
+
return null;
|
|
2043
|
+
}
|
|
2044
|
+
try {
|
|
2045
|
+
mkdirSync(dirname2(configPath), { recursive: true });
|
|
2046
|
+
writeFileSync(configPath, rendered.stdout);
|
|
2047
|
+
} catch (error2) {
|
|
2048
|
+
options.log(
|
|
2049
|
+
`Could not write ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2050
|
+
"error"
|
|
2051
|
+
);
|
|
2052
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2053
|
+
reportRecord(
|
|
2054
|
+
"restore",
|
|
2055
|
+
"restore_misconfigured",
|
|
2056
|
+
"litestream_config_unavailable",
|
|
2057
|
+
null,
|
|
2058
|
+
options
|
|
2059
|
+
);
|
|
2060
|
+
return null;
|
|
2061
|
+
}
|
|
2062
|
+
const version2 = await runCommand("litestream", ["version"], {
|
|
2063
|
+
env,
|
|
2064
|
+
timeoutMs: 1e4
|
|
2065
|
+
});
|
|
2066
|
+
const litestreamVersion = version2.code === 0 ? version2.stdout.trim() || "unknown" : "unknown";
|
|
2067
|
+
const regionEmpty = !/^\s*region:\s*\S+/m.test(rendered.stdout);
|
|
2068
|
+
options.log(
|
|
2069
|
+
`litestream ${litestreamVersion}; AWS_REGION=${env.AWS_REGION ?? "<unset>"} AWS_DEFAULT_REGION=${env.AWS_DEFAULT_REGION ?? "<unset>"}; rendered litestream.yml region empty: ${regionEmpty ? "yes" : "no"}`
|
|
2070
|
+
);
|
|
2071
|
+
return configPath;
|
|
2072
|
+
}
|
|
2073
|
+
function restoreGiveUp(options, reason, message, litestreamExitCode, outcome = "fresh_session_db") {
|
|
2074
|
+
discardSessionDbDebris(options);
|
|
2075
|
+
markNoReplicate(options, message);
|
|
2076
|
+
reportRecord("restore", outcome, reason, litestreamExitCode, options);
|
|
2077
|
+
}
|
|
2078
|
+
async function restoreSessionDb(options, configPath, env) {
|
|
2079
|
+
const restored = await runCommand(
|
|
2080
|
+
"litestream",
|
|
2081
|
+
["restore", "-config", configPath, "-if-db-not-exists", "-if-replica-exists", options.dbPath],
|
|
2082
|
+
{ env, timeoutMs: SESSION_DB_RESTORE_TIMEOUT_MS }
|
|
2083
|
+
);
|
|
2084
|
+
for (const line of splitDiagnostics(restored.stderr)) options.log(line, "warn");
|
|
2085
|
+
if (restored.timedOut || restored.code === 124 || restored.code === 137) {
|
|
2086
|
+
restoreGiveUp(
|
|
2087
|
+
options,
|
|
2088
|
+
"restore_deadline_exceeded",
|
|
2089
|
+
`SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within ${SESSION_DB_RESTORE_TIMEOUT_MS}ms; opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2090
|
+
restored.code ?? 124
|
|
2091
|
+
);
|
|
2092
|
+
return;
|
|
2093
|
+
}
|
|
2094
|
+
if (restored.code === null || restored.code === 125 || restored.code === 126 || restored.code === 127) {
|
|
2095
|
+
options.log(`litestream restore could not run (${commandError(restored)})`, "error");
|
|
2096
|
+
restoreGiveUp(
|
|
2097
|
+
options,
|
|
2098
|
+
"restore_tool_unusable",
|
|
2099
|
+
`restore tool is broken (${commandError(restored)}); opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2100
|
+
restored.code
|
|
2101
|
+
);
|
|
2102
|
+
return;
|
|
2103
|
+
}
|
|
2104
|
+
const classified = await runSynchroniser(
|
|
2105
|
+
[
|
|
2106
|
+
"session-db-classify",
|
|
2107
|
+
String(restored.code ?? 1),
|
|
2108
|
+
"1",
|
|
2109
|
+
"--on-unusable-replica=leave",
|
|
2110
|
+
"--fresh-db-fallback"
|
|
2111
|
+
],
|
|
2112
|
+
{ timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS, env }
|
|
2113
|
+
);
|
|
2114
|
+
logSynchroniserDiagnostics(classified, options);
|
|
2115
|
+
const classifyCode = classified.code;
|
|
2116
|
+
switch (classifyCode) {
|
|
2117
|
+
case 0:
|
|
2118
|
+
return;
|
|
2119
|
+
case 31:
|
|
2120
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2121
|
+
options.log(
|
|
2122
|
+
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
2123
|
+
"warn"
|
|
2124
|
+
);
|
|
2125
|
+
return;
|
|
2126
|
+
case 32:
|
|
2127
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2128
|
+
discardSessionDbDebris(options);
|
|
2129
|
+
markNoReplicate(
|
|
2130
|
+
options,
|
|
2131
|
+
"session-db-classify asked for another restore attempt (32), but this boot has budget for only one; treating it as a give-up rather than retrying"
|
|
2132
|
+
);
|
|
2133
|
+
return;
|
|
2134
|
+
case 30:
|
|
2135
|
+
restoreGiveUp(
|
|
2136
|
+
options,
|
|
2137
|
+
"classification_fatal",
|
|
2138
|
+
"session-db-classify returned fatal (30); see the FATAL message above",
|
|
2139
|
+
restored.code,
|
|
2140
|
+
"restore_misconfigured"
|
|
2141
|
+
);
|
|
2142
|
+
return;
|
|
2143
|
+
default:
|
|
2144
|
+
restoreGiveUp(
|
|
2145
|
+
options,
|
|
2146
|
+
"classification_unrecognised",
|
|
2147
|
+
`session-db-classify exited ${classifyCode ?? "null"}, which is none of its documented answers`,
|
|
2148
|
+
restored.code
|
|
2149
|
+
);
|
|
2150
|
+
}
|
|
2151
|
+
}
|
|
2152
|
+
async function verifySessionDb(options, configPath, env) {
|
|
2153
|
+
if (!options.noReplicateMarker || !fileExists(options.noReplicateMarker)) {
|
|
2154
|
+
const result = await runSynchroniser(["session-db-verify", configPath], {
|
|
2155
|
+
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
2156
|
+
env: {
|
|
2157
|
+
...env,
|
|
2158
|
+
// The synchroniser reads this value in SECONDS. Keep this at 120, not
|
|
2159
|
+
// 120_000, so the walkback gives up before the outer process bound.
|
|
2160
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
2161
|
+
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
2162
|
+
)
|
|
2163
|
+
}
|
|
2164
|
+
});
|
|
2165
|
+
logSynchroniserDiagnostics(result, options);
|
|
2166
|
+
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
2167
|
+
options.log(
|
|
2168
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
2169
|
+
"warn"
|
|
2170
|
+
);
|
|
2171
|
+
return false;
|
|
2172
|
+
}
|
|
2173
|
+
if (result.code === 34) {
|
|
2174
|
+
reportRecord(
|
|
2175
|
+
"verify",
|
|
2176
|
+
"session_db_boot_refused",
|
|
2177
|
+
result.stderr.includes("SESSION-DB-LOCAL-DISCARD-FAILED") ? "local_discard_failed" : "replica_separation_unproven",
|
|
2178
|
+
null,
|
|
2179
|
+
options
|
|
2180
|
+
);
|
|
2181
|
+
return true;
|
|
2182
|
+
}
|
|
2183
|
+
if (result.code === 33) {
|
|
2184
|
+
options.log(
|
|
2185
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
2186
|
+
"warn"
|
|
2187
|
+
);
|
|
2188
|
+
return false;
|
|
2189
|
+
}
|
|
2190
|
+
if (result.code !== 0) {
|
|
2191
|
+
options.log(
|
|
2192
|
+
`SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${result.code ?? "null"}, which is none of its documented answers; continuing with the restored opencode.db as-is`,
|
|
2193
|
+
"warn"
|
|
2194
|
+
);
|
|
2195
|
+
}
|
|
2196
|
+
return false;
|
|
2197
|
+
}
|
|
2198
|
+
options.log(
|
|
2199
|
+
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
2200
|
+
"debug"
|
|
2201
|
+
);
|
|
2202
|
+
return false;
|
|
2203
|
+
}
|
|
2204
|
+
function fileExists(path) {
|
|
2205
|
+
try {
|
|
2206
|
+
statSync2(path);
|
|
2207
|
+
return true;
|
|
2208
|
+
} catch (error2) {
|
|
2209
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
2210
|
+
return true;
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
async function restoreAndVerifySessionDb(options) {
|
|
2214
|
+
const env = options.env ?? process.env;
|
|
2215
|
+
clearMarker(options);
|
|
2216
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2217
|
+
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
2218
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2219
|
+
env
|
|
2220
|
+
});
|
|
2221
|
+
logSynchroniserDiagnostics(synchroniserEnv, options);
|
|
2222
|
+
if (synchroniserEnv.timedOut || synchroniserEnv.code !== 0) {
|
|
2223
|
+
options.log(
|
|
2224
|
+
`runner-synchroniser env could not resolve the session-DB configuration (${commandError(synchroniserEnv)})`,
|
|
2225
|
+
"error"
|
|
2226
|
+
);
|
|
2227
|
+
markNoReplicate(
|
|
2228
|
+
options,
|
|
2229
|
+
"could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
2230
|
+
);
|
|
2231
|
+
reportRecord(
|
|
2232
|
+
"restore",
|
|
2233
|
+
"restore_misconfigured",
|
|
2234
|
+
"synchroniser_config_unresolved",
|
|
2235
|
+
null,
|
|
2236
|
+
options
|
|
2237
|
+
);
|
|
2238
|
+
return { verifyFatal: false };
|
|
2239
|
+
}
|
|
2240
|
+
const values = parseSynchroniserEnv(synchroniserEnv.stdout);
|
|
2241
|
+
if (!values) {
|
|
2242
|
+
markNoReplicate(
|
|
2243
|
+
options,
|
|
2244
|
+
"the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
2245
|
+
);
|
|
2246
|
+
reportRecord(
|
|
2247
|
+
"restore",
|
|
2248
|
+
"restore_misconfigured",
|
|
2249
|
+
"synchroniser_config_unevaluable",
|
|
2250
|
+
null,
|
|
2251
|
+
options
|
|
2252
|
+
);
|
|
2253
|
+
return { verifyFatal: false };
|
|
2254
|
+
}
|
|
2255
|
+
const synchroniserDbPath = values.OPENCODE_DB_PATH;
|
|
2256
|
+
if (!synchroniserDbPath) {
|
|
2257
|
+
markNoReplicate(
|
|
2258
|
+
options,
|
|
2259
|
+
"run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
2260
|
+
);
|
|
2261
|
+
reportRecord(
|
|
2262
|
+
"restore",
|
|
2263
|
+
"restore_misconfigured",
|
|
2264
|
+
"synchroniser_config_incomplete",
|
|
2265
|
+
null,
|
|
2266
|
+
options
|
|
2267
|
+
);
|
|
2268
|
+
return { verifyFatal: false };
|
|
2269
|
+
}
|
|
2270
|
+
if (resolvePath(synchroniserDbPath) !== resolvePath(options.dbPath)) {
|
|
2271
|
+
options.log(
|
|
2272
|
+
`runner-synchroniser reported OPENCODE_DB_PATH=${synchroniserDbPath}, but OpenCode uses ${options.dbPath}; continuing with OpenCode's session-DB path`,
|
|
2273
|
+
"warn"
|
|
2274
|
+
);
|
|
2275
|
+
}
|
|
2276
|
+
if (!values.PERSISTENCE_BUCKET) {
|
|
2277
|
+
options.log(
|
|
2278
|
+
"SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated.",
|
|
2279
|
+
"warn"
|
|
2280
|
+
);
|
|
2281
|
+
return { verifyFatal: false };
|
|
2282
|
+
}
|
|
2283
|
+
const configPath = await ensureLitestreamConfig(options, env);
|
|
2284
|
+
if (!configPath) return { verifyFatal: false };
|
|
2285
|
+
await restoreSessionDb(options, configPath, env);
|
|
2286
|
+
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
2287
|
+
return { verifyFatal: false };
|
|
2288
|
+
}
|
|
2289
|
+
return { verifyFatal: await verifySessionDb(options, configPath, env) };
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2292
|
+
// src/lib/opencode/session-db-provenance.ts
|
|
2293
|
+
import { createRequire } from "node:module";
|
|
2294
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
|
|
2295
|
+
import { dirname as dirname3, join as join3 } from "node:path";
|
|
2296
|
+
var require2 = createRequire(import.meta.url);
|
|
2297
|
+
function readSessionDbMigrationIds(dbPath) {
|
|
2298
|
+
let db;
|
|
2299
|
+
try {
|
|
2300
|
+
const { DatabaseSync } = require2("node:sqlite");
|
|
2301
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
2302
|
+
const columns = db.prepare("PRAGMA table_info(migration)").all();
|
|
2303
|
+
const hasExpectedShape = columns.length === 2 && columns.some(
|
|
2304
|
+
(column) => column.name === "id" && typeof column.type === "string" && column.type.toUpperCase() === "TEXT" && column.pk === 1
|
|
2305
|
+
) && columns.some(
|
|
2306
|
+
(column) => column.name === "time_completed" && typeof column.type === "string" && column.type.toUpperCase() === "INTEGER" && column.notnull === 1 && column.pk === 0
|
|
2307
|
+
);
|
|
2308
|
+
if (!hasExpectedShape) {
|
|
2309
|
+
console.warn(`[readSessionDbMigrationIds] unsupported migration table shape in ${dbPath}`);
|
|
2310
|
+
return null;
|
|
2311
|
+
}
|
|
2312
|
+
const rows = db.prepare("SELECT id FROM migration ORDER BY id").all();
|
|
2313
|
+
if (rows.some((row) => typeof row.id !== "string")) return null;
|
|
2314
|
+
return rows.map((row) => row.id);
|
|
2315
|
+
} catch (error2) {
|
|
2316
|
+
console.warn(
|
|
2317
|
+
`[readSessionDbMigrationIds] could not read migration history from ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2318
|
+
);
|
|
2319
|
+
return null;
|
|
2320
|
+
} finally {
|
|
2321
|
+
try {
|
|
2322
|
+
db?.close();
|
|
2323
|
+
} catch (error2) {
|
|
2324
|
+
console.warn(
|
|
2325
|
+
`[readSessionDbMigrationIds] could not close ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2326
|
+
);
|
|
2327
|
+
}
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
2331
|
+
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
2332
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
2333
|
+
}
|
|
2334
|
+
function loadSessionDbProvenanceState(path) {
|
|
2335
|
+
let value;
|
|
2336
|
+
try {
|
|
2337
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
2338
|
+
} catch (error2) {
|
|
2339
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return {};
|
|
2340
|
+
console.error(
|
|
2341
|
+
`[session-db-provenance] could not read ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2342
|
+
);
|
|
2343
|
+
return {};
|
|
2344
|
+
}
|
|
2345
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2346
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2347
|
+
return {};
|
|
2348
|
+
}
|
|
2349
|
+
const state = {};
|
|
2350
|
+
for (const [dbPath, record] of Object.entries(value)) {
|
|
2351
|
+
if (!isSessionDbProvenanceRecord(record)) {
|
|
2352
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2353
|
+
return {};
|
|
2354
|
+
}
|
|
2355
|
+
state[dbPath] = record;
|
|
2356
|
+
}
|
|
2357
|
+
return state;
|
|
2358
|
+
}
|
|
2359
|
+
function saveSessionDbProvenanceState(path, state) {
|
|
2360
|
+
try {
|
|
2361
|
+
mkdirSync2(dirname3(path), { recursive: true });
|
|
2362
|
+
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
2363
|
+
`, "utf8");
|
|
2364
|
+
} catch (error2) {
|
|
2365
|
+
console.error(
|
|
2366
|
+
`[session-db-provenance] could not write ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2367
|
+
);
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
function evaluateSessionDbProvenance(input) {
|
|
2371
|
+
const { currentVersion, currentIds, previous } = input;
|
|
2372
|
+
if (!previous) return { anomaly: false, reason: null };
|
|
2373
|
+
const current = new Set(currentIds);
|
|
2374
|
+
const prior = new Set(previous.migrationIds);
|
|
2375
|
+
for (const id of prior) {
|
|
2376
|
+
if (!current.has(id)) return { anomaly: true, reason: "migration-history-regressed" };
|
|
2377
|
+
}
|
|
2378
|
+
if (current.size > prior.size && previous.opencodeVersion === currentVersion) {
|
|
2379
|
+
return { anomaly: true, reason: "foreign-version-migrations" };
|
|
2380
|
+
}
|
|
2381
|
+
return { anomaly: false, reason: null };
|
|
2382
|
+
}
|
|
2383
|
+
function checkSessionDbProvenance(input) {
|
|
2384
|
+
const { dbPath, currentVersion, homeDir, env } = input;
|
|
2385
|
+
const path = sessionDbProvenanceStatePath(homeDir, env);
|
|
2386
|
+
const state = loadSessionDbProvenanceState(path);
|
|
2387
|
+
const previous = state[dbPath];
|
|
2388
|
+
const currentIds = readSessionDbMigrationIds(dbPath);
|
|
2389
|
+
if (currentIds === null) {
|
|
2390
|
+
return {
|
|
2391
|
+
anomaly: false,
|
|
2392
|
+
reason: null,
|
|
2393
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2394
|
+
migrationDelta: null
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
const decision = evaluateSessionDbProvenance({ currentVersion, currentIds, previous });
|
|
2398
|
+
const migrationDelta = previous ? new Set(currentIds).size - new Set(previous.migrationIds).size : null;
|
|
2399
|
+
state[dbPath] = {
|
|
2400
|
+
opencodeVersion: currentVersion,
|
|
2401
|
+
migrationIds: currentIds,
|
|
2402
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2403
|
+
};
|
|
2404
|
+
saveSessionDbProvenanceState(path, state);
|
|
2405
|
+
return {
|
|
2406
|
+
...decision,
|
|
2407
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2408
|
+
migrationDelta
|
|
2409
|
+
};
|
|
2410
|
+
}
|
|
2411
|
+
function isSessionDbProvenanceRecord(value) {
|
|
2412
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2413
|
+
const record = value;
|
|
2414
|
+
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
2415
|
+
}
|
|
2416
|
+
|
|
1696
2417
|
// src/lib/opencode/opencode-version-gate.ts
|
|
1697
2418
|
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
1698
2419
|
function isQueueValidatedVersion(version2) {
|
|
@@ -1707,7 +2428,7 @@ function buildOpenCodeVersionWarning(version2) {
|
|
|
1707
2428
|
}
|
|
1708
2429
|
|
|
1709
2430
|
// src/lib/opencode/process.ts
|
|
1710
|
-
import { execSync, spawn } from "child_process";
|
|
2431
|
+
import { execSync, spawn as spawn3 } from "child_process";
|
|
1711
2432
|
|
|
1712
2433
|
// src/lib/process-stop.ts
|
|
1713
2434
|
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
@@ -1717,7 +2438,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1717
2438
|
if (child.exitCode !== null || child.signalCode !== null) {
|
|
1718
2439
|
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
1719
2440
|
}
|
|
1720
|
-
return new Promise((
|
|
2441
|
+
return new Promise((resolve4, reject) => {
|
|
1721
2442
|
let forced = false;
|
|
1722
2443
|
let settled = false;
|
|
1723
2444
|
const timer = setTimeout(() => {
|
|
@@ -1737,7 +2458,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1737
2458
|
settled = true;
|
|
1738
2459
|
clearTimeout(timer);
|
|
1739
2460
|
child.removeListener("exit", onExit);
|
|
1740
|
-
|
|
2461
|
+
resolve4(result);
|
|
1741
2462
|
};
|
|
1742
2463
|
const fail = (error2) => {
|
|
1743
2464
|
if (settled) return;
|
|
@@ -1765,6 +2486,17 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1765
2486
|
|
|
1766
2487
|
// src/lib/opencode/process.ts
|
|
1767
2488
|
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
2489
|
+
var VALID_OPENCODE_LOG_LEVELS = /* @__PURE__ */ new Set(["DEBUG", "INFO", "WARN", "ERROR"]);
|
|
2490
|
+
function resolveOpenCodeLogLevel(env) {
|
|
2491
|
+
const raw = env.OPENCODE_LOG_LEVEL;
|
|
2492
|
+
if (!raw) return "INFO";
|
|
2493
|
+
const upper = raw.toUpperCase();
|
|
2494
|
+
if (VALID_OPENCODE_LOG_LEVELS.has(upper)) return upper;
|
|
2495
|
+
console.warn(
|
|
2496
|
+
`startOpenCode: ignoring invalid OPENCODE_LOG_LEVEL "${raw}" (expected DEBUG|INFO|WARN|ERROR) \u2014 using INFO`
|
|
2497
|
+
);
|
|
2498
|
+
return "INFO";
|
|
2499
|
+
}
|
|
1768
2500
|
function getProcessCwd(pid) {
|
|
1769
2501
|
const platform = process.platform;
|
|
1770
2502
|
try {
|
|
@@ -1813,14 +2545,14 @@ function findAvailablePort(startPort, maxAttempts = 10) {
|
|
|
1813
2545
|
}
|
|
1814
2546
|
return null;
|
|
1815
2547
|
}
|
|
1816
|
-
function
|
|
2548
|
+
function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
1817
2549
|
const instances = [];
|
|
1818
2550
|
try {
|
|
1819
2551
|
const platform = process.platform;
|
|
1820
2552
|
if (platform === "darwin" || platform === "linux") {
|
|
1821
2553
|
let pids = [];
|
|
1822
2554
|
try {
|
|
1823
|
-
const pgrepOutput = execSync(
|
|
2555
|
+
const pgrepOutput = execSync(`pgrep -f "${pgrepPattern}"`, {
|
|
1824
2556
|
encoding: "utf-8",
|
|
1825
2557
|
stdio: ["pipe", "pipe", "pipe"]
|
|
1826
2558
|
}).trim();
|
|
@@ -1829,7 +2561,7 @@ function findOpenCodeProcesses() {
|
|
|
1829
2561
|
}
|
|
1830
2562
|
} catch {
|
|
1831
2563
|
try {
|
|
1832
|
-
const psOutput = execSync(
|
|
2564
|
+
const psOutput = execSync(`ps aux | grep -E "${psPattern}" | grep -v grep`, {
|
|
1833
2565
|
encoding: "utf-8",
|
|
1834
2566
|
stdio: ["pipe", "pipe", "pipe"]
|
|
1835
2567
|
}).trim();
|
|
@@ -1875,6 +2607,9 @@ function findOpenCodeProcesses() {
|
|
|
1875
2607
|
}
|
|
1876
2608
|
return instances;
|
|
1877
2609
|
}
|
|
2610
|
+
function findOpenCodeProcesses() {
|
|
2611
|
+
return findProcessesByPattern("opencode serve|opencode-serve", "opencode (serve|--port)");
|
|
2612
|
+
}
|
|
1878
2613
|
async function scanPortsForOpenCode() {
|
|
1879
2614
|
const instances = [];
|
|
1880
2615
|
const checks = OPENCODE_PORT_RANGE.map(async (port) => {
|
|
@@ -1919,18 +2654,27 @@ async function findHealthyOpenCodeInstances() {
|
|
|
1919
2654
|
}
|
|
1920
2655
|
return healthy;
|
|
1921
2656
|
}
|
|
1922
|
-
async function startOpenCode(port) {
|
|
2657
|
+
async function startOpenCode(port, options = {}) {
|
|
1923
2658
|
let command = "opencode";
|
|
1924
|
-
|
|
2659
|
+
const printLogs = options.inheritStdio ? ["--print-logs", "--log-level", resolveOpenCodeLogLevel(process.env)] : [];
|
|
2660
|
+
let args = ["serve", "--port", port.toString(), "--hostname", "127.0.0.1", ...printLogs];
|
|
1925
2661
|
try {
|
|
1926
2662
|
execSync("which opencode", { stdio: "ignore" });
|
|
1927
2663
|
} catch {
|
|
1928
2664
|
command = "npx";
|
|
1929
|
-
args = [
|
|
1930
|
-
|
|
1931
|
-
|
|
2665
|
+
args = [
|
|
2666
|
+
"opencode",
|
|
2667
|
+
"serve",
|
|
2668
|
+
"--port",
|
|
2669
|
+
port.toString(),
|
|
2670
|
+
"--hostname",
|
|
2671
|
+
"127.0.0.1",
|
|
2672
|
+
...printLogs
|
|
2673
|
+
];
|
|
2674
|
+
}
|
|
2675
|
+
const child = spawn3(command, args, {
|
|
1932
2676
|
detached: true,
|
|
1933
|
-
stdio: "ignore",
|
|
2677
|
+
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
1934
2678
|
cwd: process.cwd()
|
|
1935
2679
|
});
|
|
1936
2680
|
return child;
|
|
@@ -1969,16 +2713,33 @@ function isOpenCodeInstalled() {
|
|
|
1969
2713
|
return false;
|
|
1970
2714
|
}
|
|
1971
2715
|
}
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
2716
|
+
function isOpenCode2Installed() {
|
|
2717
|
+
try {
|
|
2718
|
+
const platform = process.platform;
|
|
2719
|
+
if (platform === "win32") {
|
|
2720
|
+
execSync2("where opencode2", { stdio: "ignore" });
|
|
2721
|
+
} else {
|
|
2722
|
+
execSync2("which opencode2", { stdio: "ignore" });
|
|
2723
|
+
}
|
|
2724
|
+
return true;
|
|
2725
|
+
} catch {
|
|
2726
|
+
return false;
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
async function promptOpenCodeInstall(interactive) {
|
|
2730
|
+
if (!interactive) {
|
|
2731
|
+
console.log(
|
|
2732
|
+
JSON.stringify({
|
|
1976
2733
|
status: "error",
|
|
1977
2734
|
error: "OpenCode is not installed",
|
|
1978
2735
|
install_url: OPENCODE_INSTALL_URL,
|
|
1979
2736
|
install_commands: {
|
|
1980
2737
|
npm: "npm install -g opencode-ai",
|
|
1981
|
-
curl: "curl -fsSL https://opencode.ai/install.sh | sh"
|
|
2738
|
+
curl: "curl -fsSL https://opencode.ai/install.sh | sh",
|
|
2739
|
+
v2: {
|
|
2740
|
+
npm: "npm install -g @opencode-ai/cli@beta",
|
|
2741
|
+
curl: "curl -fsSL https://opencode.ai/v2/install | bash"
|
|
2742
|
+
}
|
|
1982
2743
|
}
|
|
1983
2744
|
})
|
|
1984
2745
|
);
|
|
@@ -2416,7 +3177,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2416
3177
|
}
|
|
2417
3178
|
}
|
|
2418
3179
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2419
|
-
await new Promise((
|
|
3180
|
+
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
2420
3181
|
}
|
|
2421
3182
|
}
|
|
2422
3183
|
return null;
|
|
@@ -2462,21 +3223,112 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2462
3223
|
}
|
|
2463
3224
|
return lastOk ?? last;
|
|
2464
3225
|
}
|
|
2465
|
-
function
|
|
2466
|
-
if (!messages || messages.length === 0) return
|
|
2467
|
-
const
|
|
2468
|
-
(
|
|
3226
|
+
function collectSubagentSessions(messages, userMessageId) {
|
|
3227
|
+
if (!messages || messages.length === 0) return [];
|
|
3228
|
+
const byParent = messages.filter(
|
|
3229
|
+
(message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
|
|
2469
3230
|
);
|
|
2470
|
-
const
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
3231
|
+
const assistants = byParent.length > 0 ? byParent : [];
|
|
3232
|
+
if (assistants.length === 0) {
|
|
3233
|
+
const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
|
|
3234
|
+
if (userIndex === -1) return [];
|
|
3235
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
3236
|
+
const message = messages[i];
|
|
3237
|
+
if (roleOf(message) === "user") break;
|
|
3238
|
+
if (roleOf(message) === "assistant") assistants.push(message);
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
const refs = [];
|
|
3242
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3243
|
+
for (const message of assistants) {
|
|
3244
|
+
const parts = Array.isArray(message.parts) ? message.parts : [];
|
|
3245
|
+
for (const part of parts) {
|
|
3246
|
+
if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
|
|
3247
|
+
continue;
|
|
3248
|
+
const state = part.state;
|
|
3249
|
+
if (!state || typeof state !== "object") continue;
|
|
3250
|
+
const metadata = state.metadata;
|
|
3251
|
+
if (!metadata || typeof metadata !== "object") continue;
|
|
3252
|
+
const sessionId = metadata.sessionId;
|
|
3253
|
+
if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
|
|
3254
|
+
seen.add(sessionId);
|
|
3255
|
+
const start = state.time?.start;
|
|
3256
|
+
refs.push({
|
|
3257
|
+
sessionId,
|
|
3258
|
+
startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
|
|
3259
|
+
});
|
|
3260
|
+
}
|
|
2478
3261
|
}
|
|
2479
|
-
|
|
3262
|
+
return refs;
|
|
3263
|
+
}
|
|
3264
|
+
function finiteNumber(value) {
|
|
3265
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
3266
|
+
}
|
|
3267
|
+
function taskCallModel(value) {
|
|
3268
|
+
if (!value || typeof value !== "object") return null;
|
|
3269
|
+
const model = value;
|
|
3270
|
+
const modelID = typeof model.modelID === "string" ? model.modelID : void 0;
|
|
3271
|
+
const providerID = typeof model.providerID === "string" ? model.providerID : void 0;
|
|
3272
|
+
return modelID || providerID ? { modelID, providerID } : null;
|
|
3273
|
+
}
|
|
3274
|
+
function collectTaskCalls(messages, userMessageId) {
|
|
3275
|
+
if (!messages || messages.length === 0) return [];
|
|
3276
|
+
const calls = [];
|
|
3277
|
+
for (const message of messages) {
|
|
3278
|
+
if (roleOf(message) !== "assistant" || parentIdOf(message) !== userMessageId) continue;
|
|
3279
|
+
for (const part of message.parts ?? []) {
|
|
3280
|
+
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
3281
|
+
continue;
|
|
3282
|
+
}
|
|
3283
|
+
const rawName = part.state.input?.subagent_type;
|
|
3284
|
+
const subagentName = typeof rawName === "string" && rawName.trim().length > 0 ? rawName : rawName === void 0 ? "general" : "unknown";
|
|
3285
|
+
const metadata = part.state.metadata;
|
|
3286
|
+
calls.push({
|
|
3287
|
+
callID: part.callID,
|
|
3288
|
+
subagentName,
|
|
3289
|
+
childSessionId: typeof metadata?.sessionId === "string" ? metadata.sessionId : null,
|
|
3290
|
+
parentSessionId: typeof metadata?.parentSessionId === "string" ? metadata.parentSessionId : null,
|
|
3291
|
+
model: taskCallModel(metadata?.model),
|
|
3292
|
+
status: part.state.status ?? "unknown",
|
|
3293
|
+
timeStart: finiteNumber(part.state.time?.start),
|
|
3294
|
+
timeEnd: finiteNumber(part.state.time?.end)
|
|
3295
|
+
});
|
|
3296
|
+
}
|
|
3297
|
+
}
|
|
3298
|
+
return calls;
|
|
3299
|
+
}
|
|
3300
|
+
function attributeTaskCallUsage(messages, windows) {
|
|
3301
|
+
const eligibleWindows = windows.filter(
|
|
3302
|
+
(window) => window.timeStart !== null && Number.isFinite(window.timeStart)
|
|
3303
|
+
);
|
|
3304
|
+
const assignments = /* @__PURE__ */ new Map();
|
|
3305
|
+
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
3306
|
+
const unattributed = [];
|
|
3307
|
+
for (const message of messages ?? []) {
|
|
3308
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3309
|
+
const created = finiteNumber(createdOf(message));
|
|
3310
|
+
const matching = created === null ? [] : eligibleWindows.filter(
|
|
3311
|
+
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
3312
|
+
);
|
|
3313
|
+
if (matching.length === 0) {
|
|
3314
|
+
unattributed.push(message);
|
|
3315
|
+
continue;
|
|
3316
|
+
}
|
|
3317
|
+
matching.sort((a, b) => a.timeStart - b.timeStart);
|
|
3318
|
+
assignments.get(matching[0].callID)?.push(message);
|
|
3319
|
+
}
|
|
3320
|
+
return {
|
|
3321
|
+
invocations: eligibleWindows.map((window) => {
|
|
3322
|
+
const assigned = assignments.get(window.callID) ?? [];
|
|
3323
|
+
return { callID: window.callID, messages: assigned, usage: sumAssistantUsage(assigned) };
|
|
3324
|
+
}),
|
|
3325
|
+
unattributed
|
|
3326
|
+
};
|
|
3327
|
+
}
|
|
3328
|
+
function sumAssistantUsage(messages) {
|
|
3329
|
+
if (!messages || messages.length === 0) return null;
|
|
3330
|
+
const nonErrored = messages.filter((message) => errorOf(message) == null);
|
|
3331
|
+
const selected = nonErrored.length > 0 ? nonErrored : messages;
|
|
2480
3332
|
let sawAnyUsage = false;
|
|
2481
3333
|
let inputSum = 0;
|
|
2482
3334
|
let outputSum = 0;
|
|
@@ -2487,7 +3339,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2487
3339
|
let sawCost = false;
|
|
2488
3340
|
let modelId = null;
|
|
2489
3341
|
let providerId = null;
|
|
2490
|
-
for (const m of
|
|
3342
|
+
for (const m of selected) {
|
|
2491
3343
|
const info = m.info;
|
|
2492
3344
|
if (!info) continue;
|
|
2493
3345
|
const tokens = info.tokens;
|
|
@@ -2522,12 +3374,28 @@ function messageUsage(messages, userMessageId) {
|
|
|
2522
3374
|
usage_tokens_reasoning: reasoningSum,
|
|
2523
3375
|
usage_tokens_cache_read: cacheReadSum,
|
|
2524
3376
|
usage_tokens_cache_write: cacheWriteSum,
|
|
2525
|
-
// NULL means
|
|
2526
|
-
//
|
|
2527
|
-
// `sawCost` true with `costSum === 0`.
|
|
3377
|
+
// NULL means OpenCode never reported a cost; it is distinct from a genuine
|
|
3378
|
+
// zero-cost message, which sets `sawCost` with `costSum === 0`.
|
|
2528
3379
|
usage_cost_usd: sawCost ? costSum : null
|
|
2529
3380
|
};
|
|
2530
3381
|
}
|
|
3382
|
+
function messageUsage(messages, userMessageId) {
|
|
3383
|
+
if (!messages || messages.length === 0) return null;
|
|
3384
|
+
const byParentAll = messages.filter(
|
|
3385
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
3386
|
+
);
|
|
3387
|
+
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
3388
|
+
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
3389
|
+
let correlated;
|
|
3390
|
+
if (byParent.length > 0) {
|
|
3391
|
+
correlated = byParent;
|
|
3392
|
+
} else {
|
|
3393
|
+
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
3394
|
+
correlated = reply ? [reply] : [];
|
|
3395
|
+
}
|
|
3396
|
+
if (correlated.length === 0) return null;
|
|
3397
|
+
return sumAssistantUsage(correlated);
|
|
3398
|
+
}
|
|
2531
3399
|
function messageRunState(messages, userMessageId) {
|
|
2532
3400
|
if (!messages || messages.length === 0) return "unknown";
|
|
2533
3401
|
const hasUser = messages.some((m) => idOf(m) === userMessageId);
|
|
@@ -2590,8 +3458,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2590
3458
|
}
|
|
2591
3459
|
return false;
|
|
2592
3460
|
}
|
|
2593
|
-
function
|
|
2594
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3461
|
+
function classifyReplyAuthError(reply) {
|
|
2595
3462
|
const error2 = errorOf(reply);
|
|
2596
3463
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2597
3464
|
const e = error2;
|
|
@@ -2616,6 +3483,32 @@ function messageFailure(messages, userMessageId) {
|
|
|
2616
3483
|
}
|
|
2617
3484
|
return null;
|
|
2618
3485
|
}
|
|
3486
|
+
function messageFailure(messages, userMessageId) {
|
|
3487
|
+
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
3488
|
+
}
|
|
3489
|
+
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
3490
|
+
if (!messages || messages.length === 0) return null;
|
|
3491
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3492
|
+
const message = messages[i];
|
|
3493
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3494
|
+
const created = createdOf(message);
|
|
3495
|
+
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
3496
|
+
const failure = classifyReplyAuthError(message);
|
|
3497
|
+
if (failure) {
|
|
3498
|
+
if (!failure.providerId) return null;
|
|
3499
|
+
return { providerId: failure.providerId, outcome: "failed", failure };
|
|
3500
|
+
}
|
|
3501
|
+
const providerId = message.info?.providerID;
|
|
3502
|
+
if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
|
|
3503
|
+
return { providerId, outcome: "succeeded" };
|
|
3504
|
+
}
|
|
3505
|
+
return null;
|
|
3506
|
+
}
|
|
3507
|
+
return null;
|
|
3508
|
+
}
|
|
3509
|
+
function findSubagentAuthOutcome(messages, sinceMs) {
|
|
3510
|
+
return findLatestSubagentAuthOutcome(messages, sinceMs);
|
|
3511
|
+
}
|
|
2619
3512
|
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2620
3513
|
if (classified != null) return classified;
|
|
2621
3514
|
if (hasConfiguredProvider !== false) return null;
|
|
@@ -2663,6 +3556,94 @@ async function hasAnyConfiguredProvider(port) {
|
|
|
2663
3556
|
return null;
|
|
2664
3557
|
}
|
|
2665
3558
|
}
|
|
3559
|
+
function sessionErrorReason(error2) {
|
|
3560
|
+
const record = typeof error2 === "object" && error2 !== null ? error2 : null;
|
|
3561
|
+
const data = record?.data;
|
|
3562
|
+
const dataRecord = typeof data === "object" && data !== null ? data : null;
|
|
3563
|
+
const rawReason = typeof dataRecord?.message === "string" && dataRecord.message || typeof record?.message === "string" && record.message || typeof error2 === "string" && error2 || typeof record?.name === "string" && record.name || "OpenCode reported a session error with no details";
|
|
3564
|
+
const reason = rawReason.replace(/\s+/g, " ").trim().slice(0, 500);
|
|
3565
|
+
return reason || "OpenCode reported a session error with no details";
|
|
3566
|
+
}
|
|
3567
|
+
function parseSessionErrorFrame(data) {
|
|
3568
|
+
let parsed;
|
|
3569
|
+
try {
|
|
3570
|
+
parsed = JSON.parse(data);
|
|
3571
|
+
} catch (error2) {
|
|
3572
|
+
void error2;
|
|
3573
|
+
return null;
|
|
3574
|
+
}
|
|
3575
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
3576
|
+
const parsedRecord = parsed;
|
|
3577
|
+
const payload = parsedRecord.payload;
|
|
3578
|
+
const event = payload !== null && typeof payload === "object" && !Array.isArray(payload) ? payload : parsedRecord;
|
|
3579
|
+
if (event.type !== "session.error") return null;
|
|
3580
|
+
const properties = event.properties;
|
|
3581
|
+
if (properties === null || typeof properties !== "object" || Array.isArray(properties)) {
|
|
3582
|
+
return null;
|
|
3583
|
+
}
|
|
3584
|
+
const propertiesRecord = properties;
|
|
3585
|
+
const sessionId = propertiesRecord.sessionID;
|
|
3586
|
+
if (typeof sessionId !== "string" || sessionId.length === 0) return null;
|
|
3587
|
+
return {
|
|
3588
|
+
sessionId,
|
|
3589
|
+
reason: sessionErrorReason(propertiesRecord.error)
|
|
3590
|
+
};
|
|
3591
|
+
}
|
|
3592
|
+
async function readSessionErrorStream(port, options) {
|
|
3593
|
+
let reader = null;
|
|
3594
|
+
try {
|
|
3595
|
+
const response = await fetch(`${opencodeBase(port)}/event`, {
|
|
3596
|
+
headers: { accept: "text/event-stream" },
|
|
3597
|
+
signal: options.signal
|
|
3598
|
+
});
|
|
3599
|
+
if (!response.ok || !response.body) {
|
|
3600
|
+
return { reason: "unavailable", detail: `HTTP ${response.status}` };
|
|
3601
|
+
}
|
|
3602
|
+
reader = response.body.getReader();
|
|
3603
|
+
const decoder = new TextDecoder();
|
|
3604
|
+
let buffer = "";
|
|
3605
|
+
const processLine = (line) => {
|
|
3606
|
+
const trimmed = line.trimEnd();
|
|
3607
|
+
if (!trimmed.startsWith("data:")) return;
|
|
3608
|
+
const event = parseSessionErrorFrame(trimmed.slice("data:".length).replace(/^ /, ""));
|
|
3609
|
+
if (event) options.onSessionError(event);
|
|
3610
|
+
};
|
|
3611
|
+
while (true) {
|
|
3612
|
+
const { done, value } = await reader.read();
|
|
3613
|
+
if (done) return { reason: "ended" };
|
|
3614
|
+
buffer += decoder.decode(value, { stream: true });
|
|
3615
|
+
const lines = buffer.split("\n");
|
|
3616
|
+
buffer = lines.pop() ?? "";
|
|
3617
|
+
for (const line of lines) processLine(line);
|
|
3618
|
+
}
|
|
3619
|
+
} catch (err) {
|
|
3620
|
+
if (options.signal.aborted) return { reason: "aborted" };
|
|
3621
|
+
return {
|
|
3622
|
+
reason: "unavailable",
|
|
3623
|
+
detail: err instanceof Error ? err.message : String(err)
|
|
3624
|
+
};
|
|
3625
|
+
} finally {
|
|
3626
|
+
if (reader) void reader.cancel().catch(() => void 0);
|
|
3627
|
+
}
|
|
3628
|
+
}
|
|
3629
|
+
async function reloadProviderCache(port) {
|
|
3630
|
+
try {
|
|
3631
|
+
const res = await timedFetch(`${opencodeBase(port)}/config`, {
|
|
3632
|
+
method: "PATCH",
|
|
3633
|
+
headers: { "Content-Type": "application/json" },
|
|
3634
|
+
body: JSON.stringify({})
|
|
3635
|
+
});
|
|
3636
|
+
if (!res.ok) {
|
|
3637
|
+
console.error(
|
|
3638
|
+
`[reloadProviderCache] PATCH /config returned HTTP ${res.status} (port ${port})`
|
|
3639
|
+
);
|
|
3640
|
+
}
|
|
3641
|
+
} catch (err) {
|
|
3642
|
+
console.error(
|
|
3643
|
+
`[reloadProviderCache] PATCH /config failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
3644
|
+
);
|
|
3645
|
+
}
|
|
3646
|
+
}
|
|
2666
3647
|
|
|
2667
3648
|
// src/lib/opencode/session-cleanup.ts
|
|
2668
3649
|
var DURATION_UNIT_MS = {
|
|
@@ -2769,13 +3750,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2769
3750
|
}
|
|
2770
3751
|
|
|
2771
3752
|
// src/lib/opencode/session-db-size.ts
|
|
2772
|
-
import { statSync as
|
|
2773
|
-
import { join as
|
|
3753
|
+
import { statSync as statSync3 } from "node:fs";
|
|
3754
|
+
import { join as join4 } from "node:path";
|
|
2774
3755
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2775
3756
|
function statSessionDbBytes(homeDir) {
|
|
2776
|
-
const dbPath =
|
|
3757
|
+
const dbPath = join4(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2777
3758
|
try {
|
|
2778
|
-
return
|
|
3759
|
+
return statSync3(dbPath).size;
|
|
2779
3760
|
} catch (err) {
|
|
2780
3761
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2781
3762
|
if (!isMissingFile) {
|
|
@@ -2800,12 +3781,99 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
2800
3781
|
return null;
|
|
2801
3782
|
}
|
|
2802
3783
|
|
|
3784
|
+
// src/lib/opencode/log-tail.ts
|
|
3785
|
+
import { statSync as statSync4 } from "node:fs";
|
|
3786
|
+
import { homedir as homedir3 } from "node:os";
|
|
3787
|
+
import { join as join5 } from "node:path";
|
|
3788
|
+
import { open as open2, stat } from "node:fs/promises";
|
|
3789
|
+
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
3790
|
+
function resolveOpenCodeLogPath(homeDir = homedir3(), env = process.env) {
|
|
3791
|
+
const dataDir = env.XDG_DATA_HOME || join5(homeDir, ".local", "share");
|
|
3792
|
+
return join5(dataDir, "opencode", "log", "opencode.log");
|
|
3793
|
+
}
|
|
3794
|
+
function isEnoent(error2) {
|
|
3795
|
+
return error2?.code === "ENOENT";
|
|
3796
|
+
}
|
|
3797
|
+
function reportFailure(operation, logPath, error2) {
|
|
3798
|
+
console.error(
|
|
3799
|
+
`[opencode-log-tail] ${operation} failed for ${logPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3800
|
+
);
|
|
3801
|
+
}
|
|
3802
|
+
function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
3803
|
+
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
3804
|
+
let offset = 0;
|
|
3805
|
+
let inode = null;
|
|
3806
|
+
let baselineReady = true;
|
|
3807
|
+
try {
|
|
3808
|
+
const initial = statSync4(logPath);
|
|
3809
|
+
offset = initial.size;
|
|
3810
|
+
inode = initial.ino;
|
|
3811
|
+
} catch (error2) {
|
|
3812
|
+
if (!isEnoent(error2)) {
|
|
3813
|
+
reportFailure("initial stat", logPath, error2);
|
|
3814
|
+
baselineReady = false;
|
|
3815
|
+
}
|
|
3816
|
+
}
|
|
3817
|
+
let polling = false;
|
|
3818
|
+
let stopped = false;
|
|
3819
|
+
const poll = async () => {
|
|
3820
|
+
if (polling || stopped) return;
|
|
3821
|
+
polling = true;
|
|
3822
|
+
try {
|
|
3823
|
+
let current;
|
|
3824
|
+
try {
|
|
3825
|
+
current = await stat(logPath);
|
|
3826
|
+
} catch (error2) {
|
|
3827
|
+
if (!isEnoent(error2)) reportFailure("stat", logPath, error2);
|
|
3828
|
+
return;
|
|
3829
|
+
}
|
|
3830
|
+
if (!baselineReady) {
|
|
3831
|
+
offset = current.size;
|
|
3832
|
+
inode = current.ino;
|
|
3833
|
+
baselineReady = true;
|
|
3834
|
+
return;
|
|
3835
|
+
}
|
|
3836
|
+
if (inode !== null && current.ino !== inode || current.size < offset) {
|
|
3837
|
+
offset = 0;
|
|
3838
|
+
}
|
|
3839
|
+
inode = current.ino;
|
|
3840
|
+
if (current.size === offset) return;
|
|
3841
|
+
const length = current.size - offset;
|
|
3842
|
+
const fh = await open2(logPath, "r");
|
|
3843
|
+
try {
|
|
3844
|
+
const buf = Buffer.alloc(length);
|
|
3845
|
+
const { bytesRead } = await fh.read(buf, 0, length, offset);
|
|
3846
|
+
offset += bytesRead;
|
|
3847
|
+
if (bytesRead > 0) onChunk(buf.subarray(0, bytesRead));
|
|
3848
|
+
} finally {
|
|
3849
|
+
await fh.close();
|
|
3850
|
+
}
|
|
3851
|
+
} catch (error2) {
|
|
3852
|
+
if (!isEnoent(error2)) reportFailure("poll", logPath, error2);
|
|
3853
|
+
} finally {
|
|
3854
|
+
polling = false;
|
|
3855
|
+
}
|
|
3856
|
+
};
|
|
3857
|
+
const interval = setInterval(() => void poll(), pollIntervalMs);
|
|
3858
|
+
void poll();
|
|
3859
|
+
return {
|
|
3860
|
+
stop: () => {
|
|
3861
|
+
stopped = true;
|
|
3862
|
+
clearInterval(interval);
|
|
3863
|
+
}
|
|
3864
|
+
};
|
|
3865
|
+
}
|
|
3866
|
+
|
|
2803
3867
|
// src/lib/opencode/session-db-reclaim.ts
|
|
2804
|
-
import { statSync as
|
|
2805
|
-
import { dirname as
|
|
3868
|
+
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
3869
|
+
import { dirname as dirname4 } from "node:path";
|
|
3870
|
+
function errorMessage(error2) {
|
|
3871
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
3872
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
3873
|
+
}
|
|
2806
3874
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2807
3875
|
try {
|
|
2808
|
-
const fsStats = statfsSync(
|
|
3876
|
+
const fsStats = statfsSync(dirname4(dbPath));
|
|
2809
3877
|
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2810
3878
|
if (availableBytes < requiredBytes) {
|
|
2811
3879
|
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
@@ -2828,17 +3896,17 @@ async function probeReclaimAvailability(input) {
|
|
|
2828
3896
|
const { dbPath, requiredBytes } = input;
|
|
2829
3897
|
let sqlite;
|
|
2830
3898
|
try {
|
|
2831
|
-
sqlite = await import("sqlite");
|
|
3899
|
+
sqlite = await import("node:sqlite");
|
|
2832
3900
|
} catch (err) {
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
return "sqlite-unavailable";
|
|
3901
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
3902
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
3903
|
+
return { reason: "sqlite-unavailable", detail };
|
|
2837
3904
|
}
|
|
2838
3905
|
let autoVacuum = null;
|
|
2839
3906
|
try {
|
|
2840
3907
|
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
2841
3908
|
try {
|
|
3909
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
2842
3910
|
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2843
3911
|
} finally {
|
|
2844
3912
|
db.close();
|
|
@@ -2849,23 +3917,25 @@ async function probeReclaimAvailability(input) {
|
|
|
2849
3917
|
);
|
|
2850
3918
|
}
|
|
2851
3919
|
if (autoVacuum !== 0) return null;
|
|
2852
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? "insufficient-disk-space" : null;
|
|
3920
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
2853
3921
|
}
|
|
2854
3922
|
async function reclaimSessionDbSpace(input) {
|
|
2855
3923
|
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
2856
3924
|
let sqlite;
|
|
2857
3925
|
try {
|
|
2858
|
-
sqlite = await import("sqlite");
|
|
3926
|
+
sqlite = await import("node:sqlite");
|
|
2859
3927
|
} catch (err) {
|
|
3928
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
2860
3929
|
console.warn(
|
|
2861
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${
|
|
3930
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
2862
3931
|
);
|
|
2863
|
-
return { ok: false, skipped: "sqlite-unavailable" };
|
|
3932
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
2864
3933
|
}
|
|
2865
3934
|
const { DatabaseSync } = sqlite;
|
|
2866
3935
|
let db;
|
|
2867
3936
|
try {
|
|
2868
3937
|
db = new DatabaseSync(dbPath);
|
|
3938
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
2869
3939
|
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2870
3940
|
if (autoVacuum === 0) {
|
|
2871
3941
|
if (!allowFullVacuum) {
|
|
@@ -2874,7 +3944,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2874
3944
|
);
|
|
2875
3945
|
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2876
3946
|
}
|
|
2877
|
-
const fileBytesForGuard =
|
|
3947
|
+
const fileBytesForGuard = statSync5(dbPath).size;
|
|
2878
3948
|
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2879
3949
|
if (skipReason !== null) {
|
|
2880
3950
|
console.warn(
|
|
@@ -2902,10 +3972,12 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2902
3972
|
);
|
|
2903
3973
|
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
2904
3974
|
} catch (err) {
|
|
2905
|
-
console.error(
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
3975
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
3976
|
+
return {
|
|
3977
|
+
ok: false,
|
|
3978
|
+
skipped: "reclaim-error",
|
|
3979
|
+
detail: errorMessage(err)
|
|
3980
|
+
};
|
|
2909
3981
|
} finally {
|
|
2910
3982
|
db?.close();
|
|
2911
3983
|
}
|
|
@@ -2946,7 +4018,6 @@ var StreamForwarder = class {
|
|
|
2946
4018
|
handleFrame(frame) {
|
|
2947
4019
|
switch (frame.type) {
|
|
2948
4020
|
case "open":
|
|
2949
|
-
this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
|
|
2950
4021
|
void this.handleOpen(frame);
|
|
2951
4022
|
break;
|
|
2952
4023
|
case "req_data":
|
|
@@ -2982,12 +4053,21 @@ var StreamForwarder = class {
|
|
|
2982
4053
|
const { sid, method, path, headers, has_body } = frame;
|
|
2983
4054
|
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
2984
4055
|
const startedAt = Date.now();
|
|
4056
|
+
if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
|
|
4057
|
+
this.callbacks.onOpen?.(sid, method, path);
|
|
4058
|
+
}
|
|
2985
4059
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
2986
4060
|
this.callbacks.onDrainPing?.();
|
|
2987
4061
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
2988
4062
|
this.send({ type: "res_end", sid });
|
|
2989
4063
|
return;
|
|
2990
4064
|
}
|
|
4065
|
+
if (path === TUNNEL_USAGE_REARM_PING_PATH) {
|
|
4066
|
+
this.callbacks.onUsageRearmPing?.();
|
|
4067
|
+
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
4068
|
+
this.send({ type: "res_end", sid });
|
|
4069
|
+
return;
|
|
4070
|
+
}
|
|
2991
4071
|
if (process.env.DEBUG) {
|
|
2992
4072
|
log("debug", "agent_request", {
|
|
2993
4073
|
correlation_id: correlationId,
|
|
@@ -3002,12 +4082,12 @@ var StreamForwarder = class {
|
|
|
3002
4082
|
let endBody;
|
|
3003
4083
|
if (has_body) {
|
|
3004
4084
|
const chunks = [];
|
|
3005
|
-
bodyPromise = new Promise((
|
|
4085
|
+
bodyPromise = new Promise((resolve4) => {
|
|
3006
4086
|
pushBody = (buf) => {
|
|
3007
4087
|
chunks.push(buf);
|
|
3008
4088
|
};
|
|
3009
4089
|
endBody = () => {
|
|
3010
|
-
|
|
4090
|
+
resolve4(Buffer.concat(chunks));
|
|
3011
4091
|
};
|
|
3012
4092
|
});
|
|
3013
4093
|
}
|
|
@@ -3132,11 +4212,12 @@ function connectTunnel(options) {
|
|
|
3132
4212
|
onResponse,
|
|
3133
4213
|
onInfo,
|
|
3134
4214
|
onWarning,
|
|
3135
|
-
onDrainPing
|
|
4215
|
+
onDrainPing,
|
|
4216
|
+
onUsageRearmPing
|
|
3136
4217
|
} = options;
|
|
3137
4218
|
const tunnelUrl = getTunnelUrlConfig();
|
|
3138
4219
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
3139
|
-
return new Promise((
|
|
4220
|
+
return new Promise((resolve4, reject) => {
|
|
3140
4221
|
const ws = new WebSocket2(url, {
|
|
3141
4222
|
headers: {
|
|
3142
4223
|
Authorization: authHeader
|
|
@@ -3144,7 +4225,8 @@ function connectTunnel(options) {
|
|
|
3144
4225
|
});
|
|
3145
4226
|
const forwarder = new StreamForwarder(ws, port, {
|
|
3146
4227
|
onHead: () => onResponse?.(),
|
|
3147
|
-
onDrainPing: () => onDrainPing?.()
|
|
4228
|
+
onDrainPing: () => onDrainPing?.(),
|
|
4229
|
+
onUsageRearmPing: () => onUsageRearmPing?.()
|
|
3148
4230
|
});
|
|
3149
4231
|
const connectionTimeout = setTimeout(() => {
|
|
3150
4232
|
ws.close();
|
|
@@ -3187,8 +4269,8 @@ function connectTunnel(options) {
|
|
|
3187
4269
|
try {
|
|
3188
4270
|
message = JSON.parse(data.toString());
|
|
3189
4271
|
} catch (error2) {
|
|
3190
|
-
const
|
|
3191
|
-
onError?.(`Failed to handle message: ${
|
|
4272
|
+
const errorMessage3 = error2 instanceof Error ? error2.message : "Unknown error";
|
|
4273
|
+
onError?.(`Failed to handle message: ${errorMessage3}`);
|
|
3192
4274
|
return;
|
|
3193
4275
|
}
|
|
3194
4276
|
if (isStreamFrame(message)) {
|
|
@@ -3200,7 +4282,7 @@ function connectTunnel(options) {
|
|
|
3200
4282
|
clearTimeout(connectionTimeout);
|
|
3201
4283
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
3202
4284
|
onConnected?.(connectedAgentId);
|
|
3203
|
-
|
|
4285
|
+
resolve4({
|
|
3204
4286
|
ws,
|
|
3205
4287
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
3206
4288
|
});
|
|
@@ -3305,6 +4387,7 @@ var RunnerConnection = class {
|
|
|
3305
4387
|
onError: (error2) => events.onError?.(error2),
|
|
3306
4388
|
onResponse: () => events.onResponse?.(),
|
|
3307
4389
|
onDrainPing: () => events.onDrainPing?.(),
|
|
4390
|
+
onUsageRearmPing: () => events.onUsageRearmPing?.(),
|
|
3308
4391
|
onInfo: (message) => events.onInfo?.(message),
|
|
3309
4392
|
onWarning: (message) => events.onWarning?.(message)
|
|
3310
4393
|
});
|
|
@@ -3331,10 +4414,10 @@ var RunnerConnection = class {
|
|
|
3331
4414
|
};
|
|
3332
4415
|
|
|
3333
4416
|
// src/lib/tunnel/ready-marker.ts
|
|
3334
|
-
import { writeFileSync } from "fs";
|
|
4417
|
+
import { writeFileSync as writeFileSync3 } from "node:fs";
|
|
3335
4418
|
function writeTunnelReadyMarker(path, agentId) {
|
|
3336
4419
|
try {
|
|
3337
|
-
|
|
4420
|
+
writeFileSync3(path, `${agentId}
|
|
3338
4421
|
`);
|
|
3339
4422
|
return { ok: true };
|
|
3340
4423
|
} catch (error2) {
|
|
@@ -3343,9 +4426,9 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
3343
4426
|
}
|
|
3344
4427
|
|
|
3345
4428
|
// src/lib/replication.ts
|
|
3346
|
-
import { spawn as
|
|
4429
|
+
import { spawn as spawn4 } from "node:child_process";
|
|
3347
4430
|
function startSessionDbReplication(configPath) {
|
|
3348
|
-
return
|
|
4431
|
+
return spawn4("litestream", ["replicate", "-config", configPath], {
|
|
3349
4432
|
stdio: "inherit"
|
|
3350
4433
|
});
|
|
3351
4434
|
}
|
|
@@ -3358,10 +4441,36 @@ async function stopSessionDbReplication(child, timeoutMs) {
|
|
|
3358
4441
|
);
|
|
3359
4442
|
}
|
|
3360
4443
|
|
|
4444
|
+
// src/lib/process-liveness.ts
|
|
4445
|
+
import { readFileSync as readFileSync4 } from "node:fs";
|
|
4446
|
+
function isProcessAlive(pid) {
|
|
4447
|
+
try {
|
|
4448
|
+
process.kill(pid, 0);
|
|
4449
|
+
} catch (error2) {
|
|
4450
|
+
const code = error2.code;
|
|
4451
|
+
if (code === "ESRCH") return false;
|
|
4452
|
+
if (code === "EPERM") return true;
|
|
4453
|
+
console.error(
|
|
4454
|
+
`[process-liveness] could not probe process ${pid}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4455
|
+
);
|
|
4456
|
+
return false;
|
|
4457
|
+
}
|
|
4458
|
+
if (process.platform !== "linux") return true;
|
|
4459
|
+
try {
|
|
4460
|
+
const status2 = readFileSync4(`/proc/${pid}/status`, "utf8");
|
|
4461
|
+
return !/^State:\s+Z(?:\s|$)/m.test(status2);
|
|
4462
|
+
} catch (error2) {
|
|
4463
|
+
console.error(
|
|
4464
|
+
`[process-liveness] could not inspect /proc/${pid}/status: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4465
|
+
);
|
|
4466
|
+
return true;
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
|
|
3361
4470
|
// src/lib/openai-usage.ts
|
|
3362
|
-
import { readFileSync as
|
|
3363
|
-
import { homedir as
|
|
3364
|
-
import { join as
|
|
4471
|
+
import { readFileSync as readFileSync5 } from "node:fs";
|
|
4472
|
+
import { homedir as homedir4 } from "node:os";
|
|
4473
|
+
import { join as join6 } from "node:path";
|
|
3365
4474
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3366
4475
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3367
4476
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3375,7 +4484,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3375
4484
|
}
|
|
3376
4485
|
function readOpenCodeChatGptCredentials() {
|
|
3377
4486
|
try {
|
|
3378
|
-
const raw =
|
|
4487
|
+
const raw = readFileSync5(join6(homedir4(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3379
4488
|
let parsed;
|
|
3380
4489
|
try {
|
|
3381
4490
|
parsed = JSON.parse(raw);
|
|
@@ -3397,6 +4506,23 @@ function readOpenCodeChatGptCredentials() {
|
|
|
3397
4506
|
return null;
|
|
3398
4507
|
}
|
|
3399
4508
|
}
|
|
4509
|
+
function parseChatGptIdentity(accessToken) {
|
|
4510
|
+
const segments = accessToken.split(".");
|
|
4511
|
+
if (segments.length !== 3) return null;
|
|
4512
|
+
let payload;
|
|
4513
|
+
try {
|
|
4514
|
+
const parsed = JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8"));
|
|
4515
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
4516
|
+
payload = parsed;
|
|
4517
|
+
} catch {
|
|
4518
|
+
return null;
|
|
4519
|
+
}
|
|
4520
|
+
const profile = payload["https://api.openai.com/profile"];
|
|
4521
|
+
const auth = payload["https://api.openai.com/auth"];
|
|
4522
|
+
const ownerEmail = typeof profile?.email === "string" ? profile.email.trim() || null : null;
|
|
4523
|
+
const planType = typeof auth?.chatgpt_plan_type === "string" ? auth.chatgpt_plan_type.trim() || null : null;
|
|
4524
|
+
return ownerEmail === null && planType === null ? null : { ownerEmail, planType, organizationName: null };
|
|
4525
|
+
}
|
|
3400
4526
|
function toWindow2(headers, name) {
|
|
3401
4527
|
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
3402
4528
|
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
@@ -3472,6 +4598,7 @@ async function getOpenAiUsage(port) {
|
|
|
3472
4598
|
"credentials_expired"
|
|
3473
4599
|
);
|
|
3474
4600
|
}
|
|
4601
|
+
const subscription = parseChatGptIdentity(credentials2.accessToken);
|
|
3475
4602
|
const models = await resolveProbeModels(port);
|
|
3476
4603
|
if (models.length === 0) {
|
|
3477
4604
|
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
@@ -3504,7 +4631,7 @@ async function getOpenAiUsage(port) {
|
|
|
3504
4631
|
"no_usable_window"
|
|
3505
4632
|
);
|
|
3506
4633
|
}
|
|
3507
|
-
return usage;
|
|
4634
|
+
return { ...usage, subscription };
|
|
3508
4635
|
}
|
|
3509
4636
|
if (res.status === 401) {
|
|
3510
4637
|
throw new OpenAiUsageError(
|
|
@@ -3619,8 +4746,8 @@ function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
|
3619
4746
|
}
|
|
3620
4747
|
|
|
3621
4748
|
// src/lib/resource-usage.ts
|
|
3622
|
-
import { cpus, totalmem, freemem } from "os";
|
|
3623
|
-
import { statfsSync as statfsSync2 } from "fs";
|
|
4749
|
+
import { cpus, totalmem, freemem } from "node:os";
|
|
4750
|
+
import { statfsSync as statfsSync2 } from "node:fs";
|
|
3624
4751
|
|
|
3625
4752
|
// src/lib/ecs-task-metadata.ts
|
|
3626
4753
|
var ECS_METADATA_TIMEOUT_MS = 2e3;
|
|
@@ -3705,58 +4832,97 @@ function readDisk(homeDir) {
|
|
|
3705
4832
|
};
|
|
3706
4833
|
}
|
|
3707
4834
|
}
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
4835
|
+
var CPU_PEAK_WINDOW_MS = 6e4;
|
|
4836
|
+
var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
|
|
4837
|
+
var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4838
|
+
function createCpuPeakSampler() {
|
|
4839
|
+
const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4840
|
+
sampleHistory[0] = readCpuSample();
|
|
4841
|
+
let nextSampleIndex = 1;
|
|
4842
|
+
let sampleCount = 1;
|
|
4843
|
+
let peak = null;
|
|
4844
|
+
const timer = setInterval(() => {
|
|
3711
4845
|
const current = readCpuSample();
|
|
3712
|
-
const
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
const warnings = [];
|
|
3719
|
-
if (disk.warning) warnings.push(disk.warning);
|
|
3720
|
-
if (ecsWarning) warnings.push(ecsWarning);
|
|
3721
|
-
let cpuPercent = hostCpuPercent;
|
|
3722
|
-
let cpuCount = hostCpuCount;
|
|
3723
|
-
let memoryTotalBytes = totalmem();
|
|
3724
|
-
let memoryAvailableBytes = freemem();
|
|
3725
|
-
if (limits !== null) {
|
|
3726
|
-
cpuCount = limits.cpuCount;
|
|
3727
|
-
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3728
|
-
memoryAvailableBytes = clamp(
|
|
3729
|
-
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3730
|
-
0,
|
|
3731
|
-
limits.memoryTotalBytes
|
|
3732
|
-
);
|
|
3733
|
-
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4846
|
+
const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
|
|
4847
|
+
if (sampleFromWindowAgo !== void 0) {
|
|
4848
|
+
const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
|
|
4849
|
+
if (percentage !== null) {
|
|
4850
|
+
peak = peak === null ? percentage : Math.max(peak, percentage);
|
|
4851
|
+
}
|
|
3734
4852
|
}
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
4853
|
+
sampleHistory[nextSampleIndex] = current;
|
|
4854
|
+
nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4855
|
+
sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4856
|
+
}, CPU_PEAK_SAMPLE_INTERVAL_MS);
|
|
4857
|
+
return {
|
|
4858
|
+
takeAndReset: () => {
|
|
4859
|
+
const currentPeak = peak;
|
|
4860
|
+
peak = null;
|
|
4861
|
+
return currentPeak;
|
|
4862
|
+
},
|
|
4863
|
+
stop: () => clearInterval(timer)
|
|
4864
|
+
};
|
|
4865
|
+
}
|
|
4866
|
+
function createResourceUsageCollector(homeDir) {
|
|
4867
|
+
let previous = readCpuSample();
|
|
4868
|
+
const cpuPeakSampler = createCpuPeakSampler();
|
|
4869
|
+
return {
|
|
4870
|
+
collect: async () => {
|
|
4871
|
+
const current = readCpuSample();
|
|
4872
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
4873
|
+
const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
|
|
4874
|
+
const hostCpuCount = cpus().length;
|
|
4875
|
+
previous = current;
|
|
4876
|
+
const disk = readDisk(homeDir);
|
|
4877
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
4878
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
4879
|
+
const warnings = [];
|
|
4880
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
4881
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
4882
|
+
let cpuPercent = hostCpuPercent;
|
|
4883
|
+
let cpuPeakPercent = hostCpuPeakPercent;
|
|
4884
|
+
let cpuCount = hostCpuCount;
|
|
4885
|
+
let memoryTotalBytes = totalmem();
|
|
4886
|
+
let memoryAvailableBytes = freemem();
|
|
4887
|
+
if (limits !== null) {
|
|
4888
|
+
cpuCount = limits.cpuCount;
|
|
4889
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4890
|
+
memoryAvailableBytes = clamp(
|
|
4891
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4892
|
+
0,
|
|
4893
|
+
limits.memoryTotalBytes
|
|
4894
|
+
);
|
|
4895
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4896
|
+
cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4897
|
+
}
|
|
4898
|
+
return {
|
|
4899
|
+
usage: {
|
|
4900
|
+
cpuPercent,
|
|
4901
|
+
cpuPeakPercent,
|
|
4902
|
+
cpuCount,
|
|
4903
|
+
memoryTotalBytes,
|
|
4904
|
+
memoryAvailableBytes,
|
|
4905
|
+
diskTotalBytes: disk.totalBytes,
|
|
4906
|
+
diskFreeBytes: disk.freeBytes,
|
|
4907
|
+
opencodeDbBytes
|
|
4908
|
+
},
|
|
4909
|
+
warnings
|
|
4910
|
+
};
|
|
4911
|
+
},
|
|
4912
|
+
stop: cpuPeakSampler.stop
|
|
3747
4913
|
};
|
|
3748
4914
|
}
|
|
3749
4915
|
|
|
3750
4916
|
// src/lib/channels/driver.ts
|
|
3751
|
-
import { homedir as
|
|
4917
|
+
import { homedir as homedir5 } from "node:os";
|
|
3752
4918
|
|
|
3753
4919
|
// src/lib/runner-file-sync.ts
|
|
3754
|
-
import { join as
|
|
4920
|
+
import { join as join8 } from "node:path";
|
|
3755
4921
|
|
|
3756
4922
|
// src/lib/file-push.ts
|
|
3757
|
-
import { randomUUID } from "crypto";
|
|
3758
|
-
import { chmod, mkdir, open as
|
|
3759
|
-
import { basename, dirname as
|
|
4923
|
+
import { randomUUID } from "node:crypto";
|
|
4924
|
+
import { chmod, mkdir, open as open3, realpath, rename, unlink } from "node:fs/promises";
|
|
4925
|
+
import { basename, dirname as dirname5, isAbsolute, join as join7, relative, resolve as resolve2, sep } from "node:path";
|
|
3760
4926
|
var FILE_MODE = 384;
|
|
3761
4927
|
var DIRECTORY_MODE = 448;
|
|
3762
4928
|
async function writePushedFile(request) {
|
|
@@ -3787,9 +4953,9 @@ async function writePushedFile(request) {
|
|
|
3787
4953
|
}
|
|
3788
4954
|
try {
|
|
3789
4955
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3790
|
-
|
|
4956
|
+
dirname5(candidate)
|
|
3791
4957
|
);
|
|
3792
|
-
const realTarget =
|
|
4958
|
+
const realTarget = join7(existingAncestor, ...missingSegments, basename(candidate));
|
|
3793
4959
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3794
4960
|
if (allowedDirectory === null) {
|
|
3795
4961
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3799,8 +4965,8 @@ async function writePushedFile(request) {
|
|
|
3799
4965
|
}
|
|
3800
4966
|
if (missingSegments.length > 0) {
|
|
3801
4967
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
3802
|
-
const realParent = await realpath(
|
|
3803
|
-
if (realParent !==
|
|
4968
|
+
const realParent = await realpath(dirname5(realTarget));
|
|
4969
|
+
if (realParent !== dirname5(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
3804
4970
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
3805
4971
|
path: realTarget,
|
|
3806
4972
|
bytes,
|
|
@@ -3825,7 +4991,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3825
4991
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3826
4992
|
return null;
|
|
3827
4993
|
}
|
|
3828
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4994
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3829
4995
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3830
4996
|
return null;
|
|
3831
4997
|
}
|
|
@@ -3843,7 +5009,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
3843
5009
|
try {
|
|
3844
5010
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
3845
5011
|
} catch (err) {
|
|
3846
|
-
const parent =
|
|
5012
|
+
const parent = dirname5(current);
|
|
3847
5013
|
if (err.code !== "ENOENT" || parent === current) {
|
|
3848
5014
|
throw err;
|
|
3849
5015
|
}
|
|
@@ -3898,16 +5064,16 @@ function contains(realDirectory, realTarget) {
|
|
|
3898
5064
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3899
5065
|
let current = existingAncestor;
|
|
3900
5066
|
for (const segment of missingSegments) {
|
|
3901
|
-
current =
|
|
5067
|
+
current = join7(current, segment);
|
|
3902
5068
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3903
5069
|
await chmod(current, DIRECTORY_MODE);
|
|
3904
5070
|
}
|
|
3905
5071
|
}
|
|
3906
5072
|
async function writeAtomically(realTarget, content) {
|
|
3907
|
-
const temporaryPath =
|
|
5073
|
+
const temporaryPath = join7(dirname5(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3908
5074
|
let handle;
|
|
3909
5075
|
try {
|
|
3910
|
-
handle = await
|
|
5076
|
+
handle = await open3(temporaryPath, "wx", FILE_MODE);
|
|
3911
5077
|
await handle.writeFile(content);
|
|
3912
5078
|
await handle.chmod(FILE_MODE);
|
|
3913
5079
|
await handle.close();
|
|
@@ -4034,12 +5200,12 @@ var NOT_APPLIED = {
|
|
|
4034
5200
|
opencodeAuthApplied: false
|
|
4035
5201
|
};
|
|
4036
5202
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
4037
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4038
|
-
return expanded ===
|
|
5203
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join8(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
5204
|
+
return expanded === join8(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
4039
5205
|
}
|
|
4040
5206
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
4041
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4042
|
-
return expanded ===
|
|
5207
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join8(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
5208
|
+
return expanded === join8(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
4043
5209
|
}
|
|
4044
5210
|
async function applyOne(options, file) {
|
|
4045
5211
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -4199,6 +5365,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
4199
5365
|
baseDelayMs: 500,
|
|
4200
5366
|
maxDelayMs: 3e4
|
|
4201
5367
|
};
|
|
5368
|
+
var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
5369
|
+
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
5370
|
+
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
5371
|
+
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
4202
5372
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
4203
5373
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
4204
5374
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
@@ -4339,6 +5509,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4339
5509
|
* message; it is removed once its in-flight set empties.
|
|
4340
5510
|
*/
|
|
4341
5511
|
watchers = /* @__PURE__ */ new Map();
|
|
5512
|
+
sessionErrorStream = null;
|
|
5513
|
+
/**
|
|
5514
|
+
* Session-error failures currently being reported; entries are empty at rest
|
|
5515
|
+
* because each handoff deletes its id in `finally`.
|
|
5516
|
+
*/
|
|
5517
|
+
sessionErrorHandled = /* @__PURE__ */ new Set();
|
|
5518
|
+
/**
|
|
5519
|
+
* Session errors that arrived before their dispatch was registered. Bounded FIFO
|
|
5520
|
+
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
5521
|
+
*/
|
|
5522
|
+
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
4342
5523
|
/**
|
|
4343
5524
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
4344
5525
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -4522,6 +5703,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4522
5703
|
* no watcher) can resolve the title.
|
|
4523
5704
|
*/
|
|
4524
5705
|
sessionTitles = /* @__PURE__ */ new Map();
|
|
5706
|
+
/** One best-effort terminal subagent collection per Evident message id. */
|
|
5707
|
+
subagentInvocationCollections = /* @__PURE__ */ new Map();
|
|
5708
|
+
/**
|
|
5709
|
+
* Early snapshots are only liveness hints; they must not become the terminal
|
|
5710
|
+
* collection when the task parts or child transcript have advanced.
|
|
5711
|
+
*/
|
|
5712
|
+
subagentInvocationPrefetches = /* @__PURE__ */ new Map();
|
|
4525
5713
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
4526
5714
|
draining = false;
|
|
4527
5715
|
/**
|
|
@@ -4566,6 +5754,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4566
5754
|
* and stops opencode.
|
|
4567
5755
|
*/
|
|
4568
5756
|
stopped = false;
|
|
5757
|
+
recycleRequestedFlag = false;
|
|
4569
5758
|
constructor(config) {
|
|
4570
5759
|
this.agentId = config.agentId;
|
|
4571
5760
|
this.port = config.port;
|
|
@@ -4585,7 +5774,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4585
5774
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
4586
5775
|
this.now = config.now ?? (() => Date.now());
|
|
4587
5776
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
4588
|
-
this.homeDir = config.homeDir ??
|
|
5777
|
+
this.homeDir = config.homeDir ?? homedir5();
|
|
4589
5778
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4590
5779
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4591
5780
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
@@ -4671,6 +5860,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4671
5860
|
let dispatched = 0;
|
|
4672
5861
|
try {
|
|
4673
5862
|
const conversations = await this.getPendingConversations();
|
|
5863
|
+
if (this.recycleRequestedFlag) {
|
|
5864
|
+
this.stop();
|
|
5865
|
+
}
|
|
4674
5866
|
if (conversations.length > 0) {
|
|
4675
5867
|
const total = conversations.reduce((sum, c) => sum + (c.pending_message_count ?? 0), 0);
|
|
4676
5868
|
this.log({
|
|
@@ -4798,6 +5990,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4798
5990
|
*/
|
|
4799
5991
|
stop() {
|
|
4800
5992
|
this.stopped = true;
|
|
5993
|
+
this.sessionErrorStream?.abort.abort();
|
|
5994
|
+
this.sessionErrorStream = null;
|
|
5995
|
+
}
|
|
5996
|
+
/**
|
|
5997
|
+
* The server clears this request when a new MicroVM identity is recorded, so a
|
|
5998
|
+
* same-VM tunnel reconnect does not consume it. This is a plain read rather
|
|
5999
|
+
* than a consume; `run.ts` guards the action once-only.
|
|
6000
|
+
*/
|
|
6001
|
+
get recycleRequested() {
|
|
6002
|
+
return this.recycleRequestedFlag;
|
|
4801
6003
|
}
|
|
4802
6004
|
/**
|
|
4803
6005
|
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
@@ -4864,6 +6066,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4864
6066
|
*/
|
|
4865
6067
|
async processConversation(conv) {
|
|
4866
6068
|
const { sessionId, refusedSessionId, created: sessionCreated } = await this.ensureSession(conv);
|
|
6069
|
+
this.ensureSessionErrorStream();
|
|
4867
6070
|
const messages = await this.getPendingMessages(conv.id);
|
|
4868
6071
|
let dispatched = 0;
|
|
4869
6072
|
let skippedAlreadyDispatched = 0;
|
|
@@ -4936,7 +6139,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4936
6139
|
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
4937
6140
|
break;
|
|
4938
6141
|
}
|
|
4939
|
-
const
|
|
6142
|
+
const errorMessage3 = err instanceof Error ? err.message : String(err);
|
|
4940
6143
|
this.sessions.delete(conv.id);
|
|
4941
6144
|
this.supersede(conv.id, sessionId);
|
|
4942
6145
|
this.log({
|
|
@@ -4945,7 +6148,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4945
6148
|
conversation_id: conv.id,
|
|
4946
6149
|
message_id: message.id
|
|
4947
6150
|
});
|
|
4948
|
-
await this.markFailed(conv.id, message.id, null,
|
|
6151
|
+
await this.markFailed(conv.id, message.id, null, errorMessage3).catch((markErr) => {
|
|
4949
6152
|
this.log({
|
|
4950
6153
|
level: "warn",
|
|
4951
6154
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4956,7 +6159,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4956
6159
|
});
|
|
4957
6160
|
this.log({
|
|
4958
6161
|
level: "error",
|
|
4959
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
6162
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage3}`,
|
|
4960
6163
|
conversation_id: conv.id,
|
|
4961
6164
|
message_id: message.id
|
|
4962
6165
|
});
|
|
@@ -4977,14 +6180,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4977
6180
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4978
6181
|
this.sessions.delete(conv.id);
|
|
4979
6182
|
this.supersede(conv.id, sessionId);
|
|
4980
|
-
const
|
|
6183
|
+
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
4981
6184
|
this.log({
|
|
4982
6185
|
level: "error",
|
|
4983
|
-
message:
|
|
6186
|
+
message: errorMessage3,
|
|
4984
6187
|
conversation_id: conv.id,
|
|
4985
6188
|
message_id: message.id
|
|
4986
6189
|
});
|
|
4987
|
-
await this.markFailed(conv.id, message.id, null,
|
|
6190
|
+
await this.markFailed(conv.id, message.id, null, errorMessage3).catch((markErr) => {
|
|
4988
6191
|
this.log({
|
|
4989
6192
|
level: "warn",
|
|
4990
6193
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -5299,16 +6502,37 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5299
6502
|
if (state === "done") {
|
|
5300
6503
|
const title = await this.resolveSessionTitle(sessionId, conv.id);
|
|
5301
6504
|
const usage = messageUsage(messages, ocId ?? "");
|
|
6505
|
+
const usageAgentName = this.usageAgentName(messages, ocId ?? "");
|
|
6506
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
6507
|
+
messages,
|
|
6508
|
+
ocId ?? "",
|
|
6509
|
+
message.id
|
|
6510
|
+
);
|
|
5302
6511
|
this.log({
|
|
5303
6512
|
level: "info",
|
|
5304
6513
|
message: `Re-drive: message ${message.id.slice(0, 8)} completed while its row was wrongly reclaimed to pending \u2014 marking done instead of re-dispatching`,
|
|
5305
6514
|
conversation_id: conv.id,
|
|
5306
6515
|
message_id: message.id
|
|
5307
6516
|
});
|
|
5308
|
-
await this.markDone(
|
|
6517
|
+
await this.markDone(
|
|
6518
|
+
conv.id,
|
|
6519
|
+
message.id,
|
|
6520
|
+
sessionId,
|
|
6521
|
+
ocId,
|
|
6522
|
+
title,
|
|
6523
|
+
usage,
|
|
6524
|
+
usageAgentName,
|
|
6525
|
+
subagentInvocations
|
|
6526
|
+
);
|
|
5309
6527
|
} else {
|
|
5310
6528
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
5311
6529
|
const usage = messageUsage(messages, ocId ?? "");
|
|
6530
|
+
const usageAgentName = this.usageAgentName(messages, ocId ?? "");
|
|
6531
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
6532
|
+
messages,
|
|
6533
|
+
ocId ?? "",
|
|
6534
|
+
message.id
|
|
6535
|
+
);
|
|
5312
6536
|
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
5313
6537
|
this.log({
|
|
5314
6538
|
level: "error",
|
|
@@ -5316,7 +6540,19 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5316
6540
|
conversation_id: conv.id,
|
|
5317
6541
|
message_id: message.id
|
|
5318
6542
|
});
|
|
5319
|
-
await this.markFailed(
|
|
6543
|
+
await this.markFailed(
|
|
6544
|
+
conv.id,
|
|
6545
|
+
message.id,
|
|
6546
|
+
sessionId,
|
|
6547
|
+
error2,
|
|
6548
|
+
usage,
|
|
6549
|
+
failure,
|
|
6550
|
+
usageAgentName,
|
|
6551
|
+
subagentInvocations
|
|
6552
|
+
);
|
|
6553
|
+
}
|
|
6554
|
+
if (ocId !== null) {
|
|
6555
|
+
await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
|
|
5320
6556
|
}
|
|
5321
6557
|
} catch (err) {
|
|
5322
6558
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -5857,6 +7093,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5857
7093
|
ambiguousPinnedSinceMs: 0,
|
|
5858
7094
|
ambiguousResolved: false
|
|
5859
7095
|
});
|
|
7096
|
+
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
7097
|
+
if (!buffered) return;
|
|
7098
|
+
this.bufferedSessionErrors.delete(sessionId);
|
|
7099
|
+
if (this.now() - buffered.receivedAt < SESSION_ERROR_BUFFER_TTL_MS) {
|
|
7100
|
+
this.handleSessionError(buffered.event);
|
|
7101
|
+
}
|
|
7102
|
+
}
|
|
7103
|
+
bufferSessionError(event) {
|
|
7104
|
+
this.bufferedSessionErrors.delete(event.sessionId);
|
|
7105
|
+
this.bufferedSessionErrors.set(event.sessionId, {
|
|
7106
|
+
event,
|
|
7107
|
+
receivedAt: this.now()
|
|
7108
|
+
});
|
|
7109
|
+
while (this.bufferedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
7110
|
+
const oldest = this.bufferedSessionErrors.keys().next().value;
|
|
7111
|
+
if (typeof oldest !== "string") break;
|
|
7112
|
+
this.bufferedSessionErrors.delete(oldest);
|
|
7113
|
+
}
|
|
5860
7114
|
}
|
|
5861
7115
|
/**
|
|
5862
7116
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
@@ -6061,6 +7315,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6061
7315
|
ensureWatcherRunning(sessionId) {
|
|
6062
7316
|
const watcher = this.watchers.get(sessionId);
|
|
6063
7317
|
if (!watcher) return;
|
|
7318
|
+
this.ensureSessionErrorStream();
|
|
6064
7319
|
if (watcher.loop) return;
|
|
6065
7320
|
if (watcher.inFlight.size === 0) {
|
|
6066
7321
|
this.watchers.delete(sessionId);
|
|
@@ -6076,45 +7331,193 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6076
7331
|
});
|
|
6077
7332
|
watcher.loop = loop;
|
|
6078
7333
|
}
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6094
|
-
|
|
6095
|
-
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
|
|
6110
|
-
|
|
6111
|
-
|
|
6112
|
-
|
|
6113
|
-
|
|
6114
|
-
|
|
6115
|
-
|
|
6116
|
-
|
|
6117
|
-
|
|
7334
|
+
ensureSessionErrorStream() {
|
|
7335
|
+
if (this.sessionErrorStream || this.stopped) return;
|
|
7336
|
+
const abort = new AbortController();
|
|
7337
|
+
const loop = this.runSessionErrorStream(abort.signal);
|
|
7338
|
+
this.sessionErrorStream = { abort, loop };
|
|
7339
|
+
}
|
|
7340
|
+
async runSessionErrorStream(signal) {
|
|
7341
|
+
let attempt = 0;
|
|
7342
|
+
let warned = false;
|
|
7343
|
+
while (!this.stopped && !signal.aborted) {
|
|
7344
|
+
const openedAt = this.now();
|
|
7345
|
+
try {
|
|
7346
|
+
const outcome = await readSessionErrorStream(this.port, {
|
|
7347
|
+
signal,
|
|
7348
|
+
onSessionError: (event) => this.handleSessionError(event)
|
|
7349
|
+
});
|
|
7350
|
+
if (outcome.reason === "aborted" || signal.aborted) return;
|
|
7351
|
+
const healthy = this.now() - openedAt >= SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
7352
|
+
if (outcome.reason === "unavailable" || outcome.reason === "ended") {
|
|
7353
|
+
if (!healthy) {
|
|
7354
|
+
const detail = outcome.reason === "unavailable" ? outcome.detail : "stream ended";
|
|
7355
|
+
this.log({
|
|
7356
|
+
level: warned ? "debug" : "warn",
|
|
7357
|
+
message: `OpenCode session-error stream ${warned ? "still unavailable" : "unavailable"} (${detail}); transcript polling remains the evidence path`
|
|
7358
|
+
});
|
|
7359
|
+
warned = true;
|
|
7360
|
+
}
|
|
7361
|
+
}
|
|
7362
|
+
if (healthy) {
|
|
7363
|
+
if (warned) {
|
|
7364
|
+
this.log({
|
|
7365
|
+
level: "info",
|
|
7366
|
+
message: "OpenCode session-error stream reconnected; transcript polling remains the evidence path"
|
|
7367
|
+
});
|
|
7368
|
+
warned = false;
|
|
7369
|
+
}
|
|
7370
|
+
attempt = 0;
|
|
7371
|
+
} else {
|
|
7372
|
+
attempt += 1;
|
|
7373
|
+
}
|
|
7374
|
+
if (this.stopped || signal.aborted) return;
|
|
7375
|
+
await this.sleep(backoffDelay(healthy ? 0 : attempt - 1, this.retry));
|
|
7376
|
+
} catch (err) {
|
|
7377
|
+
if (this.stopped || signal.aborted) return;
|
|
7378
|
+
this.log({
|
|
7379
|
+
level: "error",
|
|
7380
|
+
message: `OpenCode session-error stream failed unexpectedly: ${err instanceof Error ? err.message : String(err)}`
|
|
7381
|
+
});
|
|
7382
|
+
const healthy = this.now() - openedAt >= SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
7383
|
+
const delayAttempt = healthy ? 0 : attempt;
|
|
7384
|
+
attempt = healthy ? 0 : attempt + 1;
|
|
7385
|
+
try {
|
|
7386
|
+
await this.sleep(backoffDelay(delayAttempt, this.retry));
|
|
7387
|
+
} catch (sleepErr) {
|
|
7388
|
+
this.log({
|
|
7389
|
+
level: "error",
|
|
7390
|
+
message: `OpenCode session-error stream backoff failed unexpectedly: ${sleepErr instanceof Error ? sleepErr.message : String(sleepErr)}`
|
|
7391
|
+
});
|
|
7392
|
+
}
|
|
7393
|
+
}
|
|
7394
|
+
}
|
|
7395
|
+
}
|
|
7396
|
+
handleSessionError(event) {
|
|
7397
|
+
try {
|
|
7398
|
+
const watcher = this.watchers.get(event.sessionId);
|
|
7399
|
+
if (!watcher) {
|
|
7400
|
+
this.bufferSessionError(event);
|
|
7401
|
+
this.log({
|
|
7402
|
+
level: "debug",
|
|
7403
|
+
message: `Ignoring session error for unknown session ${event.sessionId.slice(0, 8)}`
|
|
7404
|
+
});
|
|
7405
|
+
return;
|
|
7406
|
+
}
|
|
7407
|
+
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
7408
|
+
this.log({
|
|
7409
|
+
level: "debug",
|
|
7410
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
7411
|
+
conversation_id: watcher.conv.id
|
|
7412
|
+
});
|
|
7413
|
+
return;
|
|
7414
|
+
}
|
|
7415
|
+
const inFlight = [...watcher.inFlight.values()].filter((message) => !message.started && !message.done).sort((a, b) => a.dispatchedAt - b.dispatchedAt)[0];
|
|
7416
|
+
if (!inFlight) {
|
|
7417
|
+
this.bufferSessionError(event);
|
|
7418
|
+
this.log({
|
|
7419
|
+
level: "debug",
|
|
7420
|
+
message: `No queued in-flight turn to correlate with session error in ${event.sessionId.slice(0, 8)}`,
|
|
7421
|
+
conversation_id: watcher.conv.id
|
|
7422
|
+
});
|
|
7423
|
+
return;
|
|
7424
|
+
}
|
|
7425
|
+
if (this.sessionErrorHandled.has(inFlight.evidentMessageId)) return;
|
|
7426
|
+
this.sessionErrorHandled.add(inFlight.evidentMessageId);
|
|
7427
|
+
void this.failFromSessionError(watcher, event, inFlight);
|
|
7428
|
+
} catch (err) {
|
|
7429
|
+
this.log({
|
|
7430
|
+
level: "error",
|
|
7431
|
+
message: `Failed to handle OpenCode session error for ${event.sessionId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`
|
|
7432
|
+
});
|
|
7433
|
+
}
|
|
7434
|
+
}
|
|
7435
|
+
async failFromSessionError(watcher, event, inFlight) {
|
|
7436
|
+
try {
|
|
7437
|
+
const messages = await getSessionMessages(this.port, event.sessionId);
|
|
7438
|
+
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
7439
|
+
if (state !== "queued") {
|
|
7440
|
+
this.log({
|
|
7441
|
+
level: "debug",
|
|
7442
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state}; leaving it to transcript polling`,
|
|
7443
|
+
conversation_id: watcher.conv.id,
|
|
7444
|
+
message_id: inFlight.evidentMessageId
|
|
7445
|
+
});
|
|
7446
|
+
return;
|
|
7447
|
+
}
|
|
7448
|
+
this.log({
|
|
7449
|
+
level: "error",
|
|
7450
|
+
message: `OpenCode could not run message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)}: ${event.reason}`,
|
|
7451
|
+
conversation_id: watcher.conv.id,
|
|
7452
|
+
message_id: inFlight.evidentMessageId
|
|
7453
|
+
});
|
|
7454
|
+
await this.markFailed(
|
|
7455
|
+
watcher.conv.id,
|
|
7456
|
+
inFlight.evidentMessageId,
|
|
7457
|
+
event.sessionId,
|
|
7458
|
+
`OpenCode could not run this turn: ${event.reason}`
|
|
7459
|
+
);
|
|
7460
|
+
inFlight.done = true;
|
|
7461
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
7462
|
+
} catch (err) {
|
|
7463
|
+
if (err instanceof ChannelAuthError) {
|
|
7464
|
+
this.log({
|
|
7465
|
+
level: "warn",
|
|
7466
|
+
message: `OpenCode session error could not mark message ${inFlight.evidentMessageId.slice(0, 8)} failed because authentication failed: ${err.message}; leaving it to transcript polling / the existing give-up path`,
|
|
7467
|
+
conversation_id: watcher.conv.id,
|
|
7468
|
+
message_id: inFlight.evidentMessageId
|
|
7469
|
+
});
|
|
7470
|
+
} else {
|
|
7471
|
+
this.log({
|
|
7472
|
+
level: "warn",
|
|
7473
|
+
message: `OpenCode session error could not mark message ${inFlight.evidentMessageId.slice(0, 8)} failed: ${err instanceof Error ? err.message : String(err)}; leaving it to transcript polling / the existing give-up path`,
|
|
7474
|
+
conversation_id: watcher.conv.id,
|
|
7475
|
+
message_id: inFlight.evidentMessageId
|
|
7476
|
+
});
|
|
7477
|
+
}
|
|
7478
|
+
} finally {
|
|
7479
|
+
this.sessionErrorHandled.delete(inFlight.evidentMessageId);
|
|
7480
|
+
}
|
|
7481
|
+
}
|
|
7482
|
+
/**
|
|
7483
|
+
* The per-session polling loop (WI-3). Once per tick it:
|
|
7484
|
+
* 1. polls `GET /session/:id/message` once and, per in-flight message,
|
|
7485
|
+
* computes `messageRunState` and fires markProcessing (queued→running) /
|
|
7486
|
+
* markDone (done) exactly once per transition;
|
|
7487
|
+
* 2. applies the idle-path re-dispatch guard (a dispatched message that never
|
|
7488
|
+
* APPEARS → re-dispatch — D1 obligation 2);
|
|
7489
|
+
* 3. polls `/question` + `/permission` (scoped to the session) and surfaces
|
|
7490
|
+
* NEW ones via `reportInteraction`, carrying the PAUSED message's own
|
|
7491
|
+
* `source_message_id`;
|
|
7492
|
+
* 4. drops messages that completed or timed out from the in-flight set.
|
|
7493
|
+
* Exits when the in-flight set empties. Never throws.
|
|
7494
|
+
*
|
|
7495
|
+
* `generation` (#1618) is the incarnation this call was started under.
|
|
7496
|
+
* `reconcileWatchers` can restart a stalled loop by bumping
|
|
7497
|
+
* `watcher.generation` and starting a NEW `runWatcherLoop` over the same
|
|
7498
|
+
* `SessionWatcher` object — the stalled promise itself cannot be cancelled,
|
|
7499
|
+
* so this loop instead checks at the top of every iteration, right after
|
|
7500
|
+
* waking from `sleep`, and right before servicing any message, and quietly
|
|
7501
|
+
* retires (returns without touching anything) the moment it is no longer the
|
|
7502
|
+
* watcher's current generation. Retiring mid-tick can still let ONE
|
|
7503
|
+
* `serviceInFlightMessage` pass complete first — acceptable, since that
|
|
7504
|
+
* method contains no non-idempotent action.
|
|
7505
|
+
*/
|
|
7506
|
+
async runWatcherLoop(sessionId, watcher, generation) {
|
|
7507
|
+
try {
|
|
7508
|
+
while (watcher.inFlight.size > 0) {
|
|
7509
|
+
if (watcher.generation !== generation) return;
|
|
7510
|
+
watcher.lastTickAt = this.now();
|
|
7511
|
+
await this.sleep(this.pausedPollIntervalMs);
|
|
7512
|
+
if (watcher.generation !== generation) return;
|
|
7513
|
+
let messages = null;
|
|
7514
|
+
try {
|
|
7515
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
7516
|
+
if (res.ok) {
|
|
7517
|
+
const body = await res.json();
|
|
7518
|
+
messages = Array.isArray(body) ? body : null;
|
|
7519
|
+
}
|
|
7520
|
+
} catch {
|
|
6118
7521
|
}
|
|
6119
7522
|
if (messages != null && messages.length > 0) {
|
|
6120
7523
|
watcher.lastGoodPollAt = this.now();
|
|
@@ -6188,6 +7591,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6188
7591
|
const conv = watcher.conv;
|
|
6189
7592
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
6190
7593
|
const id = inFlight.evidentMessageId;
|
|
7594
|
+
if (messages && collectTaskCalls(messages, inFlight.opencodeMessageId).length > 0 && !this.subagentInvocationPrefetches.has(id)) {
|
|
7595
|
+
void this.resolveSubagentInvocations(
|
|
7596
|
+
messages,
|
|
7597
|
+
inFlight.opencodeMessageId,
|
|
7598
|
+
id,
|
|
7599
|
+
"prefetch"
|
|
7600
|
+
).catch((err) => {
|
|
7601
|
+
this.log({
|
|
7602
|
+
level: "warn",
|
|
7603
|
+
message: `Best-effort subagent usage prefetch failed for message ${id.slice(0, 8)} \u2014 omitting invocation telemetry: ${err instanceof Error ? err.message : String(err)}`,
|
|
7604
|
+
conversation_id: conv.id,
|
|
7605
|
+
message_id: id
|
|
7606
|
+
});
|
|
7607
|
+
});
|
|
7608
|
+
}
|
|
6191
7609
|
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
6192
7610
|
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
6193
7611
|
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
@@ -6241,6 +7659,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6241
7659
|
message_id: inFlight.evidentMessageId
|
|
6242
7660
|
});
|
|
6243
7661
|
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
7662
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
7663
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
7664
|
+
messages,
|
|
7665
|
+
inFlight.opencodeMessageId,
|
|
7666
|
+
inFlight.evidentMessageId
|
|
7667
|
+
);
|
|
6244
7668
|
const failure = await this.classifyModelAuthFailure(messages, inFlight.opencodeMessageId);
|
|
6245
7669
|
try {
|
|
6246
7670
|
await this.markFailed(
|
|
@@ -6249,7 +7673,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6249
7673
|
sessionId,
|
|
6250
7674
|
error2,
|
|
6251
7675
|
usage,
|
|
6252
|
-
failure
|
|
7676
|
+
failure,
|
|
7677
|
+
usageAgentName,
|
|
7678
|
+
subagentInvocations
|
|
6253
7679
|
);
|
|
6254
7680
|
} catch (err) {
|
|
6255
7681
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -6282,6 +7708,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6282
7708
|
return;
|
|
6283
7709
|
}
|
|
6284
7710
|
inFlight.done = true;
|
|
7711
|
+
await this.reportSubagentAuthFailures(
|
|
7712
|
+
watcher.conv.id,
|
|
7713
|
+
inFlight.opencodeMessageId,
|
|
7714
|
+
inFlight.evidentMessageId,
|
|
7715
|
+
messages
|
|
7716
|
+
);
|
|
6285
7717
|
}
|
|
6286
7718
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6287
7719
|
return;
|
|
@@ -6484,6 +7916,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6484
7916
|
});
|
|
6485
7917
|
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
6486
7918
|
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
7919
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
7920
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
7921
|
+
messages,
|
|
7922
|
+
inFlight.opencodeMessageId,
|
|
7923
|
+
inFlight.evidentMessageId
|
|
7924
|
+
);
|
|
6487
7925
|
try {
|
|
6488
7926
|
await this.markDone(
|
|
6489
7927
|
conv.id,
|
|
@@ -6491,7 +7929,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6491
7929
|
sessionId,
|
|
6492
7930
|
inFlight.opencodeMessageId,
|
|
6493
7931
|
title,
|
|
6494
|
-
usage
|
|
7932
|
+
usage,
|
|
7933
|
+
usageAgentName,
|
|
7934
|
+
subagentInvocations
|
|
6495
7935
|
);
|
|
6496
7936
|
} catch (err) {
|
|
6497
7937
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -6524,6 +7964,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6524
7964
|
return;
|
|
6525
7965
|
}
|
|
6526
7966
|
inFlight.done = true;
|
|
7967
|
+
await this.reportSubagentAuthFailures(
|
|
7968
|
+
watcher.conv.id,
|
|
7969
|
+
inFlight.opencodeMessageId,
|
|
7970
|
+
inFlight.evidentMessageId,
|
|
7971
|
+
messages
|
|
7972
|
+
);
|
|
6527
7973
|
}
|
|
6528
7974
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6529
7975
|
}
|
|
@@ -6664,6 +8110,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6664
8110
|
if (state === "failed" && !restartAborted) {
|
|
6665
8111
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
6666
8112
|
const usage = messageUsage(messages, ocId ?? "");
|
|
8113
|
+
const usageAgentName = this.usageAgentName(messages, ocId ?? "");
|
|
8114
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
8115
|
+
messages,
|
|
8116
|
+
ocId ?? "",
|
|
8117
|
+
row.id
|
|
8118
|
+
);
|
|
6667
8119
|
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
6668
8120
|
this.log({
|
|
6669
8121
|
level: "error",
|
|
@@ -6672,7 +8124,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6672
8124
|
message_id: row.id
|
|
6673
8125
|
});
|
|
6674
8126
|
try {
|
|
6675
|
-
await this.markFailed(
|
|
8127
|
+
await this.markFailed(
|
|
8128
|
+
row.conversation_id,
|
|
8129
|
+
row.id,
|
|
8130
|
+
sessionId,
|
|
8131
|
+
error2,
|
|
8132
|
+
usage,
|
|
8133
|
+
failure,
|
|
8134
|
+
usageAgentName,
|
|
8135
|
+
subagentInvocations
|
|
8136
|
+
);
|
|
6676
8137
|
} catch (err) {
|
|
6677
8138
|
if (err instanceof ChannelAuthError) throw err;
|
|
6678
8139
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -6694,6 +8155,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6694
8155
|
});
|
|
6695
8156
|
return;
|
|
6696
8157
|
}
|
|
8158
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
6697
8159
|
this.dontRedispatch.delete(row.id);
|
|
6698
8160
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
6699
8161
|
return;
|
|
@@ -6833,7 +8295,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6833
8295
|
try {
|
|
6834
8296
|
const title = await this.resolveSessionTitle(sessionId, row.conversation_id);
|
|
6835
8297
|
const usage = messageUsage(messages, ocId ?? "");
|
|
6836
|
-
|
|
8298
|
+
const usageAgentName = this.usageAgentName(messages, ocId ?? "");
|
|
8299
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
8300
|
+
messages,
|
|
8301
|
+
ocId ?? "",
|
|
8302
|
+
row.id
|
|
8303
|
+
);
|
|
8304
|
+
await this.markDone(
|
|
8305
|
+
row.conversation_id,
|
|
8306
|
+
row.id,
|
|
8307
|
+
sessionId,
|
|
8308
|
+
ocId,
|
|
8309
|
+
title,
|
|
8310
|
+
usage,
|
|
8311
|
+
usageAgentName,
|
|
8312
|
+
subagentInvocations
|
|
8313
|
+
);
|
|
6837
8314
|
} catch (err) {
|
|
6838
8315
|
if (err instanceof ChannelAuthError) throw err;
|
|
6839
8316
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -6855,6 +8332,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6855
8332
|
});
|
|
6856
8333
|
return;
|
|
6857
8334
|
}
|
|
8335
|
+
if (ocId !== null) {
|
|
8336
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
|
|
8337
|
+
}
|
|
6858
8338
|
this.dontRedispatch.delete(row.id);
|
|
6859
8339
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
6860
8340
|
}
|
|
@@ -6948,14 +8428,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6948
8428
|
this.unconfirmedDispatchFailures.delete(row.id);
|
|
6949
8429
|
this.sessions.delete(readoptConv.id);
|
|
6950
8430
|
this.supersede(readoptConv.id, sessionId);
|
|
6951
|
-
const
|
|
8431
|
+
const errorMessage3 = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
6952
8432
|
this.log({
|
|
6953
8433
|
level: "error",
|
|
6954
|
-
message:
|
|
8434
|
+
message: errorMessage3,
|
|
6955
8435
|
conversation_id: row.conversation_id,
|
|
6956
8436
|
message_id: row.id
|
|
6957
8437
|
});
|
|
6958
|
-
await this.markFailed(row.conversation_id, row.id, null,
|
|
8438
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
6959
8439
|
this.log({
|
|
6960
8440
|
level: "warn",
|
|
6961
8441
|
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -7234,6 +8714,166 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7234
8714
|
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
7235
8715
|
return parent;
|
|
7236
8716
|
}
|
|
8717
|
+
usageAgentName(messages, userMessageId) {
|
|
8718
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
8719
|
+
const mode = reply?.info?.mode;
|
|
8720
|
+
if (typeof mode === "string" && mode.length > 0) return mode;
|
|
8721
|
+
const agent = reply?.info?.agent;
|
|
8722
|
+
return typeof agent === "string" && agent.length > 0 ? agent : null;
|
|
8723
|
+
}
|
|
8724
|
+
async resolveSubagentInvocations(messages, userMessageId, messageId, phase = "terminal") {
|
|
8725
|
+
if (!messages) return void 0;
|
|
8726
|
+
const cache = phase === "prefetch" ? this.subagentInvocationPrefetches : this.subagentInvocationCollections;
|
|
8727
|
+
const cached = cache.get(messageId);
|
|
8728
|
+
if (cached) return cached;
|
|
8729
|
+
const collection = this.buildSubagentInvocations(messages, userMessageId, messageId).catch(
|
|
8730
|
+
(err) => {
|
|
8731
|
+
this.log({
|
|
8732
|
+
level: "warn",
|
|
8733
|
+
message: `Best-effort subagent usage collection failed for message ${messageId.slice(0, 8)} \u2014 omitting invocation telemetry: ${err instanceof Error ? err.message : String(err)}`,
|
|
8734
|
+
message_id: messageId
|
|
8735
|
+
});
|
|
8736
|
+
return void 0;
|
|
8737
|
+
}
|
|
8738
|
+
);
|
|
8739
|
+
cache.set(messageId, collection);
|
|
8740
|
+
const result = await collection;
|
|
8741
|
+
if (result === void 0 && cache.get(messageId) === collection) cache.delete(messageId);
|
|
8742
|
+
return result;
|
|
8743
|
+
}
|
|
8744
|
+
clearSubagentInvocationCaches(messageId) {
|
|
8745
|
+
this.subagentInvocationCollections.delete(messageId);
|
|
8746
|
+
this.subagentInvocationPrefetches.delete(messageId);
|
|
8747
|
+
}
|
|
8748
|
+
async buildSubagentInvocations(messages, userMessageId, messageId) {
|
|
8749
|
+
const rootCalls = collectTaskCalls(messages, userMessageId);
|
|
8750
|
+
if (rootCalls.length === 0) return void 0;
|
|
8751
|
+
const childMessages = /* @__PURE__ */ new Map();
|
|
8752
|
+
const seenCallIds = new Set(rootCalls.map((call) => call.callID));
|
|
8753
|
+
const work = rootCalls.map((call) => ({
|
|
8754
|
+
call,
|
|
8755
|
+
depth: 1
|
|
8756
|
+
}));
|
|
8757
|
+
const payload = [];
|
|
8758
|
+
const fetchChildMessages = (sessionId) => {
|
|
8759
|
+
const cached = childMessages.get(sessionId);
|
|
8760
|
+
if (cached) return cached;
|
|
8761
|
+
const pending = (async () => {
|
|
8762
|
+
try {
|
|
8763
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
8764
|
+
if (!res.ok) {
|
|
8765
|
+
this.log({
|
|
8766
|
+
level: "warn",
|
|
8767
|
+
message: `Best-effort subagent session fetch for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 omitting invocation telemetry`,
|
|
8768
|
+
message_id: messageId
|
|
8769
|
+
});
|
|
8770
|
+
return null;
|
|
8771
|
+
}
|
|
8772
|
+
const body = await res.json();
|
|
8773
|
+
if (!Array.isArray(body)) throw new Error("response body was not a message array");
|
|
8774
|
+
return body;
|
|
8775
|
+
} catch (err) {
|
|
8776
|
+
this.log({
|
|
8777
|
+
level: "warn",
|
|
8778
|
+
message: `Best-effort subagent session fetch failed for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} \u2014 omitting invocation telemetry: ${err instanceof Error ? err.message : String(err)}`,
|
|
8779
|
+
message_id: messageId
|
|
8780
|
+
});
|
|
8781
|
+
return null;
|
|
8782
|
+
}
|
|
8783
|
+
})();
|
|
8784
|
+
childMessages.set(sessionId, pending);
|
|
8785
|
+
return pending;
|
|
8786
|
+
};
|
|
8787
|
+
const fetchChildWithoutBlocking = async (sessionId) => {
|
|
8788
|
+
const pending = fetchChildMessages(sessionId);
|
|
8789
|
+
let timer;
|
|
8790
|
+
const timeout = new Promise((resolve4) => {
|
|
8791
|
+
timer = setTimeout(() => {
|
|
8792
|
+
this.log({
|
|
8793
|
+
level: "warn",
|
|
8794
|
+
message: `Best-effort subagent session fetch for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} was slow \u2014 omitting invocation telemetry without delaying completion`,
|
|
8795
|
+
message_id: messageId
|
|
8796
|
+
});
|
|
8797
|
+
resolve4(null);
|
|
8798
|
+
}, SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS);
|
|
8799
|
+
});
|
|
8800
|
+
try {
|
|
8801
|
+
return await Promise.race([pending, timeout]);
|
|
8802
|
+
} finally {
|
|
8803
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
8804
|
+
}
|
|
8805
|
+
};
|
|
8806
|
+
while (work.length > 0) {
|
|
8807
|
+
const groups = /* @__PURE__ */ new Map();
|
|
8808
|
+
for (const item of work.splice(0)) {
|
|
8809
|
+
const group = groups.get(item.call.childSessionId) ?? [];
|
|
8810
|
+
group.push(item);
|
|
8811
|
+
groups.set(item.call.childSessionId, group);
|
|
8812
|
+
}
|
|
8813
|
+
const groupResults = await Promise.all(
|
|
8814
|
+
[...groups].map(async ([sessionId, items]) => ({
|
|
8815
|
+
sessionId,
|
|
8816
|
+
items,
|
|
8817
|
+
messages: sessionId === null ? [] : await fetchChildWithoutBlocking(sessionId)
|
|
8818
|
+
}))
|
|
8819
|
+
);
|
|
8820
|
+
for (const { sessionId, items, messages: child } of groupResults) {
|
|
8821
|
+
if (sessionId !== null && child === null) continue;
|
|
8822
|
+
const attribution = sessionId === null ? { invocations: [], unattributed: [] } : attributeTaskCallUsage(
|
|
8823
|
+
child,
|
|
8824
|
+
items.map(({ call }) => ({
|
|
8825
|
+
callID: call.callID,
|
|
8826
|
+
timeStart: call.timeStart,
|
|
8827
|
+
timeEnd: call.timeEnd
|
|
8828
|
+
}))
|
|
8829
|
+
);
|
|
8830
|
+
if (sessionId !== null && attribution.unattributed.length > 0) {
|
|
8831
|
+
this.log({
|
|
8832
|
+
level: "warn",
|
|
8833
|
+
message: `Omitted ${attribution.unattributed.length} unattributable assistant message(s) from subagent usage for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} \u2014 assigned to no invocation window`,
|
|
8834
|
+
message_id: messageId
|
|
8835
|
+
});
|
|
8836
|
+
}
|
|
8837
|
+
const usageByCall = new Map(
|
|
8838
|
+
attribution.invocations.map((invocation) => [invocation.callID, invocation.usage])
|
|
8839
|
+
);
|
|
8840
|
+
const messagesByCall = new Map(
|
|
8841
|
+
attribution.invocations.map((invocation) => [invocation.callID, invocation.messages])
|
|
8842
|
+
);
|
|
8843
|
+
for (const { call, depth } of items) {
|
|
8844
|
+
const usage = usageByCall.get(call.callID) ?? null;
|
|
8845
|
+
payload.push({
|
|
8846
|
+
tool_call_id: call.callID,
|
|
8847
|
+
agent_name: call.subagentName,
|
|
8848
|
+
opencode_session_id: call.childSessionId,
|
|
8849
|
+
parent_opencode_session_id: call.parentSessionId,
|
|
8850
|
+
depth,
|
|
8851
|
+
status: call.status,
|
|
8852
|
+
started_at: call.timeStart === null ? null : new Date(call.timeStart).toISOString(),
|
|
8853
|
+
ended_at: call.timeEnd === null ? null : new Date(call.timeEnd).toISOString(),
|
|
8854
|
+
usage_provider_id: usage?.usage_provider_id ?? call.model?.providerID ?? null,
|
|
8855
|
+
usage_model_id: usage?.usage_model_id ?? call.model?.modelID ?? null,
|
|
8856
|
+
usage_tokens_input: usage?.usage_tokens_input ?? null,
|
|
8857
|
+
usage_tokens_output: usage?.usage_tokens_output ?? null,
|
|
8858
|
+
usage_tokens_reasoning: usage?.usage_tokens_reasoning ?? null,
|
|
8859
|
+
usage_tokens_cache_read: usage?.usage_tokens_cache_read ?? null,
|
|
8860
|
+
usage_tokens_cache_write: usage?.usage_tokens_cache_write ?? null,
|
|
8861
|
+
usage_cost_usd: usage?.usage_cost_usd ?? null
|
|
8862
|
+
});
|
|
8863
|
+
for (const assigned of messagesByCall.get(call.callID) ?? []) {
|
|
8864
|
+
const parentId = assigned.info?.parentID ?? assigned.parentID;
|
|
8865
|
+
if (!parentId) continue;
|
|
8866
|
+
for (const nested of collectTaskCalls([assigned], parentId)) {
|
|
8867
|
+
if (seenCallIds.has(nested.callID)) continue;
|
|
8868
|
+
seenCallIds.add(nested.callID);
|
|
8869
|
+
work.push({ call: nested, depth: depth + 1 });
|
|
8870
|
+
}
|
|
8871
|
+
}
|
|
8872
|
+
}
|
|
8873
|
+
}
|
|
8874
|
+
}
|
|
8875
|
+
return payload.length > 0 ? payload : void 0;
|
|
8876
|
+
}
|
|
7237
8877
|
/**
|
|
7238
8878
|
* OpenCode's synchronous default session title (e.g.
|
|
7239
8879
|
* `"New session - 1737800000000"`), assigned immediately when a session is
|
|
@@ -7570,6 +9210,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7570
9210
|
throw new Error(`Failed to get pending conversations: HTTP ${res.status}`);
|
|
7571
9211
|
}
|
|
7572
9212
|
const data = await res.json();
|
|
9213
|
+
this.recycleRequestedFlag = data.recycle_requested === true;
|
|
7573
9214
|
let conversations = data.conversations;
|
|
7574
9215
|
if (this.conversationFilter) {
|
|
7575
9216
|
conversations = conversations.filter((c) => c.id === this.conversationFilter);
|
|
@@ -7716,7 +9357,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7716
9357
|
* watcher retries next tick within the
|
|
7717
9358
|
* deadline, Finding 4).
|
|
7718
9359
|
*/
|
|
7719
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage) {
|
|
9360
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
7720
9361
|
const res = await this.fetchImpl(
|
|
7721
9362
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
7722
9363
|
{
|
|
@@ -7732,15 +9373,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7732
9373
|
opencode_session_id: sessionId,
|
|
7733
9374
|
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
7734
9375
|
...title ? { title } : {},
|
|
7735
|
-
...usage ? usage : {}
|
|
9376
|
+
...usage ? usage : {},
|
|
9377
|
+
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
9378
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
7736
9379
|
})
|
|
7737
9380
|
}
|
|
7738
9381
|
);
|
|
7739
9382
|
this.assertAuth(res, "marking message as done");
|
|
7740
|
-
if (res.ok)
|
|
9383
|
+
if (res.ok) {
|
|
9384
|
+
this.clearSubagentInvocationCaches(messageId);
|
|
9385
|
+
return;
|
|
9386
|
+
}
|
|
7741
9387
|
if (isRetryableStatus(res.status)) {
|
|
7742
9388
|
throw new Error(`marking message as done: HTTP ${res.status}`);
|
|
7743
9389
|
}
|
|
9390
|
+
this.clearSubagentInvocationCaches(messageId);
|
|
7744
9391
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
7745
9392
|
}
|
|
7746
9393
|
/**
|
|
@@ -7755,7 +9402,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7755
9402
|
* exists but is wedged, so the next attempt must get a fresh one
|
|
7756
9403
|
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
7757
9404
|
*/
|
|
7758
|
-
async markFailed(conversationId, messageId, sessionId, error2, usage, failure) {
|
|
9405
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage, failure, usageAgentName, subagentInvocations) {
|
|
7759
9406
|
const body = { status: "failed" };
|
|
7760
9407
|
if (sessionId === null) {
|
|
7761
9408
|
body.opencode_session_id = null;
|
|
@@ -7764,23 +9411,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7764
9411
|
}
|
|
7765
9412
|
if (error2 !== void 0) body.error = error2;
|
|
7766
9413
|
if (usage) Object.assign(body, usage);
|
|
9414
|
+
if (usageAgentName) body.usage_agent_name = usageAgentName;
|
|
9415
|
+
if (subagentInvocations && subagentInvocations.length > 0) {
|
|
9416
|
+
body.subagent_invocations = subagentInvocations;
|
|
9417
|
+
}
|
|
7767
9418
|
if (failure) {
|
|
7768
9419
|
body.failure_kind = failure.kind;
|
|
7769
9420
|
body.failure_provider_id = failure.providerId;
|
|
7770
9421
|
body.failure_model_id = failure.modelId;
|
|
7771
9422
|
body.failure_reason = failure.reason;
|
|
7772
9423
|
}
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
|
|
7779
|
-
|
|
7780
|
-
|
|
7781
|
-
|
|
7782
|
-
|
|
7783
|
-
|
|
9424
|
+
try {
|
|
9425
|
+
await this.callWithRetry(
|
|
9426
|
+
"marking message as failed",
|
|
9427
|
+
() => this.fetchImpl(
|
|
9428
|
+
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
9429
|
+
{
|
|
9430
|
+
method: "PATCH",
|
|
9431
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
9432
|
+
body: JSON.stringify(body)
|
|
9433
|
+
}
|
|
9434
|
+
)
|
|
9435
|
+
);
|
|
9436
|
+
} catch (err) {
|
|
9437
|
+
if (err instanceof ChannelTerminalError) this.clearSubagentInvocationCaches(messageId);
|
|
9438
|
+
throw err;
|
|
9439
|
+
}
|
|
9440
|
+
this.clearSubagentInvocationCaches(messageId);
|
|
7784
9441
|
}
|
|
7785
9442
|
/**
|
|
7786
9443
|
* Classify an errored turn as a model-auth failure (#736 P1-4), for `markFailed`.
|
|
@@ -7805,6 +9462,111 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7805
9462
|
reply?.info?.modelID ?? null
|
|
7806
9463
|
);
|
|
7807
9464
|
}
|
|
9465
|
+
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
9466
|
+
const providerId = failure.providerId ?? "(unknown)";
|
|
9467
|
+
try {
|
|
9468
|
+
const res = await this.fetchImpl(
|
|
9469
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
|
|
9470
|
+
{
|
|
9471
|
+
method: "POST",
|
|
9472
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
9473
|
+
body: JSON.stringify({
|
|
9474
|
+
provider_id: failure.providerId,
|
|
9475
|
+
model_id: failure.modelId,
|
|
9476
|
+
reason: failure.reason
|
|
9477
|
+
})
|
|
9478
|
+
}
|
|
9479
|
+
);
|
|
9480
|
+
if (!res.ok) {
|
|
9481
|
+
this.log({
|
|
9482
|
+
level: "warn",
|
|
9483
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
9484
|
+
conversation_id: conversationId,
|
|
9485
|
+
message_id: messageId
|
|
9486
|
+
});
|
|
9487
|
+
}
|
|
9488
|
+
} catch (err) {
|
|
9489
|
+
this.log({
|
|
9490
|
+
level: "warn",
|
|
9491
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
9492
|
+
conversation_id: conversationId,
|
|
9493
|
+
message_id: messageId
|
|
9494
|
+
});
|
|
9495
|
+
}
|
|
9496
|
+
}
|
|
9497
|
+
async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
|
|
9498
|
+
try {
|
|
9499
|
+
const res = await this.fetchImpl(
|
|
9500
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
|
|
9501
|
+
{
|
|
9502
|
+
method: "DELETE",
|
|
9503
|
+
headers: { Authorization: this.getAuthHeader() }
|
|
9504
|
+
}
|
|
9505
|
+
);
|
|
9506
|
+
if (!res.ok) {
|
|
9507
|
+
this.log({
|
|
9508
|
+
level: "warn",
|
|
9509
|
+
message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
9510
|
+
conversation_id: conversationId,
|
|
9511
|
+
message_id: messageId
|
|
9512
|
+
});
|
|
9513
|
+
}
|
|
9514
|
+
} catch (err) {
|
|
9515
|
+
this.log({
|
|
9516
|
+
level: "warn",
|
|
9517
|
+
message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
9518
|
+
conversation_id: conversationId,
|
|
9519
|
+
message_id: messageId
|
|
9520
|
+
});
|
|
9521
|
+
}
|
|
9522
|
+
}
|
|
9523
|
+
async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
|
|
9524
|
+
const refs = collectSubagentSessions(messages, opencodeMessageId);
|
|
9525
|
+
if (refs.length === 0) return;
|
|
9526
|
+
const failedProviders = /* @__PURE__ */ new Map();
|
|
9527
|
+
const succeededProviders = /* @__PURE__ */ new Set();
|
|
9528
|
+
for (const ref of refs) {
|
|
9529
|
+
try {
|
|
9530
|
+
const childMessages = await getSessionMessages(this.port, ref.sessionId);
|
|
9531
|
+
if (childMessages === null) {
|
|
9532
|
+
this.log({
|
|
9533
|
+
level: "debug",
|
|
9534
|
+
message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
|
|
9535
|
+
conversation_id: conversationId,
|
|
9536
|
+
message_id: evidentMessageId
|
|
9537
|
+
});
|
|
9538
|
+
continue;
|
|
9539
|
+
}
|
|
9540
|
+
const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
|
|
9541
|
+
if (!outcome) continue;
|
|
9542
|
+
if (outcome.outcome === "failed") {
|
|
9543
|
+
failedProviders.set(outcome.providerId, outcome.failure);
|
|
9544
|
+
} else {
|
|
9545
|
+
succeededProviders.add(outcome.providerId);
|
|
9546
|
+
}
|
|
9547
|
+
} catch (err) {
|
|
9548
|
+
this.log({
|
|
9549
|
+
level: "warn",
|
|
9550
|
+
message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
9551
|
+
conversation_id: conversationId,
|
|
9552
|
+
message_id: evidentMessageId
|
|
9553
|
+
});
|
|
9554
|
+
}
|
|
9555
|
+
}
|
|
9556
|
+
for (const [providerId, failure] of failedProviders) {
|
|
9557
|
+
this.log({
|
|
9558
|
+
level: "warn",
|
|
9559
|
+
message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
|
|
9560
|
+
conversation_id: conversationId,
|
|
9561
|
+
message_id: evidentMessageId
|
|
9562
|
+
});
|
|
9563
|
+
await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
|
|
9564
|
+
}
|
|
9565
|
+
for (const providerId of succeededProviders) {
|
|
9566
|
+
if (failedProviders.has(providerId)) continue;
|
|
9567
|
+
await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
|
|
9568
|
+
}
|
|
9569
|
+
}
|
|
7808
9570
|
/**
|
|
7809
9571
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
7810
9572
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -7958,6 +9720,13 @@ import chalk5 from "chalk";
|
|
|
7958
9720
|
import ora2 from "ora";
|
|
7959
9721
|
import { select as select2 } from "@inquirer/prompts";
|
|
7960
9722
|
var INTERACTIVE_START_TIMEOUT_MS = 3e4;
|
|
9723
|
+
function checkNonInteractivePortConflict(port, isPortInUseFn) {
|
|
9724
|
+
if (isPortInUseFn(port)) {
|
|
9725
|
+
throw new Error(
|
|
9726
|
+
`Port ${port} is already in use by a non-OpenCode process. Free it or pass --port.`
|
|
9727
|
+
);
|
|
9728
|
+
}
|
|
9729
|
+
}
|
|
7961
9730
|
async function ensureOpenCodeRunning(ctx) {
|
|
7962
9731
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
7963
9732
|
if (healthCheck.healthy) {
|
|
@@ -8005,8 +9774,9 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
8005
9774
|
}
|
|
8006
9775
|
}
|
|
8007
9776
|
if (!ctx.interactive) {
|
|
9777
|
+
checkNonInteractivePortConflict(ctx.port, isPortInUse);
|
|
8008
9778
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
8009
|
-
const proc = await startOpenCode(ctx.port);
|
|
9779
|
+
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
8010
9780
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
8011
9781
|
if (!health.healthy) {
|
|
8012
9782
|
return {
|
|
@@ -8024,66 +9794,721 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
8024
9794
|
notReadyReason: null
|
|
8025
9795
|
};
|
|
8026
9796
|
}
|
|
8027
|
-
let port = ctx.port;
|
|
8028
|
-
if (isPortInUse(port)) {
|
|
8029
|
-
console.log(chalk5.yellow(`
|
|
8030
|
-
Port ${port} is already in use.`));
|
|
8031
|
-
const alternativePort = findAvailablePort(port + 1);
|
|
8032
|
-
if (alternativePort) {
|
|
8033
|
-
const useAlternative = await select2({
|
|
8034
|
-
message: `Use port ${alternativePort} instead?`,
|
|
8035
|
-
choices: [
|
|
8036
|
-
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
8037
|
-
{ name: "No, I will free the port manually", value: "no" }
|
|
8038
|
-
]
|
|
8039
|
-
});
|
|
8040
|
-
if (useAlternative === "yes") {
|
|
8041
|
-
port = alternativePort;
|
|
8042
|
-
} else {
|
|
8043
|
-
throw new Error(`Port ${ctx.port} is in use`);
|
|
9797
|
+
let port = ctx.port;
|
|
9798
|
+
if (isPortInUse(port)) {
|
|
9799
|
+
console.log(chalk5.yellow(`
|
|
9800
|
+
Port ${port} is already in use.`));
|
|
9801
|
+
const alternativePort = findAvailablePort(port + 1);
|
|
9802
|
+
if (alternativePort) {
|
|
9803
|
+
const useAlternative = await select2({
|
|
9804
|
+
message: `Use port ${alternativePort} instead?`,
|
|
9805
|
+
choices: [
|
|
9806
|
+
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
9807
|
+
{ name: "No, I will free the port manually", value: "no" }
|
|
9808
|
+
]
|
|
9809
|
+
});
|
|
9810
|
+
if (useAlternative === "yes") {
|
|
9811
|
+
port = alternativePort;
|
|
9812
|
+
} else {
|
|
9813
|
+
throw new Error(`Port ${ctx.port} is in use`);
|
|
9814
|
+
}
|
|
9815
|
+
}
|
|
9816
|
+
}
|
|
9817
|
+
const action = await select2({
|
|
9818
|
+
message: "OpenCode is not running. What would you like to do?",
|
|
9819
|
+
choices: [
|
|
9820
|
+
{
|
|
9821
|
+
name: "Start OpenCode for me",
|
|
9822
|
+
value: "start",
|
|
9823
|
+
description: `Run 'opencode serve --port ${port}'`
|
|
9824
|
+
},
|
|
9825
|
+
{
|
|
9826
|
+
name: "Show me the command",
|
|
9827
|
+
value: "manual",
|
|
9828
|
+
description: "Display the command to run manually"
|
|
9829
|
+
},
|
|
9830
|
+
{
|
|
9831
|
+
name: "Continue without OpenCode",
|
|
9832
|
+
value: "continue",
|
|
9833
|
+
description: "Requests will fail until OpenCode starts"
|
|
9834
|
+
}
|
|
9835
|
+
]
|
|
9836
|
+
});
|
|
9837
|
+
if (action === "manual") {
|
|
9838
|
+
blank();
|
|
9839
|
+
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
9840
|
+
blank();
|
|
9841
|
+
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
9842
|
+
blank();
|
|
9843
|
+
throw new Error("Please start OpenCode manually");
|
|
9844
|
+
}
|
|
9845
|
+
if (action === "start") {
|
|
9846
|
+
const spinner = ora2("Starting OpenCode...").start();
|
|
9847
|
+
const proc = await startOpenCode(port, { inheritStdio: ctx.inheritStdio });
|
|
9848
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
9849
|
+
if (!health.healthy) {
|
|
9850
|
+
spinner.fail("Failed to start OpenCode");
|
|
9851
|
+
throw new Error("OpenCode failed to start");
|
|
9852
|
+
}
|
|
9853
|
+
spinner.stop();
|
|
9854
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
9855
|
+
}
|
|
9856
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
9857
|
+
}
|
|
9858
|
+
|
|
9859
|
+
// src/commands/ensure-opencode-v2.ts
|
|
9860
|
+
import chalk6 from "chalk";
|
|
9861
|
+
import { select as select3 } from "@inquirer/prompts";
|
|
9862
|
+
async function probeOpenCode2WithoutPassword(port) {
|
|
9863
|
+
try {
|
|
9864
|
+
const response = await fetch(`http://127.0.0.1:${port}/global/health`, {
|
|
9865
|
+
signal: AbortSignal.timeout(2e3)
|
|
9866
|
+
});
|
|
9867
|
+
if (response.status === 401) {
|
|
9868
|
+
return { healthy: false, authFailed: true, error: "HTTP 401" };
|
|
9869
|
+
}
|
|
9870
|
+
if (!response.ok) {
|
|
9871
|
+
return { healthy: false, error: `HTTP ${response.status}` };
|
|
9872
|
+
}
|
|
9873
|
+
return { healthy: true };
|
|
9874
|
+
} catch (error2) {
|
|
9875
|
+
return {
|
|
9876
|
+
healthy: false,
|
|
9877
|
+
error: error2 instanceof Error ? error2.message : "Unknown error"
|
|
9878
|
+
};
|
|
9879
|
+
}
|
|
9880
|
+
}
|
|
9881
|
+
function unknownPasswordError(port) {
|
|
9882
|
+
return new Error(
|
|
9883
|
+
`OpenCode V2 (opencode2) is already running on port ${port} with a password this runner doesn't know. Free the port or pass --port.`
|
|
9884
|
+
);
|
|
9885
|
+
}
|
|
9886
|
+
function v2SessionSupportIncompleteError() {
|
|
9887
|
+
return new Error(
|
|
9888
|
+
"OpenCode V2 (opencode2) session support is incomplete \u2014 see https://github.com/sroze/evident/issues/2075. Not starting a new opencode2 process."
|
|
9889
|
+
);
|
|
9890
|
+
}
|
|
9891
|
+
async function ensureOpenCode2Running(ctx) {
|
|
9892
|
+
const initialHealth = await probeOpenCode2WithoutPassword(ctx.port);
|
|
9893
|
+
if (initialHealth.authFailed) {
|
|
9894
|
+
throw unknownPasswordError(ctx.port);
|
|
9895
|
+
}
|
|
9896
|
+
if (initialHealth.healthy) {
|
|
9897
|
+
return {
|
|
9898
|
+
port: ctx.port,
|
|
9899
|
+
process: null,
|
|
9900
|
+
version: null,
|
|
9901
|
+
notReadyReason: null,
|
|
9902
|
+
password: null
|
|
9903
|
+
};
|
|
9904
|
+
}
|
|
9905
|
+
if (!isOpenCode2Installed()) {
|
|
9906
|
+
throw new Error(
|
|
9907
|
+
"OpenCode V2 (opencode2) is not installed. Install it with: npm install -g @opencode-ai/cli@beta"
|
|
9908
|
+
);
|
|
9909
|
+
}
|
|
9910
|
+
let port = ctx.port;
|
|
9911
|
+
if (!ctx.interactive) {
|
|
9912
|
+
checkNonInteractivePortConflict(port, isPortInUse);
|
|
9913
|
+
} else if (isPortInUse(port)) {
|
|
9914
|
+
console.log(chalk6.yellow(`
|
|
9915
|
+
Port ${port} is already in use.`));
|
|
9916
|
+
const alternativePort = findAvailablePort(port + 1);
|
|
9917
|
+
if (alternativePort) {
|
|
9918
|
+
const useAlternative = await select3({
|
|
9919
|
+
message: `Use port ${alternativePort} instead?`,
|
|
9920
|
+
choices: [
|
|
9921
|
+
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
9922
|
+
{ name: "No, I will free the port manually", value: "no" }
|
|
9923
|
+
]
|
|
9924
|
+
});
|
|
9925
|
+
if (useAlternative === "yes") {
|
|
9926
|
+
port = alternativePort;
|
|
9927
|
+
} else {
|
|
9928
|
+
throw new Error(`Port ${ctx.port} is in use`);
|
|
9929
|
+
}
|
|
9930
|
+
}
|
|
9931
|
+
}
|
|
9932
|
+
if (!ctx.interactive) {
|
|
9933
|
+
throw v2SessionSupportIncompleteError();
|
|
9934
|
+
}
|
|
9935
|
+
console.log(chalk6.yellow(`
|
|
9936
|
+
${v2SessionSupportIncompleteError().message}`));
|
|
9937
|
+
const action = await select3({
|
|
9938
|
+
message: "OpenCode V2 is not running. What would you like to do?",
|
|
9939
|
+
choices: [
|
|
9940
|
+
{
|
|
9941
|
+
name: "Show me the command",
|
|
9942
|
+
value: "manual",
|
|
9943
|
+
description: "Display the command to run manually"
|
|
9944
|
+
},
|
|
9945
|
+
{
|
|
9946
|
+
name: "Continue without OpenCode V2",
|
|
9947
|
+
value: "continue",
|
|
9948
|
+
description: "Requests will fail until OpenCode V2 starts"
|
|
9949
|
+
}
|
|
9950
|
+
]
|
|
9951
|
+
});
|
|
9952
|
+
if (action === "manual") {
|
|
9953
|
+
blank();
|
|
9954
|
+
console.log(chalk6.bold("Run this command in another terminal:"));
|
|
9955
|
+
blank();
|
|
9956
|
+
console.log(` ${chalk6.cyan(`opencode2 serve --port ${port}`)}`);
|
|
9957
|
+
blank();
|
|
9958
|
+
throw new Error("Please start OpenCode V2 manually");
|
|
9959
|
+
}
|
|
9960
|
+
return {
|
|
9961
|
+
port,
|
|
9962
|
+
process: null,
|
|
9963
|
+
version: null,
|
|
9964
|
+
notReadyReason: "you chose to continue without OpenCode V2",
|
|
9965
|
+
password: null
|
|
9966
|
+
};
|
|
9967
|
+
}
|
|
9968
|
+
|
|
9969
|
+
// src/lib/runner-credentials.ts
|
|
9970
|
+
import { chmodSync as chmodSync2, writeFileSync as writeFileSync4 } from "node:fs";
|
|
9971
|
+
import { spawn as spawn5 } from "node:child_process";
|
|
9972
|
+
var RUNNER_SECRET_FETCH_TIMEOUT_MS = 6e4;
|
|
9973
|
+
var CREDENTIAL_RESTORE_TIMEOUT_MS = 6e4;
|
|
9974
|
+
var GITHUB_PROBE_TIMEOUT_MS = 1e4;
|
|
9975
|
+
var GIT_CONFIG_GLOBAL = "/tmp/gitconfig";
|
|
9976
|
+
var GIT_CREDENTIAL_HELPER = "/tmp/git-credential-helper.sh";
|
|
9977
|
+
function commandError2(result) {
|
|
9978
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
9979
|
+
}
|
|
9980
|
+
var runCommand2 = (command, args, opts) => {
|
|
9981
|
+
return new Promise((resolve4) => {
|
|
9982
|
+
let child;
|
|
9983
|
+
let stdout = "";
|
|
9984
|
+
let stderr = "";
|
|
9985
|
+
let settled = false;
|
|
9986
|
+
const timer = {};
|
|
9987
|
+
const finish = (result) => {
|
|
9988
|
+
if (settled) return;
|
|
9989
|
+
settled = true;
|
|
9990
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
9991
|
+
resolve4(result);
|
|
9992
|
+
};
|
|
9993
|
+
try {
|
|
9994
|
+
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
9995
|
+
} catch (error2) {
|
|
9996
|
+
finish({
|
|
9997
|
+
code: null,
|
|
9998
|
+
stdout,
|
|
9999
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
10000
|
+
timedOut: false
|
|
10001
|
+
});
|
|
10002
|
+
return;
|
|
10003
|
+
}
|
|
10004
|
+
child.stdout?.setEncoding("utf8");
|
|
10005
|
+
child.stdout?.on("data", (chunk) => {
|
|
10006
|
+
stdout += chunk;
|
|
10007
|
+
});
|
|
10008
|
+
child.stderr?.setEncoding("utf8");
|
|
10009
|
+
child.stderr?.on("data", (chunk) => {
|
|
10010
|
+
stderr += chunk;
|
|
10011
|
+
});
|
|
10012
|
+
child.once("error", (error2) => {
|
|
10013
|
+
finish({
|
|
10014
|
+
code: null,
|
|
10015
|
+
stdout,
|
|
10016
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
10017
|
+
${error2.message}`,
|
|
10018
|
+
timedOut: false
|
|
10019
|
+
});
|
|
10020
|
+
});
|
|
10021
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
10022
|
+
timer.handle = setTimeout(
|
|
10023
|
+
() => {
|
|
10024
|
+
child.kill("SIGKILL");
|
|
10025
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
10026
|
+
},
|
|
10027
|
+
Math.max(0, opts.timeoutMs)
|
|
10028
|
+
);
|
|
10029
|
+
});
|
|
10030
|
+
};
|
|
10031
|
+
function isEnvironmentObject(value) {
|
|
10032
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10033
|
+
}
|
|
10034
|
+
function secretFailure(marker, detail, log3) {
|
|
10035
|
+
const message = `${marker}: ${detail}`;
|
|
10036
|
+
log3(message, "error");
|
|
10037
|
+
return new Error(message);
|
|
10038
|
+
}
|
|
10039
|
+
async function installRunnerSecret({
|
|
10040
|
+
env,
|
|
10041
|
+
log: log3,
|
|
10042
|
+
commandRunner
|
|
10043
|
+
}) {
|
|
10044
|
+
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
10045
|
+
if (!arn) {
|
|
10046
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
10047
|
+
return false;
|
|
10048
|
+
}
|
|
10049
|
+
const result = await (commandRunner ?? runCommand2)(
|
|
10050
|
+
"aws",
|
|
10051
|
+
[
|
|
10052
|
+
"secretsmanager",
|
|
10053
|
+
"get-secret-value",
|
|
10054
|
+
"--secret-id",
|
|
10055
|
+
arn,
|
|
10056
|
+
"--query",
|
|
10057
|
+
"SecretString",
|
|
10058
|
+
"--output",
|
|
10059
|
+
"text"
|
|
10060
|
+
],
|
|
10061
|
+
{ env, timeoutMs: RUNNER_SECRET_FETCH_TIMEOUT_MS }
|
|
10062
|
+
);
|
|
10063
|
+
if (result.timedOut) {
|
|
10064
|
+
throw secretFailure(
|
|
10065
|
+
"CREDENTIAL-RESTORE-TIMEOUT",
|
|
10066
|
+
`runner-secret did not finish within ${RUNNER_SECRET_FETCH_TIMEOUT_MS}ms`,
|
|
10067
|
+
log3
|
|
10068
|
+
);
|
|
10069
|
+
}
|
|
10070
|
+
if (result.code !== 0) {
|
|
10071
|
+
throw secretFailure("RUNNER-SECRET-UNREADABLE", commandError2(result), log3);
|
|
10072
|
+
}
|
|
10073
|
+
let payload;
|
|
10074
|
+
try {
|
|
10075
|
+
payload = JSON.parse(result.stdout);
|
|
10076
|
+
} catch (error2) {
|
|
10077
|
+
log3(
|
|
10078
|
+
`RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
10079
|
+
"warn"
|
|
10080
|
+
);
|
|
10081
|
+
return false;
|
|
10082
|
+
}
|
|
10083
|
+
if (!isEnvironmentObject(payload)) {
|
|
10084
|
+
log3("RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object", "warn");
|
|
10085
|
+
return false;
|
|
10086
|
+
}
|
|
10087
|
+
let populated = 0;
|
|
10088
|
+
let skipped = 0;
|
|
10089
|
+
let githubTokenPopulated = false;
|
|
10090
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
10091
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
10092
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
10093
|
+
log3(
|
|
10094
|
+
`RUNNER-SECRET-KEY-SKIPPED: ${JSON.stringify(key)} is not a valid environment variable name`,
|
|
10095
|
+
"warn"
|
|
10096
|
+
);
|
|
10097
|
+
skipped += 1;
|
|
10098
|
+
continue;
|
|
10099
|
+
}
|
|
10100
|
+
env[key] = value;
|
|
10101
|
+
populated += 1;
|
|
10102
|
+
if (key === "GH_TOKEN") githubTokenPopulated = true;
|
|
10103
|
+
}
|
|
10104
|
+
if (populated === 0) {
|
|
10105
|
+
log3(
|
|
10106
|
+
"RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md",
|
|
10107
|
+
"warn"
|
|
10108
|
+
);
|
|
10109
|
+
} else {
|
|
10110
|
+
log3(
|
|
10111
|
+
`RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names`
|
|
10112
|
+
);
|
|
10113
|
+
}
|
|
10114
|
+
return githubTokenPopulated;
|
|
10115
|
+
}
|
|
10116
|
+
function restoreFailure(operation, result, log3) {
|
|
10117
|
+
const message = `CREDENTIAL-RESTORE-FAILED: ${operation} failed: ${commandError2(result)}`;
|
|
10118
|
+
log3(message, "error");
|
|
10119
|
+
return new Error(message);
|
|
10120
|
+
}
|
|
10121
|
+
async function runCredentialRestore(args, operation, log3, synchroniserRunner) {
|
|
10122
|
+
const result = await synchroniserRunner(args, { timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS });
|
|
10123
|
+
if (result.timedOut) {
|
|
10124
|
+
log3(
|
|
10125
|
+
`CREDENTIAL-RESTORE-TIMEOUT: ${operation} did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
10126
|
+
"warn"
|
|
10127
|
+
);
|
|
10128
|
+
return result;
|
|
10129
|
+
}
|
|
10130
|
+
if (result.code !== 0) throw restoreFailure(operation, result, log3);
|
|
10131
|
+
return result;
|
|
10132
|
+
}
|
|
10133
|
+
async function restoreCredentialStores({
|
|
10134
|
+
env,
|
|
10135
|
+
log: log3,
|
|
10136
|
+
synchroniserRunner = runSynchroniser
|
|
10137
|
+
}) {
|
|
10138
|
+
await runCredentialRestore(["restore", "claude"], "restore-claude", log3, synchroniserRunner);
|
|
10139
|
+
await runCredentialRestore(["restore", "opencode"], "restore-opencode", log3, synchroniserRunner);
|
|
10140
|
+
const result = await synchroniserRunner(["model-auth-ready"], {
|
|
10141
|
+
timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS
|
|
10142
|
+
});
|
|
10143
|
+
if (result.timedOut) {
|
|
10144
|
+
log3(
|
|
10145
|
+
`CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
10146
|
+
"warn"
|
|
10147
|
+
);
|
|
10148
|
+
return;
|
|
10149
|
+
}
|
|
10150
|
+
switch (result.code) {
|
|
10151
|
+
case 0:
|
|
10152
|
+
return;
|
|
10153
|
+
case 10:
|
|
10154
|
+
log3(
|
|
10155
|
+
`no model credentials under s3://${env.LITESTREAM_BUCKET ?? ""}/${env.LITESTREAM_PREFIX ?? ""}/ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs a model provider fails until one is connected. See 'Seeding a credential store' in infrastructure/evident-runner/MICROVM.md.`,
|
|
10156
|
+
"warn"
|
|
10157
|
+
);
|
|
10158
|
+
return;
|
|
10159
|
+
default:
|
|
10160
|
+
log3("could not determine whether this VM has model credentials", "warn");
|
|
10161
|
+
}
|
|
10162
|
+
}
|
|
10163
|
+
var GIT_CREDENTIAL_HELPER_CONTENT = [
|
|
10164
|
+
"#!/usr/bin/env bash",
|
|
10165
|
+
'[ "$1" = get ] || exit 0',
|
|
10166
|
+
"echo username=x-access-token",
|
|
10167
|
+
'echo "password=${GH_TOKEN}"',
|
|
10168
|
+
""
|
|
10169
|
+
].join("\n");
|
|
10170
|
+
async function probeGitHubAccess({ env, log: log3 }) {
|
|
10171
|
+
const auth = await runCommand2("gh", ["api", "user", "--jq", ".login"], {
|
|
10172
|
+
env,
|
|
10173
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
10174
|
+
});
|
|
10175
|
+
if (auth.timedOut) {
|
|
10176
|
+
log3(`GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`, "warn");
|
|
10177
|
+
return;
|
|
10178
|
+
}
|
|
10179
|
+
if (auth.code !== 0) {
|
|
10180
|
+
log3(`GITHUB-AUTH-REJECTED: ${commandError2(auth)}`, "warn");
|
|
10181
|
+
return;
|
|
10182
|
+
}
|
|
10183
|
+
log3(`GITHUB-AUTH-OK: ${auth.stdout.trim()}`, "debug");
|
|
10184
|
+
const remote = await runCommand2("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
|
10185
|
+
env,
|
|
10186
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
10187
|
+
});
|
|
10188
|
+
if (remote.code !== 0 || remote.timedOut) return;
|
|
10189
|
+
const repo = remote.stdout.trim().replace(/^(?:https:\/\/github\.com\/|git@github\.com:)/, "").replace(/\.git$/, "");
|
|
10190
|
+
if (!repo) return;
|
|
10191
|
+
const repository = await runCommand2("gh", ["api", `repos/${repo}`, "--jq", ".full_name"], {
|
|
10192
|
+
env,
|
|
10193
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
10194
|
+
});
|
|
10195
|
+
if (repository.timedOut) {
|
|
10196
|
+
log3(
|
|
10197
|
+
`GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`,
|
|
10198
|
+
"warn"
|
|
10199
|
+
);
|
|
10200
|
+
} else if (repository.code !== 0) {
|
|
10201
|
+
log3(`GITHUB-REPO-INACCESSIBLE: ${repo}: ${commandError2(repository)}`, "warn");
|
|
10202
|
+
}
|
|
10203
|
+
}
|
|
10204
|
+
async function configureGitHubAccess({ env, log: log3 }) {
|
|
10205
|
+
if (!env.GH_TOKEN) {
|
|
10206
|
+
log3("GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above", "warn");
|
|
10207
|
+
return;
|
|
10208
|
+
}
|
|
10209
|
+
try {
|
|
10210
|
+
env.GIT_CONFIG_GLOBAL = GIT_CONFIG_GLOBAL;
|
|
10211
|
+
writeFileSync4(GIT_CONFIG_GLOBAL, "");
|
|
10212
|
+
writeFileSync4(GIT_CREDENTIAL_HELPER, GIT_CREDENTIAL_HELPER_CONTENT, { mode: 448 });
|
|
10213
|
+
chmodSync2(GIT_CREDENTIAL_HELPER, 448);
|
|
10214
|
+
const config = [
|
|
10215
|
+
["user.name", env.GIT_USER_NAME ?? "evident-bot"],
|
|
10216
|
+
["user.email", env.GIT_USER_EMAIL ?? "evident-bot@users.noreply.github.com"],
|
|
10217
|
+
["init.defaultBranch", "main"],
|
|
10218
|
+
["credential.https://github.com.helper", GIT_CREDENTIAL_HELPER]
|
|
10219
|
+
];
|
|
10220
|
+
for (const [key, value] of config) {
|
|
10221
|
+
const result = await runCommand2("git", ["config", "--global", key, value], {
|
|
10222
|
+
env,
|
|
10223
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
10224
|
+
});
|
|
10225
|
+
if (result.timedOut || result.code !== 0) throw new Error(commandError2(result));
|
|
10226
|
+
}
|
|
10227
|
+
} catch (error2) {
|
|
10228
|
+
log3(
|
|
10229
|
+
`GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
10230
|
+
"warn"
|
|
10231
|
+
);
|
|
10232
|
+
return;
|
|
10233
|
+
}
|
|
10234
|
+
void probeGitHubAccess({ env, log: log3 }).catch((error2) => {
|
|
10235
|
+
log3(
|
|
10236
|
+
`GITHUB-SETUP-FAILED: GitHub access probe failed: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10237
|
+
"warn"
|
|
10238
|
+
);
|
|
10239
|
+
});
|
|
10240
|
+
}
|
|
10241
|
+
|
|
10242
|
+
// src/lib/opencode/config-overlay.ts
|
|
10243
|
+
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
10244
|
+
import { copyFileSync, existsSync as existsSync2, statSync as statSync6 } from "node:fs";
|
|
10245
|
+
import { isAbsolute as isAbsolute2, join as join9, resolve as resolve3 } from "node:path";
|
|
10246
|
+
function isFile(filePath) {
|
|
10247
|
+
return existsSync2(filePath) && statSync6(filePath).isFile();
|
|
10248
|
+
}
|
|
10249
|
+
function applyRunnerOpenCodeConfig({
|
|
10250
|
+
overlayPath,
|
|
10251
|
+
cwd = process.cwd(),
|
|
10252
|
+
log: log3
|
|
10253
|
+
}) {
|
|
10254
|
+
if (!overlayPath) {
|
|
10255
|
+
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
10256
|
+
return;
|
|
10257
|
+
}
|
|
10258
|
+
const source = isAbsolute2(overlayPath) ? overlayPath : resolve3(cwd, overlayPath);
|
|
10259
|
+
const target = isFile(join9(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
10260
|
+
if (!isFile(source)) {
|
|
10261
|
+
log3(
|
|
10262
|
+
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
10263
|
+
"error"
|
|
10264
|
+
);
|
|
10265
|
+
return;
|
|
10266
|
+
}
|
|
10267
|
+
copyFileSync(source, join9(cwd, target));
|
|
10268
|
+
try {
|
|
10269
|
+
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
10270
|
+
stdio: "ignore"
|
|
10271
|
+
});
|
|
10272
|
+
} catch (error2) {
|
|
10273
|
+
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
10274
|
+
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
10275
|
+
}
|
|
10276
|
+
log3(`Applied runner OpenCode config ${source} to ${join9(cwd, target)}`);
|
|
10277
|
+
}
|
|
10278
|
+
|
|
10279
|
+
// src/lib/credential-sync.ts
|
|
10280
|
+
import { renameSync, writeFileSync as writeFileSync5 } from "node:fs";
|
|
10281
|
+
var CREDENTIAL_SYNC_TICK_TIMEOUT_MS = 3e4;
|
|
10282
|
+
var CREDENTIAL_FLUSH_DEADLINE_MS = 8e3;
|
|
10283
|
+
var CREDENTIAL_FLUSH_ABORT_GRACE_MS = 500;
|
|
10284
|
+
var DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS = 60;
|
|
10285
|
+
var STORES = ["claude", "opencode"];
|
|
10286
|
+
var MAX_FLUSH_PASSES = 2;
|
|
10287
|
+
function outcomesWith(outcome) {
|
|
10288
|
+
return { claude: outcome, opencode: outcome };
|
|
10289
|
+
}
|
|
10290
|
+
function errorMessage2(error2) {
|
|
10291
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
10292
|
+
}
|
|
10293
|
+
function waitForSettlement(promise, timeoutMs) {
|
|
10294
|
+
return new Promise((resolve4) => {
|
|
10295
|
+
let settled = false;
|
|
10296
|
+
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
10297
|
+
const finish = (value) => {
|
|
10298
|
+
if (settled) return;
|
|
10299
|
+
settled = true;
|
|
10300
|
+
clearTimeout(timer);
|
|
10301
|
+
resolve4(value);
|
|
10302
|
+
};
|
|
10303
|
+
promise.then(
|
|
10304
|
+
() => finish(true),
|
|
10305
|
+
() => finish(true)
|
|
10306
|
+
);
|
|
10307
|
+
});
|
|
10308
|
+
}
|
|
10309
|
+
function writeMarker(markerPath, outcomes, log3) {
|
|
10310
|
+
const body = `${STORES.map((store) => `${store}=${outcomes[store]}`).join("\n")}
|
|
10311
|
+
`;
|
|
10312
|
+
const temporaryPath = `${markerPath}.tmp`;
|
|
10313
|
+
try {
|
|
10314
|
+
writeFileSync5(temporaryPath, body, { mode: 384 });
|
|
10315
|
+
renameSync(temporaryPath, markerPath);
|
|
10316
|
+
} catch (error2) {
|
|
10317
|
+
log3(`CREDENTIAL-FLUSH-MARKER-UNWRITABLE: ${markerPath}: ${errorMessage2(error2)}`, "warn");
|
|
10318
|
+
}
|
|
10319
|
+
}
|
|
10320
|
+
function intervalSeconds(env, log3) {
|
|
10321
|
+
const raw = env.CREDS_SYNC_INTERVAL;
|
|
10322
|
+
if (raw === void 0 || /^[1-9][0-9]*$/.test(raw) && Number.isSafeInteger(Number(raw))) {
|
|
10323
|
+
return raw === void 0 ? DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS : Number(raw);
|
|
10324
|
+
}
|
|
10325
|
+
log3(
|
|
10326
|
+
`CREDENTIAL-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${raw}' is not a positive integer; using ${DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS}s`,
|
|
10327
|
+
"warn"
|
|
10328
|
+
);
|
|
10329
|
+
return DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS;
|
|
10330
|
+
}
|
|
10331
|
+
async function runFlushStore(store, env, synchroniserRunner, log3, deadlineAt) {
|
|
10332
|
+
const remainingMs = deadlineAt - Date.now();
|
|
10333
|
+
if (remainingMs <= 0) return { outcome: "timeout", orphaned: false };
|
|
10334
|
+
const controller = new AbortController();
|
|
10335
|
+
let result;
|
|
10336
|
+
let failed = false;
|
|
10337
|
+
const completion = Promise.resolve().then(
|
|
10338
|
+
() => synchroniserRunner(["sync-once", store], {
|
|
10339
|
+
timeoutMs: remainingMs,
|
|
10340
|
+
env,
|
|
10341
|
+
signal: controller.signal
|
|
10342
|
+
})
|
|
10343
|
+
).then(
|
|
10344
|
+
(value) => {
|
|
10345
|
+
result = value;
|
|
10346
|
+
},
|
|
10347
|
+
(error2) => {
|
|
10348
|
+
failed = true;
|
|
10349
|
+
log3(`CREDENTIAL-FLUSH-ERROR: ${store}: ${errorMessage2(error2)}`, "warn");
|
|
10350
|
+
}
|
|
10351
|
+
);
|
|
10352
|
+
const abortTimer = setTimeout(() => controller.abort(), remainingMs);
|
|
10353
|
+
const settledBeforeDeadline = await waitForSettlement(completion, remainingMs);
|
|
10354
|
+
clearTimeout(abortTimer);
|
|
10355
|
+
if (!settledBeforeDeadline) {
|
|
10356
|
+
controller.abort();
|
|
10357
|
+
const settledAfterAbort = await waitForSettlement(completion, CREDENTIAL_FLUSH_ABORT_GRACE_MS);
|
|
10358
|
+
if (!settledAfterAbort) return { outcome: "timeout", orphaned: true };
|
|
10359
|
+
return { outcome: "timeout", orphaned: false };
|
|
10360
|
+
}
|
|
10361
|
+
if (failed || !result) return { outcome: "failed", orphaned: false };
|
|
10362
|
+
if (result.timedOut || Date.now() >= deadlineAt) {
|
|
10363
|
+
return { outcome: "timeout", orphaned: false };
|
|
10364
|
+
}
|
|
10365
|
+
return { outcome: result.code === 0 ? "ok" : "failed", orphaned: false };
|
|
10366
|
+
}
|
|
10367
|
+
function createCredentialSync({
|
|
10368
|
+
markerPath,
|
|
10369
|
+
env,
|
|
10370
|
+
log: log3,
|
|
10371
|
+
synchroniserRunner = runSynchroniser
|
|
10372
|
+
}) {
|
|
10373
|
+
const persistenceDisabled = !env.PERSISTENCE_BUCKET;
|
|
10374
|
+
let disabled = persistenceDisabled;
|
|
10375
|
+
let armed = false;
|
|
10376
|
+
let stopped = false;
|
|
10377
|
+
let timer;
|
|
10378
|
+
let inFlight;
|
|
10379
|
+
let activeTickAbort;
|
|
10380
|
+
let lastTickFailed;
|
|
10381
|
+
let flushPromise;
|
|
10382
|
+
const scheduleTick = (intervalMs, startTick2) => {
|
|
10383
|
+
if (stopped) return;
|
|
10384
|
+
timer = setTimeout(() => {
|
|
10385
|
+
timer = void 0;
|
|
10386
|
+
startTick2();
|
|
10387
|
+
}, intervalMs);
|
|
10388
|
+
};
|
|
10389
|
+
const startTick = (intervalMs) => {
|
|
10390
|
+
if (stopped) return;
|
|
10391
|
+
const controller = new AbortController();
|
|
10392
|
+
activeTickAbort = controller;
|
|
10393
|
+
const tick = (async () => {
|
|
10394
|
+
const outcomes = {
|
|
10395
|
+
claude: "failed",
|
|
10396
|
+
opencode: "failed"
|
|
10397
|
+
};
|
|
10398
|
+
for (const store of STORES) {
|
|
10399
|
+
if (controller.signal.aborted) break;
|
|
10400
|
+
try {
|
|
10401
|
+
const result = await synchroniserRunner(["sync-once", store], {
|
|
10402
|
+
timeoutMs: CREDENTIAL_SYNC_TICK_TIMEOUT_MS,
|
|
10403
|
+
env,
|
|
10404
|
+
signal: controller.signal
|
|
10405
|
+
});
|
|
10406
|
+
outcomes[store] = !result.timedOut && result.code === 0 ? "ok" : "failed";
|
|
10407
|
+
} catch (error2) {
|
|
10408
|
+
outcomes[store] = "failed";
|
|
10409
|
+
log3(`CREDENTIAL-SYNC-TICK-ERROR: ${store}: ${errorMessage2(error2)}`, "debug");
|
|
10410
|
+
}
|
|
10411
|
+
}
|
|
10412
|
+
const failed = STORES.some((store) => outcomes[store] === "failed");
|
|
10413
|
+
log3(
|
|
10414
|
+
`CREDENTIAL-SYNC-TICK: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
10415
|
+
"debug"
|
|
10416
|
+
);
|
|
10417
|
+
if (failed && lastTickFailed !== true) {
|
|
10418
|
+
log3(
|
|
10419
|
+
"CREDENTIAL-SYNC-FAILED: an interval credential sync failed; retrying on the next tick",
|
|
10420
|
+
"warn"
|
|
10421
|
+
);
|
|
10422
|
+
} else if (!failed && lastTickFailed === true) {
|
|
10423
|
+
log3("CREDENTIAL-SYNC-RECOVERED: interval credential sync succeeded again", "warn");
|
|
10424
|
+
}
|
|
10425
|
+
lastTickFailed = failed;
|
|
10426
|
+
})().finally(() => {
|
|
10427
|
+
if (activeTickAbort === controller) activeTickAbort = void 0;
|
|
10428
|
+
if (inFlight === tick) inFlight = void 0;
|
|
10429
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
10430
|
+
});
|
|
10431
|
+
inFlight = tick;
|
|
10432
|
+
};
|
|
10433
|
+
const performFlush = async () => {
|
|
10434
|
+
stopped = true;
|
|
10435
|
+
if (timer) {
|
|
10436
|
+
clearTimeout(timer);
|
|
10437
|
+
timer = void 0;
|
|
10438
|
+
}
|
|
10439
|
+
const deadlineAt = Date.now() + CREDENTIAL_FLUSH_DEADLINE_MS;
|
|
10440
|
+
if (inFlight) {
|
|
10441
|
+
const settled = await waitForSettlement(inFlight, CREDENTIAL_FLUSH_DEADLINE_MS);
|
|
10442
|
+
if (!settled) {
|
|
10443
|
+
activeTickAbort?.abort();
|
|
10444
|
+
const settledAfterAbort = await waitForSettlement(
|
|
10445
|
+
inFlight,
|
|
10446
|
+
CREDENTIAL_FLUSH_ABORT_GRACE_MS
|
|
10447
|
+
);
|
|
10448
|
+
if (!settledAfterAbort) {
|
|
10449
|
+
log3(
|
|
10450
|
+
"CREDENTIAL-FLUSH-ORPHANED-TICK: a sync-once child could not be proven settled; skipping the boundary flush rather than becoming a second writer",
|
|
10451
|
+
"warn"
|
|
10452
|
+
);
|
|
10453
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
10454
|
+
}
|
|
8044
10455
|
}
|
|
8045
10456
|
}
|
|
8046
|
-
|
|
8047
|
-
|
|
8048
|
-
|
|
8049
|
-
|
|
8050
|
-
{
|
|
8051
|
-
|
|
8052
|
-
|
|
8053
|
-
|
|
8054
|
-
|
|
8055
|
-
|
|
8056
|
-
name: "Show me the command",
|
|
8057
|
-
value: "manual",
|
|
8058
|
-
description: "Display the command to run manually"
|
|
8059
|
-
},
|
|
8060
|
-
{
|
|
8061
|
-
name: "Continue without OpenCode",
|
|
8062
|
-
value: "continue",
|
|
8063
|
-
description: "Requests will fail until OpenCode starts"
|
|
10457
|
+
if (disabled) return { outcomes: outcomesWith("skipped"), orphaned: false };
|
|
10458
|
+
const outcomes = outcomesWith("timeout");
|
|
10459
|
+
for (const store of STORES) {
|
|
10460
|
+
const result = await runFlushStore(store, env, synchroniserRunner, log3, deadlineAt);
|
|
10461
|
+
if (result.orphaned) {
|
|
10462
|
+
log3(
|
|
10463
|
+
"CREDENTIAL-FLUSH-ORPHANED-CHILD: a flush sync-once child could not be proven settled; refusing another flush pass rather than becoming a second writer",
|
|
10464
|
+
"warn"
|
|
10465
|
+
);
|
|
10466
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
8064
10467
|
}
|
|
8065
|
-
|
|
8066
|
-
});
|
|
8067
|
-
if (action === "manual") {
|
|
8068
|
-
blank();
|
|
8069
|
-
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
8070
|
-
blank();
|
|
8071
|
-
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
8072
|
-
blank();
|
|
8073
|
-
throw new Error("Please start OpenCode manually");
|
|
8074
|
-
}
|
|
8075
|
-
if (action === "start") {
|
|
8076
|
-
const spinner = ora2("Starting OpenCode...").start();
|
|
8077
|
-
const proc = await startOpenCode(port);
|
|
8078
|
-
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
8079
|
-
if (!health.healthy) {
|
|
8080
|
-
spinner.fail("Failed to start OpenCode");
|
|
8081
|
-
throw new Error("OpenCode failed to start");
|
|
10468
|
+
outcomes[store] = result.outcome;
|
|
8082
10469
|
}
|
|
8083
|
-
|
|
8084
|
-
|
|
8085
|
-
|
|
8086
|
-
|
|
10470
|
+
return { outcomes, orphaned: false };
|
|
10471
|
+
};
|
|
10472
|
+
let flushPasses = 0;
|
|
10473
|
+
let lastFlush;
|
|
10474
|
+
return {
|
|
10475
|
+
arm() {
|
|
10476
|
+
if (stopped || armed) return;
|
|
10477
|
+
armed = true;
|
|
10478
|
+
if (persistenceDisabled) {
|
|
10479
|
+
disabled = true;
|
|
10480
|
+
log3(
|
|
10481
|
+
"CREDENTIAL-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot",
|
|
10482
|
+
"warn"
|
|
10483
|
+
);
|
|
10484
|
+
return;
|
|
10485
|
+
}
|
|
10486
|
+
disabled = false;
|
|
10487
|
+
const intervalMs = intervalSeconds(env, log3) * 1e3;
|
|
10488
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
10489
|
+
},
|
|
10490
|
+
async stopAndFlush(publish) {
|
|
10491
|
+
let result;
|
|
10492
|
+
const runningFlush = flushPromise;
|
|
10493
|
+
if (runningFlush) {
|
|
10494
|
+
result = await runningFlush;
|
|
10495
|
+
} else if (flushPasses >= MAX_FLUSH_PASSES || lastFlush?.orphaned) {
|
|
10496
|
+
result = lastFlush ?? { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
10497
|
+
} else {
|
|
10498
|
+
flushPasses++;
|
|
10499
|
+
const currentFlush = performFlush();
|
|
10500
|
+
flushPromise = currentFlush;
|
|
10501
|
+
try {
|
|
10502
|
+
result = await currentFlush;
|
|
10503
|
+
lastFlush = result;
|
|
10504
|
+
} finally {
|
|
10505
|
+
if (flushPromise === currentFlush) flushPromise = void 0;
|
|
10506
|
+
}
|
|
10507
|
+
}
|
|
10508
|
+
if (publish) writeMarker(markerPath, result.outcomes, log3);
|
|
10509
|
+
return result.outcomes;
|
|
10510
|
+
}
|
|
10511
|
+
};
|
|
8087
10512
|
}
|
|
8088
10513
|
|
|
8089
10514
|
// src/commands/run.ts
|
|
@@ -8123,11 +10548,11 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
8123
10548
|
if (trimmed === "") {
|
|
8124
10549
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
8125
10550
|
}
|
|
8126
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
8127
|
-
if (!
|
|
10551
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join10(homeDir, trimmed.slice(2)) : trimmed;
|
|
10552
|
+
if (!isAbsolute3(expanded)) {
|
|
8128
10553
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
8129
10554
|
}
|
|
8130
|
-
const normalized =
|
|
10555
|
+
const normalized = resolvePath2(expanded);
|
|
8131
10556
|
if (parse(normalized).root === normalized) {
|
|
8132
10557
|
throw new Error(
|
|
8133
10558
|
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
@@ -8147,6 +10572,28 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
8147
10572
|
var DEFAULT_OPENCODE_START_TIMEOUT_SECONDS = 180;
|
|
8148
10573
|
var MAX_OPENCODE_START_TIMEOUT_SECONDS = 3600;
|
|
8149
10574
|
var OPENCODE_START_TIMEOUT_ENV = "EVIDENT_OPENCODE_START_TIMEOUT";
|
|
10575
|
+
var OPENCODE_VERSION_ENV = "EVIDENT_OPENCODE_VERSION";
|
|
10576
|
+
function resolveOpenCodeVersion(options, env = process.env) {
|
|
10577
|
+
let raw;
|
|
10578
|
+
let source;
|
|
10579
|
+
if (options.opencodeVersion !== void 0) {
|
|
10580
|
+
raw = options.opencodeVersion;
|
|
10581
|
+
source = "--opencode-version";
|
|
10582
|
+
} else if (env[OPENCODE_VERSION_ENV] !== void 0 && env[OPENCODE_VERSION_ENV] !== "") {
|
|
10583
|
+
raw = env[OPENCODE_VERSION_ENV];
|
|
10584
|
+
source = OPENCODE_VERSION_ENV;
|
|
10585
|
+
} else {
|
|
10586
|
+
return { version: "v1", warnings: [] };
|
|
10587
|
+
}
|
|
10588
|
+
const normalized = raw.trim().toLowerCase();
|
|
10589
|
+
if (normalized !== "v1" && normalized !== "v2") {
|
|
10590
|
+
return {
|
|
10591
|
+
version: "v1",
|
|
10592
|
+
warnings: [`Ignoring invalid ${source} "${raw}": expected v1 or v2; using the default v1`]
|
|
10593
|
+
};
|
|
10594
|
+
}
|
|
10595
|
+
return { version: normalized, warnings: [] };
|
|
10596
|
+
}
|
|
8150
10597
|
function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
8151
10598
|
const defaultMs = DEFAULT_OPENCODE_START_TIMEOUT_SECONDS * 1e3;
|
|
8152
10599
|
let raw;
|
|
@@ -8213,7 +10660,7 @@ function log2(state, message, level = "info") {
|
|
|
8213
10660
|
})
|
|
8214
10661
|
);
|
|
8215
10662
|
} else if (!state.interactive) {
|
|
8216
|
-
const prefix = level === "error" ?
|
|
10663
|
+
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
8217
10664
|
console.log(`${prefix} ${message}`);
|
|
8218
10665
|
}
|
|
8219
10666
|
}
|
|
@@ -8243,7 +10690,7 @@ function logActivity(state, entry) {
|
|
|
8243
10690
|
}
|
|
8244
10691
|
function reportSessionDbRecovery(state) {
|
|
8245
10692
|
try {
|
|
8246
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
10693
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir6(), env: process.env });
|
|
8247
10694
|
for (const record of report.records) {
|
|
8248
10695
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
8249
10696
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -8261,21 +10708,31 @@ function reportSessionDbRecovery(state) {
|
|
|
8261
10708
|
);
|
|
8262
10709
|
}
|
|
8263
10710
|
}
|
|
10711
|
+
function reportSessionDbRecoveryRecord(state, record) {
|
|
10712
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10713
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
10714
|
+
logActivity(state, {
|
|
10715
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10716
|
+
level: activity.level,
|
|
10717
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10718
|
+
metadata: activity.metadata
|
|
10719
|
+
});
|
|
10720
|
+
}
|
|
8264
10721
|
function displayStatus(state) {
|
|
8265
10722
|
if (!state.interactive) return;
|
|
8266
10723
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
8267
|
-
const tunnel = state.connected ?
|
|
8268
|
-
const opencode = state.opencodeConnected ?
|
|
8269
|
-
const messages = state.messageCount > 0 ?
|
|
10724
|
+
const tunnel = state.connected ? chalk7.green("tunnel: connected") : attempt > 0 ? chalk7.yellow(`tunnel: reconnecting (#${attempt})`) : chalk7.yellow("tunnel: connecting");
|
|
10725
|
+
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
10726
|
+
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
8270
10727
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
8271
|
-
const detail = last ?
|
|
10728
|
+
const detail = last ? chalk7.dim(` \xB7 ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`) : "";
|
|
8272
10729
|
const agent = state.agentName ?? state.agentId;
|
|
8273
10730
|
console.log(
|
|
8274
|
-
`${
|
|
10731
|
+
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
8275
10732
|
);
|
|
8276
10733
|
}
|
|
8277
10734
|
async function promptForLogin(promptMessage, successMessage) {
|
|
8278
|
-
const action = await
|
|
10735
|
+
const action = await select4({
|
|
8279
10736
|
message: promptMessage,
|
|
8280
10737
|
choices: [
|
|
8281
10738
|
{
|
|
@@ -8291,7 +10748,7 @@ async function promptForLogin(promptMessage, successMessage) {
|
|
|
8291
10748
|
]
|
|
8292
10749
|
});
|
|
8293
10750
|
if (action === "exit") {
|
|
8294
|
-
console.log(
|
|
10751
|
+
console.log(chalk7.dim(`
|
|
8295
10752
|
You can log in later by running: ${getCliName()} login`));
|
|
8296
10753
|
process.exit(0);
|
|
8297
10754
|
}
|
|
@@ -8302,7 +10759,7 @@ You can log in later by running: ${getCliName()} login`));
|
|
|
8302
10759
|
process.exit(1);
|
|
8303
10760
|
}
|
|
8304
10761
|
blank();
|
|
8305
|
-
console.log(
|
|
10762
|
+
console.log(chalk7.green(successMessage));
|
|
8306
10763
|
blank();
|
|
8307
10764
|
return { token: credentials2.token, authType: "bearer", user: credentials2.user };
|
|
8308
10765
|
}
|
|
@@ -8315,12 +10772,12 @@ async function handleAuthError(state, error2) {
|
|
|
8315
10772
|
if (state.interactive) displayStatus(state);
|
|
8316
10773
|
if (!state.interactive) {
|
|
8317
10774
|
blank();
|
|
8318
|
-
console.log(
|
|
8319
|
-
console.log(
|
|
10775
|
+
console.log(chalk7.red("Authentication expired"));
|
|
10776
|
+
console.log(chalk7.dim("Your authentication token is no longer valid."));
|
|
8320
10777
|
blank();
|
|
8321
|
-
console.log(
|
|
8322
|
-
console.log(
|
|
8323
|
-
console.log(
|
|
10778
|
+
console.log(chalk7.dim("To fix this:"));
|
|
10779
|
+
console.log(chalk7.dim(` 1. Run '${getCliName()} login' to re-authenticate`));
|
|
10780
|
+
console.log(chalk7.dim(" 2. Restart this command"));
|
|
8324
10781
|
blank();
|
|
8325
10782
|
await cleanup(state);
|
|
8326
10783
|
await shutdownTelemetry();
|
|
@@ -8328,7 +10785,7 @@ async function handleAuthError(state, error2) {
|
|
|
8328
10785
|
return { success: false };
|
|
8329
10786
|
}
|
|
8330
10787
|
blank();
|
|
8331
|
-
console.log(
|
|
10788
|
+
console.log(chalk7.yellow("Your authentication has expired."));
|
|
8332
10789
|
blank();
|
|
8333
10790
|
try {
|
|
8334
10791
|
const credentials2 = await promptForLogin(
|
|
@@ -8373,6 +10830,10 @@ async function driveChannels(state, driver) {
|
|
|
8373
10830
|
consecutiveDrainFailures = 0;
|
|
8374
10831
|
unreachableMs = 0;
|
|
8375
10832
|
state.messageCount += processed;
|
|
10833
|
+
if (driver.recycleRequested) {
|
|
10834
|
+
await beginGracefulShutdown(state, "recycle");
|
|
10835
|
+
return;
|
|
10836
|
+
}
|
|
8376
10837
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
8377
10838
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
8378
10839
|
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
@@ -8388,6 +10849,14 @@ async function driveChannels(state, driver) {
|
|
|
8388
10849
|
const opencodeAuthApplied = opencodeAuthApplies !== lastSeenOpencodeAuthApplies;
|
|
8389
10850
|
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
8390
10851
|
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
10852
|
+
if (claudeCredentialApplied || opencodeAuthApplied) {
|
|
10853
|
+
void reloadProviderCache(state.port).catch(
|
|
10854
|
+
(error2) => logActivity(state, {
|
|
10855
|
+
type: "error",
|
|
10856
|
+
error: `OpenCode provider cache reload failed on port ${state.port}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10857
|
+
})
|
|
10858
|
+
);
|
|
10859
|
+
}
|
|
8391
10860
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
8392
10861
|
idlePolls = 0;
|
|
8393
10862
|
idleMs = 0;
|
|
@@ -8415,8 +10884,8 @@ async function driveChannels(state, driver) {
|
|
|
8415
10884
|
state.running = false;
|
|
8416
10885
|
break;
|
|
8417
10886
|
}
|
|
8418
|
-
const
|
|
8419
|
-
logActivity(state, { type: "error", error: `Channel processing error: ${
|
|
10887
|
+
const errorMessage3 = error2 instanceof Error ? error2.message : String(error2);
|
|
10888
|
+
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage3}` });
|
|
8420
10889
|
if (state.interactive) displayStatus(state);
|
|
8421
10890
|
if (driver.hasInFlightWatchers()) {
|
|
8422
10891
|
consecutiveDrainFailures = 0;
|
|
@@ -8433,7 +10902,7 @@ async function driveChannels(state, driver) {
|
|
|
8433
10902
|
}
|
|
8434
10903
|
}
|
|
8435
10904
|
}
|
|
8436
|
-
await new Promise((
|
|
10905
|
+
await new Promise((resolve4) => setTimeout(resolve4, CHANNEL_POLL_INTERVAL_MS));
|
|
8437
10906
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
8438
10907
|
if (idleThisCycle) idleMs += cycleMs;
|
|
8439
10908
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -8454,9 +10923,54 @@ async function driveChannels(state, driver) {
|
|
|
8454
10923
|
}
|
|
8455
10924
|
}
|
|
8456
10925
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
8457
|
-
var SESSION_DB_RECLAIM_MAX_PAGES =
|
|
10926
|
+
var SESSION_DB_RECLAIM_MAX_PAGES = 2e4;
|
|
10927
|
+
function shouldWarnForReclaimSkip(reason) {
|
|
10928
|
+
if (reason !== "sqlite-unavailable") return false;
|
|
10929
|
+
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
10930
|
+
if (!version2) return false;
|
|
10931
|
+
const major = Number(version2[1]);
|
|
10932
|
+
const minor = Number(version2[2]);
|
|
10933
|
+
const patch = Number(version2[3]);
|
|
10934
|
+
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
10935
|
+
}
|
|
8458
10936
|
function sessionDbPath() {
|
|
8459
|
-
return
|
|
10937
|
+
return join10(homedir6(), ".local", "share", "opencode", "opencode.db");
|
|
10938
|
+
}
|
|
10939
|
+
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
10940
|
+
const record = {
|
|
10941
|
+
v: 1,
|
|
10942
|
+
event: "session_db_recovery",
|
|
10943
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10944
|
+
stage: "verify",
|
|
10945
|
+
outcome: "schema_provenance_mismatch",
|
|
10946
|
+
severity: "error",
|
|
10947
|
+
reason: provenance.reason ?? "schema-provenance-mismatch",
|
|
10948
|
+
litestream_exit_code: null,
|
|
10949
|
+
attempt: null,
|
|
10950
|
+
replica_objects: null,
|
|
10951
|
+
replica_bytes: null,
|
|
10952
|
+
quarantine_destination: null,
|
|
10953
|
+
quarantined_objects: null,
|
|
10954
|
+
quarantine_failed_objects: null,
|
|
10955
|
+
quarantined_bytes: null,
|
|
10956
|
+
verified_restore_point: null,
|
|
10957
|
+
restore_points_tried: null,
|
|
10958
|
+
provenance_reason: provenance.reason,
|
|
10959
|
+
provenance_migration_delta: provenance.migrationDelta,
|
|
10960
|
+
replication_suspended: false,
|
|
10961
|
+
dbPath: sessionDbPath(),
|
|
10962
|
+
recorded_version: provenance.recordedVersion,
|
|
10963
|
+
current_version: currentVersion,
|
|
10964
|
+
provenance_pre_boot_migration_count: preBootMigrationCount
|
|
10965
|
+
};
|
|
10966
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10967
|
+
if (!activity) throw new Error("could not map session-DB provenance activity");
|
|
10968
|
+
logActivity(state, {
|
|
10969
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10970
|
+
level: activity.level,
|
|
10971
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10972
|
+
metadata: activity.metadata
|
|
10973
|
+
});
|
|
8460
10974
|
}
|
|
8461
10975
|
async function runSweep(state, driver, config) {
|
|
8462
10976
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -8503,7 +11017,7 @@ async function runSweep(state, driver, config) {
|
|
|
8503
11017
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
8504
11018
|
dbPath: sessionDbPath(),
|
|
8505
11019
|
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
8506
|
-
allowFullVacuum: protectedNow.size === 0
|
|
11020
|
+
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
8507
11021
|
});
|
|
8508
11022
|
if (reclaimResult.ok) {
|
|
8509
11023
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
@@ -8516,7 +11030,7 @@ async function runSweep(state, driver, config) {
|
|
|
8516
11030
|
} else {
|
|
8517
11031
|
logActivity(state, {
|
|
8518
11032
|
type: "info",
|
|
8519
|
-
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})`
|
|
11033
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : "")
|
|
8520
11034
|
});
|
|
8521
11035
|
}
|
|
8522
11036
|
} catch (error2) {
|
|
@@ -8539,13 +11053,20 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
8539
11053
|
for (const warning2 of config.warnings) {
|
|
8540
11054
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
8541
11055
|
}
|
|
8542
|
-
const dbBytes = statSessionDbBytes(
|
|
11056
|
+
const dbBytes = statSessionDbBytes(homedir6());
|
|
8543
11057
|
void (async () => {
|
|
8544
|
-
const
|
|
11058
|
+
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
11059
|
+
if (reclaimAvailability !== null) {
|
|
11060
|
+
logActivity(state, {
|
|
11061
|
+
type: "info",
|
|
11062
|
+
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
11063
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
11064
|
+
});
|
|
11065
|
+
}
|
|
8545
11066
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
8546
11067
|
dbBytes,
|
|
8547
11068
|
cleanupEnabled: config.enabled,
|
|
8548
|
-
reclaimSkipReason
|
|
11069
|
+
reclaimSkipReason: reclaimAvailability?.reason ?? null
|
|
8549
11070
|
});
|
|
8550
11071
|
if (sizeWarning !== null) {
|
|
8551
11072
|
logActivity(state, { type: "info", level: "warn", message: sizeWarning });
|
|
@@ -8740,7 +11261,8 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
8740
11261
|
});
|
|
8741
11262
|
return;
|
|
8742
11263
|
}
|
|
8743
|
-
const collect = createResourceUsageCollector(
|
|
11264
|
+
const { collect, stop } = createResourceUsageCollector(homedir6());
|
|
11265
|
+
state.stopResourceUsageSampling = stop;
|
|
8744
11266
|
let consecutiveFailures = 0;
|
|
8745
11267
|
const tick = async () => {
|
|
8746
11268
|
try {
|
|
@@ -8836,6 +11358,8 @@ async function cleanup(state, opts = {}) {
|
|
|
8836
11358
|
clearTimeout(timer);
|
|
8837
11359
|
}
|
|
8838
11360
|
state.sessionCleanupTimers = [];
|
|
11361
|
+
state.stopOpenCodeLogTail?.();
|
|
11362
|
+
state.stopOpenCodeLogTail = null;
|
|
8839
11363
|
if (state.claudeUsageTimer) {
|
|
8840
11364
|
clearTimeout(state.claudeUsageTimer);
|
|
8841
11365
|
state.claudeUsageTimer = null;
|
|
@@ -8850,21 +11374,41 @@ async function cleanup(state, opts = {}) {
|
|
|
8850
11374
|
clearTimeout(state.resourceUsageTimer);
|
|
8851
11375
|
state.resourceUsageTimer = null;
|
|
8852
11376
|
}
|
|
11377
|
+
state.stopResourceUsageSampling?.();
|
|
11378
|
+
state.stopResourceUsageSampling = null;
|
|
11379
|
+
const credentialSync = state.credentialSync;
|
|
11380
|
+
const flushCredentials = credentialSync ? async (phase, publish) => {
|
|
11381
|
+
await timeShutdownPhase(state, durations, phase, async () => {
|
|
11382
|
+
const outcomes = await credentialSync.stopAndFlush(publish);
|
|
11383
|
+
const level = Object.values(outcomes).every((outcome) => outcome === "ok") ? "info" : "warn";
|
|
11384
|
+
log2(
|
|
11385
|
+
state,
|
|
11386
|
+
`Credential flush: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
11387
|
+
level
|
|
11388
|
+
);
|
|
11389
|
+
});
|
|
11390
|
+
} : void 0;
|
|
11391
|
+
let drainSettled = true;
|
|
8853
11392
|
if (opts.graceful && state.channelDriver) {
|
|
8854
11393
|
state.channelDriver.stop();
|
|
11394
|
+
}
|
|
11395
|
+
if (flushCredentials) {
|
|
11396
|
+
await flushCredentials("credential_flush", !(opts.graceful && state.channelDriver));
|
|
11397
|
+
}
|
|
11398
|
+
if (opts.graceful && state.channelDriver) {
|
|
8855
11399
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
8856
11400
|
if (state.interactive) {
|
|
8857
11401
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
8858
11402
|
displayStatus(state);
|
|
8859
11403
|
}
|
|
8860
11404
|
const driver = state.channelDriver;
|
|
8861
|
-
|
|
11405
|
+
drainSettled = await timeShutdownPhase(
|
|
8862
11406
|
state,
|
|
8863
11407
|
durations,
|
|
8864
11408
|
"drain",
|
|
8865
11409
|
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
8866
11410
|
);
|
|
8867
|
-
if (!
|
|
11411
|
+
if (!drainSettled) {
|
|
8868
11412
|
logActivity(state, {
|
|
8869
11413
|
type: "info",
|
|
8870
11414
|
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
@@ -8872,6 +11416,9 @@ async function cleanup(state, opts = {}) {
|
|
|
8872
11416
|
if (state.interactive) displayStatus(state);
|
|
8873
11417
|
}
|
|
8874
11418
|
}
|
|
11419
|
+
if (opts.graceful && state.channelDriver && flushCredentials && drainSettled) {
|
|
11420
|
+
await flushCredentials("credential_flush_final", true);
|
|
11421
|
+
}
|
|
8875
11422
|
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
8876
11423
|
if (state.connection) {
|
|
8877
11424
|
const connection = state.connection;
|
|
@@ -8907,13 +11454,56 @@ async function cleanup(state, opts = {}) {
|
|
|
8907
11454
|
}
|
|
8908
11455
|
return durations;
|
|
8909
11456
|
}
|
|
11457
|
+
async function beginGracefulShutdown(state, trigger) {
|
|
11458
|
+
if (state.shuttingDown) return;
|
|
11459
|
+
state.shuttingDown = true;
|
|
11460
|
+
const shutdownStartedAt = Date.now();
|
|
11461
|
+
const shutdownMessage = trigger === "recycle" ? "Recycle requested \u2014 finishing in-flight work, then exiting" : "Shutting down...";
|
|
11462
|
+
if (state.interactive) {
|
|
11463
|
+
logActivity(state, { type: "info", message: shutdownMessage });
|
|
11464
|
+
displayStatus(state);
|
|
11465
|
+
} else {
|
|
11466
|
+
log2(state, shutdownMessage);
|
|
11467
|
+
}
|
|
11468
|
+
const durations = await cleanup(state, { graceful: true });
|
|
11469
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
11470
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
11471
|
+
let timer;
|
|
11472
|
+
const flushed = shutdownTelemetry().then(
|
|
11473
|
+
() => true,
|
|
11474
|
+
(error2) => {
|
|
11475
|
+
log2(
|
|
11476
|
+
state,
|
|
11477
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11478
|
+
"warn"
|
|
11479
|
+
);
|
|
11480
|
+
return true;
|
|
11481
|
+
}
|
|
11482
|
+
);
|
|
11483
|
+
const timedOut = new Promise((resolve4) => {
|
|
11484
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
11485
|
+
});
|
|
11486
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
11487
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
11488
|
+
}
|
|
11489
|
+
clearTimeout(timer);
|
|
11490
|
+
});
|
|
11491
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
11492
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
11493
|
+
process.exit(0);
|
|
11494
|
+
}
|
|
8910
11495
|
async function run(options) {
|
|
8911
11496
|
const interactive = isInteractive(options.json);
|
|
8912
11497
|
let logLevel;
|
|
8913
11498
|
let fileSyncDirectories;
|
|
8914
11499
|
try {
|
|
8915
11500
|
logLevel = resolveLogLevel(options);
|
|
8916
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
11501
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir6());
|
|
11502
|
+
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
11503
|
+
throw new Error(
|
|
11504
|
+
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
11505
|
+
);
|
|
11506
|
+
}
|
|
8917
11507
|
} catch (error2) {
|
|
8918
11508
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8919
11509
|
if (options.json) {
|
|
@@ -8937,7 +11527,9 @@ async function run(options) {
|
|
|
8937
11527
|
connected: false,
|
|
8938
11528
|
opencodeConnected: false,
|
|
8939
11529
|
opencodeVersion: null,
|
|
11530
|
+
sessionDbProvenanceAnomaly: false,
|
|
8940
11531
|
opencodeProcess: null,
|
|
11532
|
+
stopOpenCodeLogTail: null,
|
|
8941
11533
|
litestreamProcess: null,
|
|
8942
11534
|
connection: null,
|
|
8943
11535
|
channelDriver: null,
|
|
@@ -8952,9 +11544,24 @@ async function run(options) {
|
|
|
8952
11544
|
openaiUsageTimer: null,
|
|
8953
11545
|
openaiUsageRearm: null,
|
|
8954
11546
|
resourceUsageTimer: null,
|
|
11547
|
+
stopResourceUsageSampling: null,
|
|
11548
|
+
credentialSync: null,
|
|
8955
11549
|
authHeader: ""
|
|
8956
11550
|
};
|
|
8957
11551
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
11552
|
+
if (options.credentialSyncMarker) {
|
|
11553
|
+
state.credentialSync = createCredentialSync({
|
|
11554
|
+
markerPath: options.credentialSyncMarker,
|
|
11555
|
+
env: process.env,
|
|
11556
|
+
log: (message, level = "info") => {
|
|
11557
|
+
if (level === "error") {
|
|
11558
|
+
logActivity(state, { type: "error", error: message });
|
|
11559
|
+
} else {
|
|
11560
|
+
logActivity(state, { type: "info", level, message });
|
|
11561
|
+
}
|
|
11562
|
+
}
|
|
11563
|
+
});
|
|
11564
|
+
}
|
|
8958
11565
|
if (fileSyncDirectories.length > 0) {
|
|
8959
11566
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
8960
11567
|
} else {
|
|
@@ -8980,43 +11587,7 @@ async function run(options) {
|
|
|
8980
11587
|
"warn"
|
|
8981
11588
|
);
|
|
8982
11589
|
}
|
|
8983
|
-
const handleSignal =
|
|
8984
|
-
if (state.shuttingDown) return;
|
|
8985
|
-
state.shuttingDown = true;
|
|
8986
|
-
const shutdownStartedAt = Date.now();
|
|
8987
|
-
if (state.interactive) {
|
|
8988
|
-
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
8989
|
-
displayStatus(state);
|
|
8990
|
-
} else {
|
|
8991
|
-
log2(state, "Shutting down...");
|
|
8992
|
-
}
|
|
8993
|
-
const durations = await cleanup(state, { graceful: true });
|
|
8994
|
-
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
8995
|
-
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
8996
|
-
let timer;
|
|
8997
|
-
const flushed = shutdownTelemetry().then(
|
|
8998
|
-
() => true,
|
|
8999
|
-
(error2) => {
|
|
9000
|
-
log2(
|
|
9001
|
-
state,
|
|
9002
|
-
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9003
|
-
"warn"
|
|
9004
|
-
);
|
|
9005
|
-
return true;
|
|
9006
|
-
}
|
|
9007
|
-
);
|
|
9008
|
-
const timedOut = new Promise((resolve3) => {
|
|
9009
|
-
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
9010
|
-
});
|
|
9011
|
-
if (!await Promise.race([flushed, timedOut])) {
|
|
9012
|
-
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
9013
|
-
}
|
|
9014
|
-
clearTimeout(timer);
|
|
9015
|
-
});
|
|
9016
|
-
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
9017
|
-
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
9018
|
-
process.exit(0);
|
|
9019
|
-
};
|
|
11590
|
+
const handleSignal = () => beginGracefulShutdown(state, "signal");
|
|
9020
11591
|
process.on("SIGINT", handleSignal);
|
|
9021
11592
|
process.on("SIGTERM", handleSignal);
|
|
9022
11593
|
try {
|
|
@@ -9026,15 +11597,15 @@ async function run(options) {
|
|
|
9026
11597
|
printError("Authentication required");
|
|
9027
11598
|
blank();
|
|
9028
11599
|
console.log(
|
|
9029
|
-
|
|
11600
|
+
chalk7.dim("Set EVIDENT_RUNNER_KEY (or EVIDENT_AGENT_KEY) environment variable for CI")
|
|
9030
11601
|
);
|
|
9031
|
-
console.log(
|
|
11602
|
+
console.log(chalk7.dim("Or run `evident login` for interactive authentication"));
|
|
9032
11603
|
blank();
|
|
9033
11604
|
process.exit(1);
|
|
9034
11605
|
return;
|
|
9035
11606
|
}
|
|
9036
11607
|
blank();
|
|
9037
|
-
console.log(
|
|
11608
|
+
console.log(chalk7.yellow("You are not logged in to Evident."));
|
|
9038
11609
|
blank();
|
|
9039
11610
|
credentials2 = await promptForLogin(
|
|
9040
11611
|
"Would you like to log in now?",
|
|
@@ -9084,7 +11655,7 @@ async function run(options) {
|
|
|
9084
11655
|
);
|
|
9085
11656
|
blank();
|
|
9086
11657
|
console.log(
|
|
9087
|
-
|
|
11658
|
+
chalk7.dim(
|
|
9088
11659
|
"Either provide --runner/--agent <id> or set EVIDENT_RUNNER_KEY/EVIDENT_AGENT_KEY"
|
|
9089
11660
|
)
|
|
9090
11661
|
);
|
|
@@ -9107,15 +11678,15 @@ async function run(options) {
|
|
|
9107
11678
|
);
|
|
9108
11679
|
if (interactive && !state.json) {
|
|
9109
11680
|
blank();
|
|
9110
|
-
console.log(
|
|
9111
|
-
console.log(
|
|
11681
|
+
console.log(chalk7.bold("Evident Run"));
|
|
11682
|
+
console.log(chalk7.dim("-".repeat(40)));
|
|
9112
11683
|
}
|
|
9113
11684
|
const spinner = interactive && !state.json ? ora3("Validating runner...").start() : null;
|
|
9114
11685
|
let validation = await getAgentInfo(state.agentId, state.authHeader);
|
|
9115
11686
|
if (!validation.valid && validation.authFailed && interactive) {
|
|
9116
11687
|
spinner?.fail("Authentication failed");
|
|
9117
11688
|
blank();
|
|
9118
|
-
console.log(
|
|
11689
|
+
console.log(chalk7.yellow("Your authentication token is invalid or expired."));
|
|
9119
11690
|
blank();
|
|
9120
11691
|
credentials2 = await promptForLogin(
|
|
9121
11692
|
"Would you like to log in again?",
|
|
@@ -9146,27 +11717,140 @@ async function run(options) {
|
|
|
9146
11717
|
} else {
|
|
9147
11718
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
9148
11719
|
}
|
|
11720
|
+
if (options.restoreRunnerCredentials) {
|
|
11721
|
+
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
11722
|
+
const credentialContext = {
|
|
11723
|
+
env: process.env,
|
|
11724
|
+
log: (message, level = "info") => {
|
|
11725
|
+
if (level === "error") {
|
|
11726
|
+
logActivity(state, { type: "error", error: message });
|
|
11727
|
+
} else {
|
|
11728
|
+
logActivity(state, { type: "info", level, message });
|
|
11729
|
+
}
|
|
11730
|
+
}
|
|
11731
|
+
};
|
|
11732
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
11733
|
+
await restoreCredentialStores(credentialContext);
|
|
11734
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
11735
|
+
}
|
|
11736
|
+
state.credentialSync?.arm();
|
|
11737
|
+
state.stopOpenCodeLogTail = tailOpenCodeLogFile(
|
|
11738
|
+
resolveOpenCodeLogPath(homedir6(), process.env),
|
|
11739
|
+
createOpenCodeActivityForwarder(() => ({
|
|
11740
|
+
agentId: state.agentId,
|
|
11741
|
+
authHeader: state.authHeader
|
|
11742
|
+
}))
|
|
11743
|
+
).stop;
|
|
11744
|
+
let sessionDbVerifyFatal = false;
|
|
11745
|
+
if (!options.restoreSessionDb) {
|
|
11746
|
+
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
11747
|
+
} else {
|
|
11748
|
+
const health = await checkOpenCodeHealth(state.port);
|
|
11749
|
+
if (health.healthy) {
|
|
11750
|
+
log2(
|
|
11751
|
+
state,
|
|
11752
|
+
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
11753
|
+
"debug"
|
|
11754
|
+
);
|
|
11755
|
+
} else {
|
|
11756
|
+
const result = await restoreAndVerifySessionDb({
|
|
11757
|
+
dbPath: sessionDbPath(),
|
|
11758
|
+
litestreamConfig: options.litestreamConfig,
|
|
11759
|
+
noReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
11760
|
+
env: process.env,
|
|
11761
|
+
log: (message, level = "info") => {
|
|
11762
|
+
if (level === "error") {
|
|
11763
|
+
logActivity(state, { type: "error", error: message });
|
|
11764
|
+
} else {
|
|
11765
|
+
logActivity(state, { type: "info", level, message });
|
|
11766
|
+
}
|
|
11767
|
+
},
|
|
11768
|
+
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
11769
|
+
});
|
|
11770
|
+
sessionDbVerifyFatal = result.verifyFatal;
|
|
11771
|
+
}
|
|
11772
|
+
}
|
|
9149
11773
|
reportSessionDbRecovery(state);
|
|
11774
|
+
if (sessionDbVerifyFatal) {
|
|
11775
|
+
throw new Error(
|
|
11776
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
11777
|
+
);
|
|
11778
|
+
}
|
|
11779
|
+
applyRunnerOpenCodeConfig({
|
|
11780
|
+
overlayPath: options.opencodeConfigOverlay,
|
|
11781
|
+
log: (message, level = "info") => {
|
|
11782
|
+
if (level === "error") {
|
|
11783
|
+
logActivity(state, { type: "error", error: message });
|
|
11784
|
+
} else {
|
|
11785
|
+
logActivity(state, { type: "info", level, message });
|
|
11786
|
+
}
|
|
11787
|
+
}
|
|
11788
|
+
});
|
|
9150
11789
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
9151
11790
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
9152
11791
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
9153
11792
|
}
|
|
11793
|
+
const { version: opencodeVersion, warnings: opencodeVersionWarnings } = resolveOpenCodeVersion(
|
|
11794
|
+
options,
|
|
11795
|
+
process.env
|
|
11796
|
+
);
|
|
11797
|
+
for (const warning2 of opencodeVersionWarnings) {
|
|
11798
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
11799
|
+
}
|
|
9154
11800
|
const { value: maxActiveSessions, warnings: maxActiveSessionsWarnings } = resolveMaxActiveSessions(options, process.env);
|
|
9155
11801
|
for (const warning2 of maxActiveSessionsWarnings) {
|
|
9156
11802
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
9157
11803
|
}
|
|
11804
|
+
const preBootMigrationIds = readSessionDbMigrationIds(sessionDbPath());
|
|
9158
11805
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
9159
11806
|
try {
|
|
9160
|
-
const oc = await
|
|
11807
|
+
const oc = opencodeVersion === "v2" ? await ensureOpenCode2Running({
|
|
11808
|
+
port: state.port,
|
|
11809
|
+
interactive: state.interactive,
|
|
11810
|
+
agentId: state.agentId,
|
|
11811
|
+
log: (message) => log2(state, message),
|
|
11812
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
11813
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
11814
|
+
}) : await ensureOpenCodeRunning({
|
|
9161
11815
|
port: state.port,
|
|
9162
11816
|
interactive: state.interactive,
|
|
9163
11817
|
agentId: state.agentId,
|
|
9164
11818
|
log: (message) => log2(state, message),
|
|
9165
|
-
startTimeoutMs: opencodeStartTimeoutMs
|
|
11819
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
11820
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
9166
11821
|
});
|
|
9167
11822
|
state.port = oc.port;
|
|
9168
|
-
state.opencodeProcess = oc.process;
|
|
11823
|
+
state.opencodeProcess = options.opencodePidFile ? null : oc.process;
|
|
9169
11824
|
state.opencodeVersion = oc.version;
|
|
11825
|
+
if (options.opencodePidFile && oc.process?.pid !== void 0) {
|
|
11826
|
+
try {
|
|
11827
|
+
writeFileSync6(options.opencodePidFile, `${oc.process.pid}
|
|
11828
|
+
`, { mode: 384 });
|
|
11829
|
+
chmodSync3(options.opencodePidFile, 384);
|
|
11830
|
+
} catch (error2) {
|
|
11831
|
+
logActivity(state, {
|
|
11832
|
+
type: "error",
|
|
11833
|
+
error: `Failed to write OpenCode pid file ${options.opencodePidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11834
|
+
});
|
|
11835
|
+
}
|
|
11836
|
+
}
|
|
11837
|
+
if (state.opencodeVersion !== null) {
|
|
11838
|
+
const provenance = checkSessionDbProvenance({
|
|
11839
|
+
dbPath: sessionDbPath(),
|
|
11840
|
+
currentVersion: state.opencodeVersion,
|
|
11841
|
+
homeDir: homedir6(),
|
|
11842
|
+
env: process.env
|
|
11843
|
+
});
|
|
11844
|
+
if (provenance.anomaly) {
|
|
11845
|
+
state.sessionDbProvenanceAnomaly = true;
|
|
11846
|
+
logSessionDbProvenanceMismatch(
|
|
11847
|
+
state,
|
|
11848
|
+
provenance,
|
|
11849
|
+
state.opencodeVersion,
|
|
11850
|
+
preBootMigrationIds?.length ?? null
|
|
11851
|
+
);
|
|
11852
|
+
}
|
|
11853
|
+
}
|
|
9170
11854
|
state.opencodeConnected = oc.notReadyReason === null;
|
|
9171
11855
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
9172
11856
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
@@ -9181,6 +11865,7 @@ async function run(options) {
|
|
|
9181
11865
|
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
9182
11866
|
}
|
|
9183
11867
|
}
|
|
11868
|
+
await reloadProviderCache(state.port);
|
|
9184
11869
|
const noProviderWarning = buildNoProviderWarning(
|
|
9185
11870
|
await hasAnyConfiguredProvider(state.port)
|
|
9186
11871
|
);
|
|
@@ -9189,10 +11874,10 @@ async function run(options) {
|
|
|
9189
11874
|
if (state.interactive && !state.json) {
|
|
9190
11875
|
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
9191
11876
|
blank();
|
|
9192
|
-
console.log(
|
|
11877
|
+
console.log(chalk7.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
9193
11878
|
console.log(
|
|
9194
|
-
|
|
9195
|
-
`Run ${
|
|
11879
|
+
chalk7.dim(
|
|
11880
|
+
`Run ${chalk7.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
9196
11881
|
)
|
|
9197
11882
|
);
|
|
9198
11883
|
blank();
|
|
@@ -9203,7 +11888,75 @@ async function run(options) {
|
|
|
9203
11888
|
ocSpinner?.fail(error2.message);
|
|
9204
11889
|
throw error2;
|
|
9205
11890
|
}
|
|
9206
|
-
if (options.
|
|
11891
|
+
if (options.litestreamPidFile) {
|
|
11892
|
+
if (options.sessionDbNoReplicateMarker && existsSync3(options.sessionDbNoReplicateMarker)) {
|
|
11893
|
+
log2(
|
|
11894
|
+
state,
|
|
11895
|
+
`Skipping Litestream replication because ${options.sessionDbNoReplicateMarker} marks this session database unsafe to replicate`
|
|
11896
|
+
);
|
|
11897
|
+
} else if (!options.litestreamConfig) {
|
|
11898
|
+
logActivity(state, {
|
|
11899
|
+
type: "info",
|
|
11900
|
+
level: "warn",
|
|
11901
|
+
message: "Skipping Litestream replication because no configuration file was provided"
|
|
11902
|
+
});
|
|
11903
|
+
} else {
|
|
11904
|
+
let existingPid;
|
|
11905
|
+
if (existsSync3(options.litestreamPidFile)) {
|
|
11906
|
+
try {
|
|
11907
|
+
const rawPid = readFileSync6(options.litestreamPidFile, "utf8").trim();
|
|
11908
|
+
const parsedPid = Number(rawPid);
|
|
11909
|
+
if (/^\d+$/.test(rawPid) && Number.isSafeInteger(parsedPid) && parsedPid > 0) {
|
|
11910
|
+
existingPid = parsedPid;
|
|
11911
|
+
}
|
|
11912
|
+
} catch (error2) {
|
|
11913
|
+
logActivity(state, {
|
|
11914
|
+
type: "info",
|
|
11915
|
+
level: "warn",
|
|
11916
|
+
message: `Could not read Litestream pid file ${options.litestreamPidFile}; checking for a new process: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11917
|
+
});
|
|
11918
|
+
}
|
|
11919
|
+
}
|
|
11920
|
+
if (existingPid !== void 0 && isProcessAlive(existingPid)) {
|
|
11921
|
+
log2(state, `Litestream replication is already running with pid ${existingPid}`);
|
|
11922
|
+
} else {
|
|
11923
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
11924
|
+
state.litestreamProcess = null;
|
|
11925
|
+
let failureHandled = false;
|
|
11926
|
+
const reportImageOwnedReplicationFailure = (message) => {
|
|
11927
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
11928
|
+
failureHandled = true;
|
|
11929
|
+
logActivity(state, { type: "error", error: message });
|
|
11930
|
+
if (state.interactive) displayStatus(state);
|
|
11931
|
+
};
|
|
11932
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
11933
|
+
reportImageOwnedReplicationFailure(
|
|
11934
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
11935
|
+
);
|
|
11936
|
+
});
|
|
11937
|
+
litestreamProcess.on("error", (error2) => {
|
|
11938
|
+
reportImageOwnedReplicationFailure(
|
|
11939
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11940
|
+
);
|
|
11941
|
+
});
|
|
11942
|
+
try {
|
|
11943
|
+
if (litestreamProcess.pid !== void 0) {
|
|
11944
|
+
writeFileSync6(options.litestreamPidFile, `${litestreamProcess.pid}
|
|
11945
|
+
`, {
|
|
11946
|
+
mode: 384
|
|
11947
|
+
});
|
|
11948
|
+
chmodSync3(options.litestreamPidFile, 384);
|
|
11949
|
+
}
|
|
11950
|
+
} catch (error2) {
|
|
11951
|
+
logActivity(state, {
|
|
11952
|
+
type: "error",
|
|
11953
|
+
error: `Failed to write Litestream pid file ${options.litestreamPidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11954
|
+
});
|
|
11955
|
+
}
|
|
11956
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
11957
|
+
}
|
|
11958
|
+
}
|
|
11959
|
+
} else if (options.litestreamConfig) {
|
|
9207
11960
|
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
9208
11961
|
state.litestreamProcess = litestreamProcess;
|
|
9209
11962
|
let failureHandled = false;
|
|
@@ -9248,7 +12001,7 @@ async function run(options) {
|
|
|
9248
12001
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
9249
12002
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
9250
12003
|
fileSyncDirectories,
|
|
9251
|
-
homeDir:
|
|
12004
|
+
homeDir: homedir6(),
|
|
9252
12005
|
maxActiveSessions,
|
|
9253
12006
|
log: (entry) => (
|
|
9254
12007
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -9374,6 +12127,18 @@ async function run(options) {
|
|
|
9374
12127
|
if (state.interactive) displayStatus(state);
|
|
9375
12128
|
});
|
|
9376
12129
|
},
|
|
12130
|
+
// Both loops are rearmed because `rearm()` is idempotent for the
|
|
12131
|
+
// provider that did not just connect, and is a no-op when reporting is off.
|
|
12132
|
+
onUsageRearmPing: () => {
|
|
12133
|
+
if (!state.running) return;
|
|
12134
|
+
logActivity(state, {
|
|
12135
|
+
type: "info",
|
|
12136
|
+
level: "debug",
|
|
12137
|
+
message: "Usage rearm ping received"
|
|
12138
|
+
});
|
|
12139
|
+
state.claudeUsageRearm?.();
|
|
12140
|
+
state.openaiUsageRearm?.();
|
|
12141
|
+
},
|
|
9377
12142
|
onInfo: (message) => logActivity(state, { type: "info", message })
|
|
9378
12143
|
}
|
|
9379
12144
|
});
|
|
@@ -9394,7 +12159,17 @@ async function run(options) {
|
|
|
9394
12159
|
setTimer: (timer) => {
|
|
9395
12160
|
state.openaiUsageTimer = timer;
|
|
9396
12161
|
},
|
|
9397
|
-
fetchUsage: () =>
|
|
12162
|
+
fetchUsage: async () => {
|
|
12163
|
+
const usage = await getOpenAiUsage(state.port);
|
|
12164
|
+
if (usage.subscription === null) {
|
|
12165
|
+
logActivity(state, {
|
|
12166
|
+
type: "info",
|
|
12167
|
+
level: "debug",
|
|
12168
|
+
message: "OpenAI usage subscription could not be identified from the local credential"
|
|
12169
|
+
});
|
|
12170
|
+
}
|
|
12171
|
+
return usage;
|
|
12172
|
+
},
|
|
9398
12173
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
9399
12174
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
9400
12175
|
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
@@ -9440,7 +12215,7 @@ async function run(options) {
|
|
|
9440
12215
|
}
|
|
9441
12216
|
|
|
9442
12217
|
// src/index.ts
|
|
9443
|
-
var { version } =
|
|
12218
|
+
var { version } = createRequire2(import.meta.url)("../package.json");
|
|
9444
12219
|
var program = new Command();
|
|
9445
12220
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
9446
12221
|
"--endpoint <url>",
|
|
@@ -9468,6 +12243,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9468
12243
|
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
9469
12244
|
"--opencode-start-timeout <seconds>",
|
|
9470
12245
|
"Seconds to wait for OpenCode to become healthy when the runner starts it (default: 180). Env: EVIDENT_OPENCODE_START_TIMEOUT"
|
|
12246
|
+
).option(
|
|
12247
|
+
"--opencode-version <v1|v2>",
|
|
12248
|
+
"Which OpenCode major version to launch: v1 or v2 (default: v1). Env: EVIDENT_OPENCODE_VERSION"
|
|
9471
12249
|
).option("--json", "Output in JSON format").option(
|
|
9472
12250
|
"--session-cleanup-max-age <duration>",
|
|
9473
12251
|
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
@@ -9500,6 +12278,27 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9500
12278
|
).option(
|
|
9501
12279
|
"--litestream-config <path>",
|
|
9502
12280
|
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
12281
|
+
).option(
|
|
12282
|
+
"--opencode-pid-file <path>",
|
|
12283
|
+
"Record the OpenCode pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
12284
|
+
).option(
|
|
12285
|
+
"--litestream-pid-file <path>",
|
|
12286
|
+
"Record the Litestream pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
12287
|
+
).option(
|
|
12288
|
+
"--session-db-no-replicate-marker <path>",
|
|
12289
|
+
"Read this marker as a gate before starting Litestream; the session-DB restore writes it when this boot's database is not safe to replicate."
|
|
12290
|
+
).option(
|
|
12291
|
+
"--restore-session-db",
|
|
12292
|
+
"Restore and verify the OpenCode session database before starting OpenCode; requires --litestream-config for the replica and --session-db-no-replicate-marker for give-up records."
|
|
12293
|
+
).option(
|
|
12294
|
+
"--restore-runner-credentials",
|
|
12295
|
+
"Restore the hosted runner secret and persisted credential stores before starting OpenCode."
|
|
12296
|
+
).option(
|
|
12297
|
+
"--opencode-config-overlay <path>",
|
|
12298
|
+
"Apply this runner-provided OpenCode config before starting OpenCode."
|
|
12299
|
+
).option(
|
|
12300
|
+
"--credential-sync-marker <path>",
|
|
12301
|
+
"Own the interval credential sync and write this marker once the shutdown flush has finished, so the runner image's lifecycle hooks can wait on it."
|
|
9503
12302
|
).action(
|
|
9504
12303
|
(options) => {
|
|
9505
12304
|
run({
|
|
@@ -9515,6 +12314,7 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9515
12314
|
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
9516
12315
|
// resolveOpenCodeStartTimeoutMs (flag > EVIDENT_OPENCODE_START_TIMEOUT > 180s default).
|
|
9517
12316
|
opencodeStartTimeout: options.opencodeStartTimeout,
|
|
12317
|
+
opencodeVersion: options.opencodeVersion,
|
|
9518
12318
|
json: options.json,
|
|
9519
12319
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
9520
12320
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
@@ -9532,7 +12332,14 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9532
12332
|
// resolveFileSyncDirectories.
|
|
9533
12333
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
9534
12334
|
tunnelReadyFile: options.tunnelReadyFile,
|
|
9535
|
-
litestreamConfig: options.litestreamConfig
|
|
12335
|
+
litestreamConfig: options.litestreamConfig,
|
|
12336
|
+
opencodePidFile: options.opencodePidFile,
|
|
12337
|
+
litestreamPidFile: options.litestreamPidFile,
|
|
12338
|
+
sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
12339
|
+
restoreSessionDb: options.restoreSessionDb,
|
|
12340
|
+
restoreRunnerCredentials: options.restoreRunnerCredentials,
|
|
12341
|
+
opencodeConfigOverlay: options.opencodeConfigOverlay,
|
|
12342
|
+
credentialSyncMarker: options.credentialSyncMarker
|
|
9536
12343
|
});
|
|
9537
12344
|
}
|
|
9538
12345
|
);
|