@evident-ai/cli 3.4.1-dev.8fa4d29 → 3.4.1-dev.90d85e0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -1
- package/dist/index.js +3227 -458
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire } from "module";
|
|
4
|
+
import { createRequire as createRequire2 } from "module";
|
|
5
5
|
import { Command } from "commander";
|
|
6
6
|
|
|
7
7
|
// src/commands/login.ts
|
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
-
import { dirname } from "path";
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "node:fs";
|
|
15
|
+
import { dirname } from "node:path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -371,14 +371,14 @@ function blank() {
|
|
|
371
371
|
console.log();
|
|
372
372
|
}
|
|
373
373
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
374
|
-
return new Promise((
|
|
374
|
+
return new Promise((resolve4) => {
|
|
375
375
|
process.stdout.write(chalk.dim(prompt));
|
|
376
376
|
const handler = () => {
|
|
377
377
|
process.stdin.removeListener("data", handler);
|
|
378
378
|
process.stdin.setRawMode?.(false);
|
|
379
379
|
process.stdin.pause();
|
|
380
380
|
console.log();
|
|
381
|
-
|
|
381
|
+
resolve4();
|
|
382
382
|
};
|
|
383
383
|
if (process.stdin.isTTY) {
|
|
384
384
|
process.stdin.setRawMode?.(true);
|
|
@@ -388,7 +388,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
388
388
|
});
|
|
389
389
|
}
|
|
390
390
|
function sleep(ms) {
|
|
391
|
-
return new Promise((
|
|
391
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
392
392
|
}
|
|
393
393
|
|
|
394
394
|
// src/commands/login.ts
|
|
@@ -466,19 +466,19 @@ async function tokenLogin() {
|
|
|
466
466
|
);
|
|
467
467
|
blank();
|
|
468
468
|
process.stdout.write("Paste token: ");
|
|
469
|
-
const token = await new Promise((
|
|
469
|
+
const token = await new Promise((resolve4) => {
|
|
470
470
|
let data = "";
|
|
471
471
|
process.stdin.setEncoding("utf8");
|
|
472
472
|
process.stdin.on("data", (chunk) => {
|
|
473
473
|
data += chunk;
|
|
474
474
|
});
|
|
475
475
|
process.stdin.on("end", () => {
|
|
476
|
-
|
|
476
|
+
resolve4(data.trim());
|
|
477
477
|
});
|
|
478
478
|
if (process.stdin.isTTY) {
|
|
479
479
|
process.stdin.once("data", (chunk) => {
|
|
480
480
|
process.stdin.pause();
|
|
481
|
-
|
|
481
|
+
resolve4(chunk.toString().trim());
|
|
482
482
|
});
|
|
483
483
|
process.stdin.resume();
|
|
484
484
|
}
|
|
@@ -722,6 +722,14 @@ function toReportedWindow(window) {
|
|
|
722
722
|
if (!window) return null;
|
|
723
723
|
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
724
724
|
}
|
|
725
|
+
function toReportedOwner(snapshot) {
|
|
726
|
+
if (!snapshot.owner) return null;
|
|
727
|
+
return {
|
|
728
|
+
email: snapshot.owner.email,
|
|
729
|
+
organization_name: snapshot.owner.organizationName,
|
|
730
|
+
rate_limit_tier: snapshot.owner.rateLimitTier
|
|
731
|
+
};
|
|
732
|
+
}
|
|
725
733
|
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
726
734
|
try {
|
|
727
735
|
const apiUrl = getApiUrlConfig();
|
|
@@ -730,7 +738,50 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
|
730
738
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
731
739
|
body: JSON.stringify({
|
|
732
740
|
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
733
|
-
seven_day: toReportedWindow(snapshot.sevenDay)
|
|
741
|
+
seven_day: toReportedWindow(snapshot.sevenDay),
|
|
742
|
+
owner: toReportedOwner(snapshot)
|
|
743
|
+
}),
|
|
744
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
745
|
+
});
|
|
746
|
+
if (!response.ok) {
|
|
747
|
+
const serverMessage = await readErrorMessage(response);
|
|
748
|
+
return {
|
|
749
|
+
ok: false,
|
|
750
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
return { ok: true };
|
|
754
|
+
} catch (error2) {
|
|
755
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
function toReportedOpenAiWindow(window) {
|
|
759
|
+
if (!window) return null;
|
|
760
|
+
return {
|
|
761
|
+
utilization: window.utilization,
|
|
762
|
+
window_minutes: window.windowMinutes,
|
|
763
|
+
resets_at: window.resetsAt
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
function toReportedOpenAiSubscription(snapshot) {
|
|
767
|
+
if (!snapshot.subscription) return null;
|
|
768
|
+
return {
|
|
769
|
+
owner_email: snapshot.subscription.ownerEmail,
|
|
770
|
+
plan_type: snapshot.subscription.planType
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
774
|
+
try {
|
|
775
|
+
const apiUrl = getApiUrlConfig();
|
|
776
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/openai-usage`, {
|
|
777
|
+
method: "POST",
|
|
778
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
779
|
+
body: JSON.stringify({
|
|
780
|
+
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
781
|
+
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
782
|
+
has_credits: snapshot.hasCredits,
|
|
783
|
+
credits_unlimited: snapshot.creditsUnlimited,
|
|
784
|
+
subscription: toReportedOpenAiSubscription(snapshot)
|
|
734
785
|
}),
|
|
735
786
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
736
787
|
});
|
|
@@ -754,6 +805,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
754
805
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
755
806
|
body: JSON.stringify({
|
|
756
807
|
cpu_percent: usage.cpuPercent,
|
|
808
|
+
cpu_peak_percent: usage.cpuPeakPercent,
|
|
757
809
|
cpu_count: usage.cpuCount,
|
|
758
810
|
memory_total_bytes: usage.memoryTotalBytes,
|
|
759
811
|
memory_available_bytes: usage.memoryAvailableBytes,
|
|
@@ -957,12 +1009,15 @@ async function status(options = {}) {
|
|
|
957
1009
|
}
|
|
958
1010
|
|
|
959
1011
|
// src/lib/claude-usage.ts
|
|
960
|
-
import { execFileSync } from "child_process";
|
|
961
|
-
import { readFileSync } from "fs";
|
|
962
|
-
import { homedir } from "os";
|
|
963
|
-
import { join } from "path";
|
|
1012
|
+
import { execFileSync } from "node:child_process";
|
|
1013
|
+
import { readFileSync } from "node:fs";
|
|
1014
|
+
import { homedir } from "node:os";
|
|
1015
|
+
import { join } from "node:path";
|
|
964
1016
|
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
1017
|
+
var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
|
|
1018
|
+
var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
|
|
965
1019
|
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
1020
|
+
var cachedOwner = null;
|
|
966
1021
|
var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
|
|
967
1022
|
function parseClaudeCliCredentials(raw) {
|
|
968
1023
|
let parsed;
|
|
@@ -1036,6 +1091,47 @@ function toWindow(value) {
|
|
|
1036
1091
|
}
|
|
1037
1092
|
return { utilization: window.utilization, resetsAt };
|
|
1038
1093
|
}
|
|
1094
|
+
function ownerLookupFailure(error2) {
|
|
1095
|
+
const name = error2?.name;
|
|
1096
|
+
return name === "TimeoutError" || name === "AbortError" ? "timed out" : "request failed";
|
|
1097
|
+
}
|
|
1098
|
+
async function getClaudeUsageOwner(accessToken) {
|
|
1099
|
+
if (cachedOwner?.accessToken === accessToken) {
|
|
1100
|
+
return { owner: cachedOwner.owner, ownerLookupError: null };
|
|
1101
|
+
}
|
|
1102
|
+
try {
|
|
1103
|
+
const response = await fetch(CLAUDE_PROFILE_URL, {
|
|
1104
|
+
headers: {
|
|
1105
|
+
Authorization: `Bearer ${accessToken}`,
|
|
1106
|
+
"Content-Type": "application/json",
|
|
1107
|
+
"anthropic-version": "2023-06-01"
|
|
1108
|
+
},
|
|
1109
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1110
|
+
});
|
|
1111
|
+
if (!response.ok) {
|
|
1112
|
+
return { owner: null, ownerLookupError: `HTTP ${response.status}` };
|
|
1113
|
+
}
|
|
1114
|
+
let body;
|
|
1115
|
+
try {
|
|
1116
|
+
body = await response.json();
|
|
1117
|
+
} catch (error2) {
|
|
1118
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1119
|
+
}
|
|
1120
|
+
const profile = body;
|
|
1121
|
+
if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
|
|
1122
|
+
return { owner: null, ownerLookupError: "malformed response" };
|
|
1123
|
+
}
|
|
1124
|
+
const owner = {
|
|
1125
|
+
email: profile.account.email,
|
|
1126
|
+
organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
|
|
1127
|
+
rateLimitTier: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
|
|
1128
|
+
};
|
|
1129
|
+
cachedOwner = { accessToken, owner };
|
|
1130
|
+
return { owner, ownerLookupError: null };
|
|
1131
|
+
} catch (error2) {
|
|
1132
|
+
return { owner: null, ownerLookupError: ownerLookupFailure(error2) };
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1039
1135
|
async function getClaudeUsage() {
|
|
1040
1136
|
const credentials2 = readClaudeCliCredentials();
|
|
1041
1137
|
if (!credentials2) {
|
|
@@ -1055,15 +1151,19 @@ async function getClaudeUsage() {
|
|
|
1055
1151
|
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
1056
1152
|
"Content-Type": "application/json",
|
|
1057
1153
|
"anthropic-version": "2023-06-01"
|
|
1058
|
-
}
|
|
1154
|
+
},
|
|
1155
|
+
signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
|
|
1059
1156
|
});
|
|
1060
1157
|
if (!res.ok) {
|
|
1061
1158
|
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
1062
1159
|
}
|
|
1063
1160
|
const body = await res.json();
|
|
1161
|
+
const { owner, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
|
|
1064
1162
|
return {
|
|
1065
1163
|
fiveHour: toWindow(body.five_hour),
|
|
1066
|
-
sevenDay: toWindow(body.seven_day)
|
|
1164
|
+
sevenDay: toWindow(body.seven_day),
|
|
1165
|
+
owner,
|
|
1166
|
+
ownerLookupError
|
|
1067
1167
|
};
|
|
1068
1168
|
}
|
|
1069
1169
|
|
|
@@ -1092,9 +1192,10 @@ async function claudeUsage() {
|
|
|
1092
1192
|
}
|
|
1093
1193
|
|
|
1094
1194
|
// src/commands/run.ts
|
|
1095
|
-
import {
|
|
1096
|
-
import {
|
|
1097
|
-
import
|
|
1195
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1196
|
+
import { homedir as homedir5 } from "node:os";
|
|
1197
|
+
import { isAbsolute as isAbsolute3, join as join9, parse, resolve as resolvePath2 } from "node:path";
|
|
1198
|
+
import chalk7 from "chalk";
|
|
1098
1199
|
|
|
1099
1200
|
// ../../packages/types/src/agents/index.ts
|
|
1100
1201
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
@@ -1115,6 +1216,7 @@ var TelemetryEventTypes = {
|
|
|
1115
1216
|
// ../../packages/types/src/tunnel/index.ts
|
|
1116
1217
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1117
1218
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1219
|
+
var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
|
|
1118
1220
|
|
|
1119
1221
|
// ../../packages/types/src/runner-files.ts
|
|
1120
1222
|
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
@@ -1152,7 +1254,7 @@ function stripQuery(url) {
|
|
|
1152
1254
|
|
|
1153
1255
|
// src/commands/run.ts
|
|
1154
1256
|
import ora3 from "ora";
|
|
1155
|
-
import { select as
|
|
1257
|
+
import { select as select4 } from "@inquirer/prompts";
|
|
1156
1258
|
|
|
1157
1259
|
// src/lib/telemetry.ts
|
|
1158
1260
|
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
@@ -1326,6 +1428,8 @@ var SEVERITY_BY_LEVEL = {
|
|
|
1326
1428
|
error: "error"
|
|
1327
1429
|
};
|
|
1328
1430
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1431
|
+
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1432
|
+
var MAX_METADATA_ENTRIES = 20;
|
|
1329
1433
|
var TRUNCATION_MARKER = "\u2026";
|
|
1330
1434
|
function redact(message) {
|
|
1331
1435
|
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
@@ -1334,6 +1438,24 @@ function truncate(message) {
|
|
|
1334
1438
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1335
1439
|
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
1336
1440
|
}
|
|
1441
|
+
function sanitiseMetadata(metadata) {
|
|
1442
|
+
if (!metadata) return {};
|
|
1443
|
+
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1444
|
+
if (Object.keys(metadata).length > entries.length) {
|
|
1445
|
+
console.error(
|
|
1446
|
+
`[runner-activity-telemetry] dropped ${Object.keys(metadata).length - entries.length} metadata entries`
|
|
1447
|
+
);
|
|
1448
|
+
}
|
|
1449
|
+
const sanitised = [];
|
|
1450
|
+
for (const [key, value] of entries) {
|
|
1451
|
+
if (typeof value === "string") {
|
|
1452
|
+
sanitised.push([key, truncate(redact(value).slice(0, MAX_METADATA_VALUE_LENGTH))]);
|
|
1453
|
+
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1454
|
+
sanitised.push([key, value]);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
return Object.fromEntries(sanitised);
|
|
1458
|
+
}
|
|
1337
1459
|
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1338
1460
|
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1339
1461
|
var windowStartedAt = 0;
|
|
@@ -1372,7 +1494,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1372
1494
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1373
1495
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1374
1496
|
message,
|
|
1375
|
-
metadata: { source: "cli.run" },
|
|
1497
|
+
metadata: { ...sanitiseMetadata(entry.metadata), source: "cli.run" },
|
|
1376
1498
|
agentId: context.agentId
|
|
1377
1499
|
});
|
|
1378
1500
|
} catch (err) {
|
|
@@ -1382,6 +1504,191 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1382
1504
|
}
|
|
1383
1505
|
}
|
|
1384
1506
|
|
|
1507
|
+
// src/lib/opencode/session-db-recovery-report.ts
|
|
1508
|
+
import { readFileSync as readFileSync2, unlinkSync } from "node:fs";
|
|
1509
|
+
import { join as join2 } from "node:path";
|
|
1510
|
+
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1511
|
+
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1512
|
+
return override || join2(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1513
|
+
}
|
|
1514
|
+
function drainSessionDbRecoveryReport({
|
|
1515
|
+
homeDir,
|
|
1516
|
+
env
|
|
1517
|
+
}) {
|
|
1518
|
+
const path = sessionDbRecoveryReportPath(homeDir, env);
|
|
1519
|
+
let content;
|
|
1520
|
+
try {
|
|
1521
|
+
content = readFileSync2(path, "utf8");
|
|
1522
|
+
} catch (error2) {
|
|
1523
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")
|
|
1524
|
+
return { path, records: [], skippedLines: 0, readError: null };
|
|
1525
|
+
const readError = error2 instanceof Error ? error2.message : String(error2);
|
|
1526
|
+
console.error(`[session-db-recovery-report] could not read ${path}: ${readError}`);
|
|
1527
|
+
return { path, records: [], skippedLines: 0, readError };
|
|
1528
|
+
}
|
|
1529
|
+
let skippedLines = 0;
|
|
1530
|
+
const records = content.split("\n").flatMap((line) => {
|
|
1531
|
+
if (!line.trim()) return [];
|
|
1532
|
+
try {
|
|
1533
|
+
const value = JSON.parse(line);
|
|
1534
|
+
if (!isSessionDbRecoveryRecord(value)) {
|
|
1535
|
+
skippedLines++;
|
|
1536
|
+
return [];
|
|
1537
|
+
}
|
|
1538
|
+
return [
|
|
1539
|
+
{
|
|
1540
|
+
...value,
|
|
1541
|
+
provenance_reason: value.provenance_reason ?? null,
|
|
1542
|
+
provenance_migration_delta: value.provenance_migration_delta ?? null,
|
|
1543
|
+
replication_suspended: value.replication_suspended ?? false
|
|
1544
|
+
}
|
|
1545
|
+
];
|
|
1546
|
+
} catch (error2) {
|
|
1547
|
+
skippedLines++;
|
|
1548
|
+
console.error(
|
|
1549
|
+
`[session-db-recovery-report] skipped malformed record in ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1550
|
+
);
|
|
1551
|
+
return [];
|
|
1552
|
+
}
|
|
1553
|
+
});
|
|
1554
|
+
return { path, records, skippedLines, readError: null };
|
|
1555
|
+
}
|
|
1556
|
+
function acknowledgeSessionDbRecoveryReport(path) {
|
|
1557
|
+
try {
|
|
1558
|
+
unlinkSync(path);
|
|
1559
|
+
} catch (error2) {
|
|
1560
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1561
|
+
console.error(
|
|
1562
|
+
`[session-db-recovery-report] could not remove ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1563
|
+
);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
function buildSessionDbRecoveryActivity(record) {
|
|
1567
|
+
const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
|
|
1568
|
+
if (!level) return null;
|
|
1569
|
+
const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
|
|
1570
|
+
const replication = record.replication_suspended ? " This start is not backing up its new session history; restart after fixing the cause." : "";
|
|
1571
|
+
const giveupMessage = (() => {
|
|
1572
|
+
switch (record.reason) {
|
|
1573
|
+
case "restore_deadline_exceeded":
|
|
1574
|
+
return "The restore did not finish before this startup deadline. Earlier sessions are unavailable for this start; collect the SESSION-DB-RESTORE-TRUNCATED boot log before retrying.";
|
|
1575
|
+
case "restore_tool_unusable":
|
|
1576
|
+
case "classification_unrecognised":
|
|
1577
|
+
return "The restore or classification tool produced an unsupported result. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1578
|
+
case "synchroniser_config_unevaluable":
|
|
1579
|
+
case "synchroniser_config_incomplete":
|
|
1580
|
+
return "The installed hook and synchroniser bundle could not agree on configuration. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1581
|
+
case "synchroniser_config_unresolved":
|
|
1582
|
+
return "The synchroniser configuration could not be resolved. Inspect the preceding boot error and repair the identified runner image, bundle, or local environment before restarting.";
|
|
1583
|
+
case "litestream_config_unavailable":
|
|
1584
|
+
return "The local Litestream configuration could not be generated or written. Inspect the preceding boot error and repair the runner image, bundle, local environment, or write permissions before restarting.";
|
|
1585
|
+
case "classification_fatal":
|
|
1586
|
+
return "Session history could not be restored because backup setup could not be established. Inspect the named boot marker or log and repair the identified backup configuration or permissions.";
|
|
1587
|
+
default:
|
|
1588
|
+
return null;
|
|
1589
|
+
}
|
|
1590
|
+
})();
|
|
1591
|
+
if (giveupMessage)
|
|
1592
|
+
return {
|
|
1593
|
+
level,
|
|
1594
|
+
metadata: withoutContractFields(record),
|
|
1595
|
+
message: `${giveupMessage}${replication}`
|
|
1596
|
+
};
|
|
1597
|
+
switch (record.outcome) {
|
|
1598
|
+
case "fresh_session_db":
|
|
1599
|
+
return {
|
|
1600
|
+
level,
|
|
1601
|
+
metadata: withoutContractFields(record),
|
|
1602
|
+
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.${replication}`
|
|
1603
|
+
};
|
|
1604
|
+
case "restore_retried":
|
|
1605
|
+
return {
|
|
1606
|
+
level,
|
|
1607
|
+
metadata: withoutContractFields(record),
|
|
1608
|
+
message: "The initial session database restore failed; the runner will retry it. Monitor the runner backup for another restore failure."
|
|
1609
|
+
};
|
|
1610
|
+
case "replica_recovered":
|
|
1611
|
+
if (record.reason === "quarantine")
|
|
1612
|
+
return {
|
|
1613
|
+
level,
|
|
1614
|
+
metadata: withoutContractFields(record),
|
|
1615
|
+
message: `Session database recovery quarantined ${record.quarantined_objects ?? "unknown"} objects (${record.quarantined_bytes ?? "unknown"} bytes); ${record.quarantine_failed_objects ?? "unknown"} moves failed. Review the preserved backup at ${record.quarantine_destination ?? "an unknown destination"} before deleting it.`
|
|
1616
|
+
};
|
|
1617
|
+
if (record.reason === "prune")
|
|
1618
|
+
return {
|
|
1619
|
+
level,
|
|
1620
|
+
metadata: withoutContractFields(record),
|
|
1621
|
+
message: "Session database recovery discarded a damaged newest backup and retried. Sessions recorded after the previous backup point may be unavailable. Review the runner backup for another restore failure."
|
|
1622
|
+
};
|
|
1623
|
+
if (record.reason === "clear")
|
|
1624
|
+
return {
|
|
1625
|
+
level,
|
|
1626
|
+
metadata: withoutContractFields(record),
|
|
1627
|
+
message: "Session database recovery deleted the damaged backup and prior session history is unavailable. Review the runner backup configuration before relying on restored session history."
|
|
1628
|
+
};
|
|
1629
|
+
return null;
|
|
1630
|
+
case "history_rolled_back":
|
|
1631
|
+
return {
|
|
1632
|
+
level,
|
|
1633
|
+
metadata: withoutContractFields(record),
|
|
1634
|
+
message: `Session history was rolled back to verified restore point ${record.verified_restore_point ?? "unknown"}; everything after it is unavailable. Review the runner backup for another restore failure.`
|
|
1635
|
+
};
|
|
1636
|
+
case "restore_misconfigured":
|
|
1637
|
+
return {
|
|
1638
|
+
level,
|
|
1639
|
+
metadata: withoutContractFields(record),
|
|
1640
|
+
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1641
|
+
};
|
|
1642
|
+
case "session_db_boot_refused":
|
|
1643
|
+
return {
|
|
1644
|
+
level,
|
|
1645
|
+
metadata: withoutContractFields(record),
|
|
1646
|
+
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1647
|
+
};
|
|
1648
|
+
case "schema_provenance_mismatch":
|
|
1649
|
+
return {
|
|
1650
|
+
level,
|
|
1651
|
+
metadata: withoutContractFields(record),
|
|
1652
|
+
message: `Session database schema provenance mismatch for ${record.dbPath ?? record.db_path ?? "unknown"}: recorded version=${record.recorded_version ?? "unknown"}, current version=${record.current_version ?? "unknown"}, reason=${record.provenance_reason ?? "unknown"}, migration delta=${record.provenance_migration_delta ?? "unknown"}. Inspect the session database and runner backup before continuing.`
|
|
1653
|
+
};
|
|
1654
|
+
default:
|
|
1655
|
+
return null;
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
function withoutContractFields(record) {
|
|
1659
|
+
const { v: _v, event: _event, ...metadata } = record;
|
|
1660
|
+
return metadata;
|
|
1661
|
+
}
|
|
1662
|
+
var OUTCOMES = /* @__PURE__ */ new Set([
|
|
1663
|
+
"replica_recovered",
|
|
1664
|
+
"restore_retried",
|
|
1665
|
+
"fresh_session_db",
|
|
1666
|
+
"history_rolled_back",
|
|
1667
|
+
"restore_misconfigured",
|
|
1668
|
+
"session_db_boot_refused",
|
|
1669
|
+
"schema_provenance_mismatch"
|
|
1670
|
+
]);
|
|
1671
|
+
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1672
|
+
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
1673
|
+
var NUMBER_FIELDS = [
|
|
1674
|
+
"litestream_exit_code",
|
|
1675
|
+
"attempt",
|
|
1676
|
+
"replica_objects",
|
|
1677
|
+
"replica_bytes",
|
|
1678
|
+
"quarantined_objects",
|
|
1679
|
+
"quarantine_failed_objects",
|
|
1680
|
+
"quarantined_bytes",
|
|
1681
|
+
"restore_points_tried"
|
|
1682
|
+
];
|
|
1683
|
+
var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"];
|
|
1684
|
+
function isSessionDbRecoveryRecord(value) {
|
|
1685
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1686
|
+
const record = value;
|
|
1687
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && (record.provenance_reason === void 0 || record.provenance_reason === null || typeof record.provenance_reason === "string") && (record.provenance_migration_delta === void 0 || record.provenance_migration_delta === null || Number.isInteger(record.provenance_migration_delta)) && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1688
|
+
(field) => record[field] === null || typeof record[field] === "string"
|
|
1689
|
+
);
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1385
1692
|
// src/lib/opencode/health.ts
|
|
1386
1693
|
async function checkOpenCodeHealth(port) {
|
|
1387
1694
|
try {
|
|
@@ -1406,140 +1713,855 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1406
1713
|
if (health.healthy) {
|
|
1407
1714
|
return health;
|
|
1408
1715
|
}
|
|
1409
|
-
await new Promise((
|
|
1716
|
+
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
1410
1717
|
}
|
|
1411
1718
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1412
1719
|
}
|
|
1413
1720
|
|
|
1414
|
-
// src/lib/opencode/
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
}
|
|
1420
|
-
function buildOpenCodeVersionWarning(version2) {
|
|
1421
|
-
if (isQueueValidatedVersion(version2)) return null;
|
|
1422
|
-
const detected = version2 ? `v${version2}` : "unknown";
|
|
1423
|
-
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
1424
|
-
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
1425
|
-
}
|
|
1721
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1722
|
+
import { spawn as spawn2 } from "node:child_process";
|
|
1723
|
+
import { mkdirSync, statSync as statSync2, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
1724
|
+
import { homedir as homedir2 } from "node:os";
|
|
1725
|
+
import { dirname as dirname2, resolve as resolvePath } from "node:path";
|
|
1426
1726
|
|
|
1427
|
-
// src/lib/
|
|
1428
|
-
import {
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1727
|
+
// src/lib/runner-synchroniser.ts
|
|
1728
|
+
import { spawn } from "node:child_process";
|
|
1729
|
+
function appendError(stderr, error2) {
|
|
1730
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1731
|
+
return stderr === "" ? message : `${stderr}
|
|
1732
|
+
${message}`;
|
|
1733
|
+
}
|
|
1734
|
+
function runSynchroniser(args, opts) {
|
|
1735
|
+
return new Promise((resolve4) => {
|
|
1736
|
+
let child;
|
|
1737
|
+
let stdout = "";
|
|
1738
|
+
let stderr = "";
|
|
1739
|
+
let settled = false;
|
|
1740
|
+
const timer = {};
|
|
1741
|
+
let abortListener;
|
|
1742
|
+
let spawnListener;
|
|
1743
|
+
const finish = (result) => {
|
|
1744
|
+
if (settled) return;
|
|
1745
|
+
settled = true;
|
|
1746
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
1747
|
+
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
1748
|
+
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
1749
|
+
resolve4(result);
|
|
1750
|
+
};
|
|
1751
|
+
try {
|
|
1752
|
+
child = spawn("runner-synchroniser", args, {
|
|
1753
|
+
env: opts.env ?? process.env,
|
|
1754
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1755
|
+
});
|
|
1756
|
+
} catch (error2) {
|
|
1757
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
child.stdout?.setEncoding("utf8");
|
|
1761
|
+
child.stdout?.on("data", (chunk) => {
|
|
1762
|
+
stdout += chunk;
|
|
1763
|
+
});
|
|
1764
|
+
child.stderr?.setEncoding("utf8");
|
|
1765
|
+
child.stderr?.on("data", (chunk) => {
|
|
1766
|
+
stderr += chunk;
|
|
1767
|
+
});
|
|
1768
|
+
child.once("error", (error2) => {
|
|
1769
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1770
|
+
});
|
|
1771
|
+
child.once("close", (code) => {
|
|
1772
|
+
finish({ code, stdout, stderr, timedOut: false });
|
|
1773
|
+
});
|
|
1774
|
+
if (opts.signal) {
|
|
1775
|
+
const killChild = () => {
|
|
1776
|
+
if (child.pid === void 0) {
|
|
1777
|
+
if (!spawnListener) {
|
|
1778
|
+
spawnListener = killChild;
|
|
1779
|
+
child.once("spawn", spawnListener);
|
|
1780
|
+
}
|
|
1781
|
+
return;
|
|
1442
1782
|
}
|
|
1783
|
+
child.kill("SIGKILL");
|
|
1784
|
+
};
|
|
1785
|
+
abortListener = killChild;
|
|
1786
|
+
if (opts.signal.aborted) {
|
|
1787
|
+
abortListener();
|
|
1788
|
+
} else {
|
|
1789
|
+
opts.signal.addEventListener("abort", abortListener, { once: true });
|
|
1790
|
+
if (opts.signal.aborted) abortListener();
|
|
1443
1791
|
}
|
|
1444
|
-
} else if (platform === "linux") {
|
|
1445
|
-
const output = execSync(`readlink /proc/${pid}/cwd 2>/dev/null`, {
|
|
1446
|
-
encoding: "utf-8",
|
|
1447
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1448
|
-
}).trim();
|
|
1449
|
-
if (output) return output;
|
|
1450
1792
|
}
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1793
|
+
timer.handle = setTimeout(
|
|
1794
|
+
() => {
|
|
1795
|
+
child.kill("SIGKILL");
|
|
1796
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1797
|
+
},
|
|
1798
|
+
Math.max(0, opts.timeoutMs)
|
|
1799
|
+
);
|
|
1800
|
+
});
|
|
1454
1801
|
}
|
|
1455
|
-
|
|
1456
|
-
|
|
1802
|
+
|
|
1803
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1804
|
+
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
1805
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
|
|
1806
|
+
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
1807
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
|
|
1808
|
+
function commandError(result) {
|
|
1809
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
1810
|
+
}
|
|
1811
|
+
function reportRecord(stage, outcome, reason, litestreamExitCode, options) {
|
|
1812
|
+
options.reportRecovery({
|
|
1813
|
+
v: 1,
|
|
1814
|
+
event: "session_db_recovery",
|
|
1815
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1816
|
+
stage,
|
|
1817
|
+
outcome,
|
|
1818
|
+
severity: "error",
|
|
1819
|
+
reason,
|
|
1820
|
+
litestream_exit_code: litestreamExitCode,
|
|
1821
|
+
attempt: null,
|
|
1822
|
+
replica_objects: null,
|
|
1823
|
+
replica_bytes: null,
|
|
1824
|
+
quarantine_destination: null,
|
|
1825
|
+
quarantined_objects: null,
|
|
1826
|
+
quarantine_failed_objects: null,
|
|
1827
|
+
quarantined_bytes: null,
|
|
1828
|
+
verified_restore_point: null,
|
|
1829
|
+
restore_points_tried: null,
|
|
1830
|
+
provenance_reason: null,
|
|
1831
|
+
provenance_migration_delta: null,
|
|
1832
|
+
replication_suspended: stage === "restore"
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
function clearMarker(options) {
|
|
1836
|
+
if (!options.noReplicateMarker) return;
|
|
1457
1837
|
try {
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
})
|
|
1463
|
-
|
|
1838
|
+
unlinkSync2(options.noReplicateMarker);
|
|
1839
|
+
} catch (error2) {
|
|
1840
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1841
|
+
options.log(
|
|
1842
|
+
`Could not clear the previous session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1843
|
+
"warn"
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
function markNoReplicate(options, message) {
|
|
1848
|
+
if (options.noReplicateMarker) {
|
|
1849
|
+
try {
|
|
1850
|
+
mkdirSync(dirname2(options.noReplicateMarker), { recursive: true });
|
|
1851
|
+
writeFileSync(options.noReplicateMarker, "");
|
|
1852
|
+
} catch (error2) {
|
|
1853
|
+
options.log(
|
|
1854
|
+
`Could not write session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1855
|
+
"error"
|
|
1856
|
+
);
|
|
1464
1857
|
}
|
|
1465
|
-
} catch {
|
|
1466
1858
|
}
|
|
1467
|
-
|
|
1859
|
+
options.log(`SESSION-DB-NO-REPLICATE: ${message}`, "warn");
|
|
1468
1860
|
}
|
|
1469
|
-
function
|
|
1470
|
-
for (
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1861
|
+
function discardSessionDbDebris(options) {
|
|
1862
|
+
for (const path of [options.dbPath, `${options.dbPath}-wal`, `${options.dbPath}-shm`]) {
|
|
1863
|
+
try {
|
|
1864
|
+
unlinkSync2(path);
|
|
1865
|
+
} catch (error2) {
|
|
1866
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") continue;
|
|
1867
|
+
options.log(
|
|
1868
|
+
`Could not remove session-DB debris ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1869
|
+
"warn"
|
|
1870
|
+
);
|
|
1474
1871
|
}
|
|
1475
1872
|
}
|
|
1476
|
-
return null;
|
|
1477
1873
|
}
|
|
1478
|
-
function
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
const psOutput = execSync('ps aux | grep -E "opencode (serve|--port)" | grep -v grep', {
|
|
1495
|
-
encoding: "utf-8",
|
|
1496
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1497
|
-
}).trim();
|
|
1498
|
-
if (psOutput) {
|
|
1499
|
-
for (const line of psOutput.split("\n")) {
|
|
1500
|
-
const parts = line.trim().split(/\s+/);
|
|
1501
|
-
if (parts.length >= 2) {
|
|
1502
|
-
const pid = parseInt(parts[1], 10);
|
|
1503
|
-
if (!isNaN(pid)) pids.push(pid);
|
|
1504
|
-
}
|
|
1505
|
-
}
|
|
1506
|
-
}
|
|
1507
|
-
} catch (err) {
|
|
1508
|
-
console.warn(
|
|
1509
|
-
`findOpenCodeProcesses: ps fallback failed: ${err instanceof Error ? err.message : String(err)}`
|
|
1510
|
-
);
|
|
1511
|
-
}
|
|
1512
|
-
}
|
|
1513
|
-
for (const pid of pids) {
|
|
1514
|
-
try {
|
|
1515
|
-
const lsofOutput = execSync(`lsof -Pan -p ${pid} -i TCP -sTCP:LISTEN 2>/dev/null`, {
|
|
1516
|
-
encoding: "utf-8",
|
|
1517
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1518
|
-
}).trim();
|
|
1519
|
-
for (const line of lsofOutput.split("\n")) {
|
|
1520
|
-
const portMatch = line.match(/:(\d+)\s+\(LISTEN\)/);
|
|
1521
|
-
if (portMatch) {
|
|
1522
|
-
const port = parseInt(portMatch[1], 10);
|
|
1523
|
-
if (!isNaN(port) && !instances.some((i) => i.port === port)) {
|
|
1524
|
-
const cwd = getProcessCwd(pid);
|
|
1525
|
-
instances.push({ pid, port, cwd });
|
|
1526
|
-
}
|
|
1527
|
-
}
|
|
1528
|
-
}
|
|
1529
|
-
} catch {
|
|
1530
|
-
}
|
|
1531
|
-
}
|
|
1874
|
+
function splitDiagnostics(text) {
|
|
1875
|
+
return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
1876
|
+
}
|
|
1877
|
+
function logSynchroniserDiagnostics(result, options) {
|
|
1878
|
+
for (const line of splitDiagnostics(result.stderr)) options.log(line, "warn");
|
|
1879
|
+
}
|
|
1880
|
+
function parseSingleQuotedAssignment(line) {
|
|
1881
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
1882
|
+
if (!match || !match[2].startsWith("'")) return null;
|
|
1883
|
+
const valueSource = match[2];
|
|
1884
|
+
let value = "";
|
|
1885
|
+
for (let index = 1; index < valueSource.length; index++) {
|
|
1886
|
+
const character = valueSource[index];
|
|
1887
|
+
if (character !== "'") {
|
|
1888
|
+
value += character;
|
|
1889
|
+
continue;
|
|
1532
1890
|
}
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1891
|
+
if (index === valueSource.length - 1) return [match[1], value];
|
|
1892
|
+
if (valueSource.slice(index + 1, index + 4) !== "\\''") return null;
|
|
1893
|
+
value += "'";
|
|
1894
|
+
index += 3;
|
|
1537
1895
|
}
|
|
1538
|
-
return
|
|
1896
|
+
return null;
|
|
1539
1897
|
}
|
|
1540
|
-
|
|
1541
|
-
const
|
|
1542
|
-
const
|
|
1898
|
+
function parseSynchroniserEnv(stdout) {
|
|
1899
|
+
const values = {};
|
|
1900
|
+
for (const line of stdout.split("\n")) {
|
|
1901
|
+
if (line.trim() === "") continue;
|
|
1902
|
+
const assignment = parseSingleQuotedAssignment(line);
|
|
1903
|
+
if (!assignment) return null;
|
|
1904
|
+
values[assignment[0]] = assignment[1];
|
|
1905
|
+
}
|
|
1906
|
+
return values;
|
|
1907
|
+
}
|
|
1908
|
+
function runCommand(command, args, options) {
|
|
1909
|
+
return new Promise((resolve4) => {
|
|
1910
|
+
let child;
|
|
1911
|
+
let stdout = "";
|
|
1912
|
+
let stderr = "";
|
|
1913
|
+
let settled = false;
|
|
1914
|
+
const finish = (result) => {
|
|
1915
|
+
if (settled) return;
|
|
1916
|
+
settled = true;
|
|
1917
|
+
if (timer) clearTimeout(timer);
|
|
1918
|
+
resolve4(result);
|
|
1919
|
+
};
|
|
1920
|
+
try {
|
|
1921
|
+
child = spawn2(command, args, {
|
|
1922
|
+
env: options.env,
|
|
1923
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1924
|
+
});
|
|
1925
|
+
} catch (error2) {
|
|
1926
|
+
resolve4({
|
|
1927
|
+
code: null,
|
|
1928
|
+
stdout,
|
|
1929
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
1930
|
+
timedOut: false
|
|
1931
|
+
});
|
|
1932
|
+
return;
|
|
1933
|
+
}
|
|
1934
|
+
child.stdout?.setEncoding("utf8");
|
|
1935
|
+
child.stdout?.on("data", (chunk) => {
|
|
1936
|
+
stdout += chunk;
|
|
1937
|
+
});
|
|
1938
|
+
child.stderr?.setEncoding("utf8");
|
|
1939
|
+
child.stderr?.on("data", (chunk) => {
|
|
1940
|
+
stderr += chunk;
|
|
1941
|
+
});
|
|
1942
|
+
child.once("error", (error2) => {
|
|
1943
|
+
finish({
|
|
1944
|
+
code: null,
|
|
1945
|
+
stdout,
|
|
1946
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
1947
|
+
${error2.message}`,
|
|
1948
|
+
timedOut: false
|
|
1949
|
+
});
|
|
1950
|
+
});
|
|
1951
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
1952
|
+
const timer = setTimeout(
|
|
1953
|
+
() => {
|
|
1954
|
+
child.kill("SIGKILL");
|
|
1955
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1956
|
+
},
|
|
1957
|
+
Math.max(0, options.timeoutMs)
|
|
1958
|
+
);
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
async function ensureLitestreamConfig(options, env) {
|
|
1962
|
+
const configPath = options.litestreamConfig;
|
|
1963
|
+
if (!configPath) {
|
|
1964
|
+
markNoReplicate(options, "no Litestream configuration path was provided");
|
|
1965
|
+
reportRecord(
|
|
1966
|
+
"restore",
|
|
1967
|
+
"restore_misconfigured",
|
|
1968
|
+
"litestream_config_unavailable",
|
|
1969
|
+
null,
|
|
1970
|
+
options
|
|
1971
|
+
);
|
|
1972
|
+
return null;
|
|
1973
|
+
}
|
|
1974
|
+
try {
|
|
1975
|
+
if (statSync2(configPath).size > 0) return configPath;
|
|
1976
|
+
} catch (error2) {
|
|
1977
|
+
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
1978
|
+
options.log(
|
|
1979
|
+
`Could not inspect ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1980
|
+
"warn"
|
|
1981
|
+
);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
const rendered = await runSynchroniser(["litestream-config"], {
|
|
1985
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
1986
|
+
env
|
|
1987
|
+
});
|
|
1988
|
+
logSynchroniserDiagnostics(rendered, options);
|
|
1989
|
+
if (rendered.timedOut || rendered.code !== 0) {
|
|
1990
|
+
options.log(
|
|
1991
|
+
`Could not generate ${configPath}: runner-synchroniser litestream-config failed (${commandError(rendered)})`,
|
|
1992
|
+
"error"
|
|
1993
|
+
);
|
|
1994
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
1995
|
+
reportRecord(
|
|
1996
|
+
"restore",
|
|
1997
|
+
"restore_misconfigured",
|
|
1998
|
+
"litestream_config_unavailable",
|
|
1999
|
+
null,
|
|
2000
|
+
options
|
|
2001
|
+
);
|
|
2002
|
+
return null;
|
|
2003
|
+
}
|
|
2004
|
+
try {
|
|
2005
|
+
mkdirSync(dirname2(configPath), { recursive: true });
|
|
2006
|
+
writeFileSync(configPath, rendered.stdout);
|
|
2007
|
+
} catch (error2) {
|
|
2008
|
+
options.log(
|
|
2009
|
+
`Could not write ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2010
|
+
"error"
|
|
2011
|
+
);
|
|
2012
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2013
|
+
reportRecord(
|
|
2014
|
+
"restore",
|
|
2015
|
+
"restore_misconfigured",
|
|
2016
|
+
"litestream_config_unavailable",
|
|
2017
|
+
null,
|
|
2018
|
+
options
|
|
2019
|
+
);
|
|
2020
|
+
return null;
|
|
2021
|
+
}
|
|
2022
|
+
const version2 = await runCommand("litestream", ["version"], {
|
|
2023
|
+
env,
|
|
2024
|
+
timeoutMs: 1e4
|
|
2025
|
+
});
|
|
2026
|
+
const litestreamVersion = version2.code === 0 ? version2.stdout.trim() || "unknown" : "unknown";
|
|
2027
|
+
const regionEmpty = !/^\s*region:\s*\S+/m.test(rendered.stdout);
|
|
2028
|
+
options.log(
|
|
2029
|
+
`litestream ${litestreamVersion}; AWS_REGION=${env.AWS_REGION ?? "<unset>"} AWS_DEFAULT_REGION=${env.AWS_DEFAULT_REGION ?? "<unset>"}; rendered litestream.yml region empty: ${regionEmpty ? "yes" : "no"}`
|
|
2030
|
+
);
|
|
2031
|
+
return configPath;
|
|
2032
|
+
}
|
|
2033
|
+
function restoreGiveUp(options, reason, message, litestreamExitCode, outcome = "fresh_session_db") {
|
|
2034
|
+
discardSessionDbDebris(options);
|
|
2035
|
+
markNoReplicate(options, message);
|
|
2036
|
+
reportRecord("restore", outcome, reason, litestreamExitCode, options);
|
|
2037
|
+
}
|
|
2038
|
+
async function restoreSessionDb(options, configPath, env) {
|
|
2039
|
+
const restored = await runCommand(
|
|
2040
|
+
"litestream",
|
|
2041
|
+
["restore", "-config", configPath, "-if-db-not-exists", "-if-replica-exists", options.dbPath],
|
|
2042
|
+
{ env, timeoutMs: SESSION_DB_RESTORE_TIMEOUT_MS }
|
|
2043
|
+
);
|
|
2044
|
+
for (const line of splitDiagnostics(restored.stderr)) options.log(line, "warn");
|
|
2045
|
+
if (restored.timedOut || restored.code === 124 || restored.code === 137) {
|
|
2046
|
+
restoreGiveUp(
|
|
2047
|
+
options,
|
|
2048
|
+
"restore_deadline_exceeded",
|
|
2049
|
+
`SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within ${SESSION_DB_RESTORE_TIMEOUT_MS}ms; opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2050
|
+
restored.code ?? 124
|
|
2051
|
+
);
|
|
2052
|
+
return;
|
|
2053
|
+
}
|
|
2054
|
+
if (restored.code === null || restored.code === 125 || restored.code === 126 || restored.code === 127) {
|
|
2055
|
+
options.log(`litestream restore could not run (${commandError(restored)})`, "error");
|
|
2056
|
+
restoreGiveUp(
|
|
2057
|
+
options,
|
|
2058
|
+
"restore_tool_unusable",
|
|
2059
|
+
`restore tool is broken (${commandError(restored)}); opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2060
|
+
restored.code
|
|
2061
|
+
);
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
const classified = await runSynchroniser(
|
|
2065
|
+
[
|
|
2066
|
+
"session-db-classify",
|
|
2067
|
+
String(restored.code ?? 1),
|
|
2068
|
+
"1",
|
|
2069
|
+
"--on-unusable-replica=leave",
|
|
2070
|
+
"--fresh-db-fallback"
|
|
2071
|
+
],
|
|
2072
|
+
{ timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS, env }
|
|
2073
|
+
);
|
|
2074
|
+
logSynchroniserDiagnostics(classified, options);
|
|
2075
|
+
const classifyCode = classified.code;
|
|
2076
|
+
switch (classifyCode) {
|
|
2077
|
+
case 0:
|
|
2078
|
+
return;
|
|
2079
|
+
case 31:
|
|
2080
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2081
|
+
options.log(
|
|
2082
|
+
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
2083
|
+
"warn"
|
|
2084
|
+
);
|
|
2085
|
+
return;
|
|
2086
|
+
case 32:
|
|
2087
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2088
|
+
discardSessionDbDebris(options);
|
|
2089
|
+
markNoReplicate(
|
|
2090
|
+
options,
|
|
2091
|
+
"session-db-classify asked for another restore attempt (32), but this boot has budget for only one; treating it as a give-up rather than retrying"
|
|
2092
|
+
);
|
|
2093
|
+
return;
|
|
2094
|
+
case 30:
|
|
2095
|
+
restoreGiveUp(
|
|
2096
|
+
options,
|
|
2097
|
+
"classification_fatal",
|
|
2098
|
+
"session-db-classify returned fatal (30); see the FATAL message above",
|
|
2099
|
+
restored.code,
|
|
2100
|
+
"restore_misconfigured"
|
|
2101
|
+
);
|
|
2102
|
+
return;
|
|
2103
|
+
default:
|
|
2104
|
+
restoreGiveUp(
|
|
2105
|
+
options,
|
|
2106
|
+
"classification_unrecognised",
|
|
2107
|
+
`session-db-classify exited ${classifyCode ?? "null"}, which is none of its documented answers`,
|
|
2108
|
+
restored.code
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
async function verifySessionDb(options, configPath, env) {
|
|
2113
|
+
if (!options.noReplicateMarker || !fileExists(options.noReplicateMarker)) {
|
|
2114
|
+
const result = await runSynchroniser(["session-db-verify", configPath], {
|
|
2115
|
+
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
2116
|
+
env: {
|
|
2117
|
+
...env,
|
|
2118
|
+
// The synchroniser reads this value in SECONDS. Keep this at 120, not
|
|
2119
|
+
// 120_000, so the walkback gives up before the outer process bound.
|
|
2120
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
2121
|
+
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
2122
|
+
)
|
|
2123
|
+
}
|
|
2124
|
+
});
|
|
2125
|
+
logSynchroniserDiagnostics(result, options);
|
|
2126
|
+
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
2127
|
+
options.log(
|
|
2128
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
2129
|
+
"warn"
|
|
2130
|
+
);
|
|
2131
|
+
return false;
|
|
2132
|
+
}
|
|
2133
|
+
if (result.code === 34) {
|
|
2134
|
+
reportRecord(
|
|
2135
|
+
"verify",
|
|
2136
|
+
"session_db_boot_refused",
|
|
2137
|
+
result.stderr.includes("SESSION-DB-LOCAL-DISCARD-FAILED") ? "local_discard_failed" : "replica_separation_unproven",
|
|
2138
|
+
null,
|
|
2139
|
+
options
|
|
2140
|
+
);
|
|
2141
|
+
return true;
|
|
2142
|
+
}
|
|
2143
|
+
if (result.code === 33) {
|
|
2144
|
+
options.log(
|
|
2145
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
2146
|
+
"warn"
|
|
2147
|
+
);
|
|
2148
|
+
return false;
|
|
2149
|
+
}
|
|
2150
|
+
if (result.code !== 0) {
|
|
2151
|
+
options.log(
|
|
2152
|
+
`SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${result.code ?? "null"}, which is none of its documented answers; continuing with the restored opencode.db as-is`,
|
|
2153
|
+
"warn"
|
|
2154
|
+
);
|
|
2155
|
+
}
|
|
2156
|
+
return false;
|
|
2157
|
+
}
|
|
2158
|
+
options.log(
|
|
2159
|
+
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
2160
|
+
"debug"
|
|
2161
|
+
);
|
|
2162
|
+
return false;
|
|
2163
|
+
}
|
|
2164
|
+
function fileExists(path) {
|
|
2165
|
+
try {
|
|
2166
|
+
statSync2(path);
|
|
2167
|
+
return true;
|
|
2168
|
+
} catch (error2) {
|
|
2169
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
2170
|
+
return true;
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
async function restoreAndVerifySessionDb(options) {
|
|
2174
|
+
const env = options.env ?? process.env;
|
|
2175
|
+
clearMarker(options);
|
|
2176
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2177
|
+
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
2178
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2179
|
+
env
|
|
2180
|
+
});
|
|
2181
|
+
logSynchroniserDiagnostics(synchroniserEnv, options);
|
|
2182
|
+
if (synchroniserEnv.timedOut || synchroniserEnv.code !== 0) {
|
|
2183
|
+
options.log(
|
|
2184
|
+
`runner-synchroniser env could not resolve the session-DB configuration (${commandError(synchroniserEnv)})`,
|
|
2185
|
+
"error"
|
|
2186
|
+
);
|
|
2187
|
+
markNoReplicate(
|
|
2188
|
+
options,
|
|
2189
|
+
"could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
2190
|
+
);
|
|
2191
|
+
reportRecord(
|
|
2192
|
+
"restore",
|
|
2193
|
+
"restore_misconfigured",
|
|
2194
|
+
"synchroniser_config_unresolved",
|
|
2195
|
+
null,
|
|
2196
|
+
options
|
|
2197
|
+
);
|
|
2198
|
+
return { verifyFatal: false };
|
|
2199
|
+
}
|
|
2200
|
+
const values = parseSynchroniserEnv(synchroniserEnv.stdout);
|
|
2201
|
+
if (!values) {
|
|
2202
|
+
markNoReplicate(
|
|
2203
|
+
options,
|
|
2204
|
+
"the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
2205
|
+
);
|
|
2206
|
+
reportRecord(
|
|
2207
|
+
"restore",
|
|
2208
|
+
"restore_misconfigured",
|
|
2209
|
+
"synchroniser_config_unevaluable",
|
|
2210
|
+
null,
|
|
2211
|
+
options
|
|
2212
|
+
);
|
|
2213
|
+
return { verifyFatal: false };
|
|
2214
|
+
}
|
|
2215
|
+
const synchroniserDbPath = values.OPENCODE_DB_PATH;
|
|
2216
|
+
if (!synchroniserDbPath) {
|
|
2217
|
+
markNoReplicate(
|
|
2218
|
+
options,
|
|
2219
|
+
"run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
2220
|
+
);
|
|
2221
|
+
reportRecord(
|
|
2222
|
+
"restore",
|
|
2223
|
+
"restore_misconfigured",
|
|
2224
|
+
"synchroniser_config_incomplete",
|
|
2225
|
+
null,
|
|
2226
|
+
options
|
|
2227
|
+
);
|
|
2228
|
+
return { verifyFatal: false };
|
|
2229
|
+
}
|
|
2230
|
+
if (resolvePath(synchroniserDbPath) !== resolvePath(options.dbPath)) {
|
|
2231
|
+
options.log(
|
|
2232
|
+
`runner-synchroniser reported OPENCODE_DB_PATH=${synchroniserDbPath}, but OpenCode uses ${options.dbPath}; continuing with OpenCode's session-DB path`,
|
|
2233
|
+
"warn"
|
|
2234
|
+
);
|
|
2235
|
+
}
|
|
2236
|
+
if (!values.PERSISTENCE_BUCKET) {
|
|
2237
|
+
options.log(
|
|
2238
|
+
"SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated.",
|
|
2239
|
+
"warn"
|
|
2240
|
+
);
|
|
2241
|
+
return { verifyFatal: false };
|
|
2242
|
+
}
|
|
2243
|
+
const configPath = await ensureLitestreamConfig(options, env);
|
|
2244
|
+
if (!configPath) return { verifyFatal: false };
|
|
2245
|
+
await restoreSessionDb(options, configPath, env);
|
|
2246
|
+
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
2247
|
+
return { verifyFatal: false };
|
|
2248
|
+
}
|
|
2249
|
+
return { verifyFatal: await verifySessionDb(options, configPath, env) };
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// src/lib/opencode/session-db-provenance.ts
|
|
2253
|
+
import { createRequire } from "node:module";
|
|
2254
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
|
|
2255
|
+
import { dirname as dirname3, join as join3 } from "node:path";
|
|
2256
|
+
var require2 = createRequire(import.meta.url);
|
|
2257
|
+
function readSessionDbMigrationIds(dbPath) {
|
|
2258
|
+
let db;
|
|
2259
|
+
try {
|
|
2260
|
+
const { DatabaseSync } = require2("node:sqlite");
|
|
2261
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
2262
|
+
const columns = db.prepare("PRAGMA table_info(migration)").all();
|
|
2263
|
+
const hasExpectedShape = columns.length === 2 && columns.some(
|
|
2264
|
+
(column) => column.name === "id" && typeof column.type === "string" && column.type.toUpperCase() === "TEXT" && column.pk === 1
|
|
2265
|
+
) && columns.some(
|
|
2266
|
+
(column) => column.name === "time_completed" && typeof column.type === "string" && column.type.toUpperCase() === "INTEGER" && column.notnull === 1 && column.pk === 0
|
|
2267
|
+
);
|
|
2268
|
+
if (!hasExpectedShape) {
|
|
2269
|
+
console.warn(`[readSessionDbMigrationIds] unsupported migration table shape in ${dbPath}`);
|
|
2270
|
+
return null;
|
|
2271
|
+
}
|
|
2272
|
+
const rows = db.prepare("SELECT id FROM migration ORDER BY id").all();
|
|
2273
|
+
if (rows.some((row) => typeof row.id !== "string")) return null;
|
|
2274
|
+
return rows.map((row) => row.id);
|
|
2275
|
+
} catch (error2) {
|
|
2276
|
+
console.warn(
|
|
2277
|
+
`[readSessionDbMigrationIds] could not read migration history from ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2278
|
+
);
|
|
2279
|
+
return null;
|
|
2280
|
+
} finally {
|
|
2281
|
+
try {
|
|
2282
|
+
db?.close();
|
|
2283
|
+
} catch (error2) {
|
|
2284
|
+
console.warn(
|
|
2285
|
+
`[readSessionDbMigrationIds] could not close ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2286
|
+
);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
2291
|
+
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
2292
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
2293
|
+
}
|
|
2294
|
+
function loadSessionDbProvenanceState(path) {
|
|
2295
|
+
let value;
|
|
2296
|
+
try {
|
|
2297
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
2298
|
+
} catch (error2) {
|
|
2299
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return {};
|
|
2300
|
+
console.error(
|
|
2301
|
+
`[session-db-provenance] could not read ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2302
|
+
);
|
|
2303
|
+
return {};
|
|
2304
|
+
}
|
|
2305
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2306
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2307
|
+
return {};
|
|
2308
|
+
}
|
|
2309
|
+
const state = {};
|
|
2310
|
+
for (const [dbPath, record] of Object.entries(value)) {
|
|
2311
|
+
if (!isSessionDbProvenanceRecord(record)) {
|
|
2312
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2313
|
+
return {};
|
|
2314
|
+
}
|
|
2315
|
+
state[dbPath] = record;
|
|
2316
|
+
}
|
|
2317
|
+
return state;
|
|
2318
|
+
}
|
|
2319
|
+
function saveSessionDbProvenanceState(path, state) {
|
|
2320
|
+
try {
|
|
2321
|
+
mkdirSync2(dirname3(path), { recursive: true });
|
|
2322
|
+
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
2323
|
+
`, "utf8");
|
|
2324
|
+
} catch (error2) {
|
|
2325
|
+
console.error(
|
|
2326
|
+
`[session-db-provenance] could not write ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2327
|
+
);
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
function evaluateSessionDbProvenance(input) {
|
|
2331
|
+
const { currentVersion, currentIds, previous } = input;
|
|
2332
|
+
if (!previous) return { anomaly: false, reason: null };
|
|
2333
|
+
const current = new Set(currentIds);
|
|
2334
|
+
const prior = new Set(previous.migrationIds);
|
|
2335
|
+
for (const id of prior) {
|
|
2336
|
+
if (!current.has(id)) return { anomaly: true, reason: "migration-history-regressed" };
|
|
2337
|
+
}
|
|
2338
|
+
if (current.size > prior.size && previous.opencodeVersion === currentVersion) {
|
|
2339
|
+
return { anomaly: true, reason: "foreign-version-migrations" };
|
|
2340
|
+
}
|
|
2341
|
+
return { anomaly: false, reason: null };
|
|
2342
|
+
}
|
|
2343
|
+
function checkSessionDbProvenance(input) {
|
|
2344
|
+
const { dbPath, currentVersion, homeDir, env } = input;
|
|
2345
|
+
const path = sessionDbProvenanceStatePath(homeDir, env);
|
|
2346
|
+
const state = loadSessionDbProvenanceState(path);
|
|
2347
|
+
const previous = state[dbPath];
|
|
2348
|
+
const currentIds = readSessionDbMigrationIds(dbPath);
|
|
2349
|
+
if (currentIds === null) {
|
|
2350
|
+
return {
|
|
2351
|
+
anomaly: false,
|
|
2352
|
+
reason: null,
|
|
2353
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2354
|
+
migrationDelta: null
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
const decision = evaluateSessionDbProvenance({ currentVersion, currentIds, previous });
|
|
2358
|
+
const migrationDelta = previous ? new Set(currentIds).size - new Set(previous.migrationIds).size : null;
|
|
2359
|
+
state[dbPath] = {
|
|
2360
|
+
opencodeVersion: currentVersion,
|
|
2361
|
+
migrationIds: currentIds,
|
|
2362
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2363
|
+
};
|
|
2364
|
+
saveSessionDbProvenanceState(path, state);
|
|
2365
|
+
return {
|
|
2366
|
+
...decision,
|
|
2367
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2368
|
+
migrationDelta
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
function isSessionDbProvenanceRecord(value) {
|
|
2372
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2373
|
+
const record = value;
|
|
2374
|
+
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
// src/lib/opencode/opencode-version-gate.ts
|
|
2378
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
2379
|
+
function isQueueValidatedVersion(version2) {
|
|
2380
|
+
if (!version2) return false;
|
|
2381
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
2382
|
+
}
|
|
2383
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
2384
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
2385
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
2386
|
+
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
2387
|
+
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
// src/lib/opencode/process.ts
|
|
2391
|
+
import { execSync, spawn as spawn3 } from "child_process";
|
|
2392
|
+
|
|
2393
|
+
// src/lib/process-stop.ts
|
|
2394
|
+
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
2395
|
+
if (!child.pid) {
|
|
2396
|
+
return { outcome: "not-running", code: child.exitCode, signal: child.signalCode };
|
|
2397
|
+
}
|
|
2398
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
2399
|
+
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
2400
|
+
}
|
|
2401
|
+
return new Promise((resolve4, reject) => {
|
|
2402
|
+
let forced = false;
|
|
2403
|
+
let settled = false;
|
|
2404
|
+
const timer = setTimeout(() => {
|
|
2405
|
+
forced = true;
|
|
2406
|
+
try {
|
|
2407
|
+
sendKill();
|
|
2408
|
+
} catch (error2) {
|
|
2409
|
+
if (error2.code === "ESRCH") {
|
|
2410
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2411
|
+
} else {
|
|
2412
|
+
fail(error2);
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
}, timeoutMs);
|
|
2416
|
+
const finish = (result) => {
|
|
2417
|
+
if (settled) return;
|
|
2418
|
+
settled = true;
|
|
2419
|
+
clearTimeout(timer);
|
|
2420
|
+
child.removeListener("exit", onExit);
|
|
2421
|
+
resolve4(result);
|
|
2422
|
+
};
|
|
2423
|
+
const fail = (error2) => {
|
|
2424
|
+
if (settled) return;
|
|
2425
|
+
settled = true;
|
|
2426
|
+
clearTimeout(timer);
|
|
2427
|
+
child.removeListener("exit", onExit);
|
|
2428
|
+
reject(error2);
|
|
2429
|
+
};
|
|
2430
|
+
const onExit = (code, signal) => {
|
|
2431
|
+
finish({ outcome: forced ? "killed" : "exited", code, signal });
|
|
2432
|
+
};
|
|
2433
|
+
child.once("exit", onExit);
|
|
2434
|
+
try {
|
|
2435
|
+
sendTerm();
|
|
2436
|
+
} catch (error2) {
|
|
2437
|
+
if (error2.code === "ESRCH") {
|
|
2438
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2439
|
+
} else {
|
|
2440
|
+
fail(error2);
|
|
2441
|
+
}
|
|
2442
|
+
return;
|
|
2443
|
+
}
|
|
2444
|
+
});
|
|
2445
|
+
}
|
|
2446
|
+
|
|
2447
|
+
// src/lib/opencode/process.ts
|
|
2448
|
+
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
2449
|
+
function getProcessCwd(pid) {
|
|
2450
|
+
const platform = process.platform;
|
|
2451
|
+
try {
|
|
2452
|
+
if (platform === "darwin") {
|
|
2453
|
+
const output = execSync(`lsof -a -p ${pid} -d cwd -Fn 2>/dev/null`, {
|
|
2454
|
+
encoding: "utf-8",
|
|
2455
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2456
|
+
}).trim();
|
|
2457
|
+
const lines = output.split("\n");
|
|
2458
|
+
for (const line of lines) {
|
|
2459
|
+
if (line.startsWith("n") && !line.startsWith("n ")) {
|
|
2460
|
+
return line.slice(1);
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
} else if (platform === "linux") {
|
|
2464
|
+
const output = execSync(`readlink /proc/${pid}/cwd 2>/dev/null`, {
|
|
2465
|
+
encoding: "utf-8",
|
|
2466
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2467
|
+
}).trim();
|
|
2468
|
+
if (output) return output;
|
|
2469
|
+
}
|
|
2470
|
+
} catch {
|
|
2471
|
+
}
|
|
2472
|
+
return void 0;
|
|
2473
|
+
}
|
|
2474
|
+
function isPortInUse(port) {
|
|
2475
|
+
const platform = process.platform;
|
|
2476
|
+
try {
|
|
2477
|
+
if (platform === "darwin" || platform === "linux") {
|
|
2478
|
+
execSync(`lsof -i :${port} -sTCP:LISTEN 2>/dev/null`, {
|
|
2479
|
+
encoding: "utf-8",
|
|
2480
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2481
|
+
});
|
|
2482
|
+
return true;
|
|
2483
|
+
}
|
|
2484
|
+
} catch {
|
|
2485
|
+
}
|
|
2486
|
+
return false;
|
|
2487
|
+
}
|
|
2488
|
+
function findAvailablePort(startPort, maxAttempts = 10) {
|
|
2489
|
+
for (let i = 0; i < maxAttempts; i++) {
|
|
2490
|
+
const port = startPort + i;
|
|
2491
|
+
if (!isPortInUse(port)) {
|
|
2492
|
+
return port;
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
return null;
|
|
2496
|
+
}
|
|
2497
|
+
function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
2498
|
+
const instances = [];
|
|
2499
|
+
try {
|
|
2500
|
+
const platform = process.platform;
|
|
2501
|
+
if (platform === "darwin" || platform === "linux") {
|
|
2502
|
+
let pids = [];
|
|
2503
|
+
try {
|
|
2504
|
+
const pgrepOutput = execSync(`pgrep -f "${pgrepPattern}"`, {
|
|
2505
|
+
encoding: "utf-8",
|
|
2506
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2507
|
+
}).trim();
|
|
2508
|
+
if (pgrepOutput) {
|
|
2509
|
+
pids = pgrepOutput.split("\n").map((p) => parseInt(p.trim(), 10)).filter((p) => !isNaN(p));
|
|
2510
|
+
}
|
|
2511
|
+
} catch {
|
|
2512
|
+
try {
|
|
2513
|
+
const psOutput = execSync(`ps aux | grep -E "${psPattern}" | grep -v grep`, {
|
|
2514
|
+
encoding: "utf-8",
|
|
2515
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2516
|
+
}).trim();
|
|
2517
|
+
if (psOutput) {
|
|
2518
|
+
for (const line of psOutput.split("\n")) {
|
|
2519
|
+
const parts = line.trim().split(/\s+/);
|
|
2520
|
+
if (parts.length >= 2) {
|
|
2521
|
+
const pid = parseInt(parts[1], 10);
|
|
2522
|
+
if (!isNaN(pid)) pids.push(pid);
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
} catch (err) {
|
|
2527
|
+
console.warn(
|
|
2528
|
+
`findOpenCodeProcesses: ps fallback failed: ${err instanceof Error ? err.message : String(err)}`
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
for (const pid of pids) {
|
|
2533
|
+
try {
|
|
2534
|
+
const lsofOutput = execSync(`lsof -Pan -p ${pid} -i TCP -sTCP:LISTEN 2>/dev/null`, {
|
|
2535
|
+
encoding: "utf-8",
|
|
2536
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
2537
|
+
}).trim();
|
|
2538
|
+
for (const line of lsofOutput.split("\n")) {
|
|
2539
|
+
const portMatch = line.match(/:(\d+)\s+\(LISTEN\)/);
|
|
2540
|
+
if (portMatch) {
|
|
2541
|
+
const port = parseInt(portMatch[1], 10);
|
|
2542
|
+
if (!isNaN(port) && !instances.some((i) => i.port === port)) {
|
|
2543
|
+
const cwd = getProcessCwd(pid);
|
|
2544
|
+
instances.push({ pid, port, cwd });
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
} catch {
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2551
|
+
}
|
|
2552
|
+
} catch (err) {
|
|
2553
|
+
console.warn(
|
|
2554
|
+
`findOpenCodeProcesses: process detection failed: ${err instanceof Error ? err.message : String(err)}`
|
|
2555
|
+
);
|
|
2556
|
+
}
|
|
2557
|
+
return instances;
|
|
2558
|
+
}
|
|
2559
|
+
function findOpenCodeProcesses() {
|
|
2560
|
+
return findProcessesByPattern("opencode serve|opencode-serve", "opencode (serve|--port)");
|
|
2561
|
+
}
|
|
2562
|
+
async function scanPortsForOpenCode() {
|
|
2563
|
+
const instances = [];
|
|
2564
|
+
const checks = OPENCODE_PORT_RANGE.map(async (port) => {
|
|
1543
2565
|
const health = await checkOpenCodeHealth(port);
|
|
1544
2566
|
if (health.healthy) {
|
|
1545
2567
|
let pid = 0;
|
|
@@ -1581,39 +2603,45 @@ async function findHealthyOpenCodeInstances() {
|
|
|
1581
2603
|
}
|
|
1582
2604
|
return healthy;
|
|
1583
2605
|
}
|
|
1584
|
-
async function startOpenCode(port) {
|
|
2606
|
+
async function startOpenCode(port, options = {}) {
|
|
1585
2607
|
let command = "opencode";
|
|
1586
|
-
|
|
2608
|
+
const printLogs = options.inheritStdio ? ["--print-logs"] : [];
|
|
2609
|
+
let args = ["serve", "--port", port.toString(), "--hostname", "127.0.0.1", ...printLogs];
|
|
1587
2610
|
try {
|
|
1588
2611
|
execSync("which opencode", { stdio: "ignore" });
|
|
1589
2612
|
} catch {
|
|
1590
2613
|
command = "npx";
|
|
1591
|
-
args = [
|
|
1592
|
-
|
|
1593
|
-
|
|
2614
|
+
args = [
|
|
2615
|
+
"opencode",
|
|
2616
|
+
"serve",
|
|
2617
|
+
"--port",
|
|
2618
|
+
port.toString(),
|
|
2619
|
+
"--hostname",
|
|
2620
|
+
"127.0.0.1",
|
|
2621
|
+
...printLogs
|
|
2622
|
+
];
|
|
2623
|
+
}
|
|
2624
|
+
const child = spawn3(command, args, {
|
|
1594
2625
|
detached: true,
|
|
1595
|
-
stdio: "ignore",
|
|
2626
|
+
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
1596
2627
|
cwd: process.cwd()
|
|
1597
2628
|
});
|
|
1598
2629
|
return child;
|
|
1599
2630
|
}
|
|
1600
|
-
function
|
|
1601
|
-
|
|
1602
|
-
return;
|
|
1603
|
-
}
|
|
1604
|
-
try {
|
|
2631
|
+
function stopOpenCodeAndWait(opencodeProcess, timeoutMs) {
|
|
2632
|
+
const sendSignal = (signal) => {
|
|
1605
2633
|
if (process.platform === "win32") {
|
|
1606
|
-
opencodeProcess.kill(
|
|
2634
|
+
opencodeProcess.kill(signal);
|
|
1607
2635
|
} else {
|
|
1608
|
-
process.kill(-opencodeProcess.pid,
|
|
1609
|
-
}
|
|
1610
|
-
} catch (err) {
|
|
1611
|
-
if (err.code !== "ESRCH") {
|
|
1612
|
-
console.warn(
|
|
1613
|
-
`stopOpenCode: kill failed: ${err instanceof Error ? err.message : String(err)}`
|
|
1614
|
-
);
|
|
2636
|
+
process.kill(-opencodeProcess.pid, signal);
|
|
1615
2637
|
}
|
|
1616
|
-
}
|
|
2638
|
+
};
|
|
2639
|
+
return stopProcessAndWait(
|
|
2640
|
+
opencodeProcess,
|
|
2641
|
+
timeoutMs,
|
|
2642
|
+
() => sendSignal("SIGTERM"),
|
|
2643
|
+
() => sendSignal("SIGKILL")
|
|
2644
|
+
);
|
|
1617
2645
|
}
|
|
1618
2646
|
|
|
1619
2647
|
// src/lib/opencode/install.ts
|
|
@@ -1625,9 +2653,22 @@ function isOpenCodeInstalled() {
|
|
|
1625
2653
|
try {
|
|
1626
2654
|
const platform = process.platform;
|
|
1627
2655
|
if (platform === "win32") {
|
|
1628
|
-
execSync2("where opencode", { stdio: "ignore" });
|
|
2656
|
+
execSync2("where opencode", { stdio: "ignore" });
|
|
2657
|
+
} else {
|
|
2658
|
+
execSync2("which opencode", { stdio: "ignore" });
|
|
2659
|
+
}
|
|
2660
|
+
return true;
|
|
2661
|
+
} catch {
|
|
2662
|
+
return false;
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
function isOpenCode2Installed() {
|
|
2666
|
+
try {
|
|
2667
|
+
const platform = process.platform;
|
|
2668
|
+
if (platform === "win32") {
|
|
2669
|
+
execSync2("where opencode2", { stdio: "ignore" });
|
|
1629
2670
|
} else {
|
|
1630
|
-
execSync2("which
|
|
2671
|
+
execSync2("which opencode2", { stdio: "ignore" });
|
|
1631
2672
|
}
|
|
1632
2673
|
return true;
|
|
1633
2674
|
} catch {
|
|
@@ -1643,7 +2684,11 @@ async function promptOpenCodeInstall(interactive) {
|
|
|
1643
2684
|
install_url: OPENCODE_INSTALL_URL,
|
|
1644
2685
|
install_commands: {
|
|
1645
2686
|
npm: "npm install -g opencode-ai",
|
|
1646
|
-
curl: "curl -fsSL https://opencode.ai/install.sh | sh"
|
|
2687
|
+
curl: "curl -fsSL https://opencode.ai/install.sh | sh",
|
|
2688
|
+
v2: {
|
|
2689
|
+
npm: "npm install -g @opencode-ai/cli@beta",
|
|
2690
|
+
curl: "curl -fsSL https://opencode.ai/v2/install | bash"
|
|
2691
|
+
}
|
|
1647
2692
|
}
|
|
1648
2693
|
})
|
|
1649
2694
|
);
|
|
@@ -1900,6 +2945,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1900
2945
|
return data.id;
|
|
1901
2946
|
}
|
|
1902
2947
|
async function getModelAttachmentCapability(port, model) {
|
|
2948
|
+
const { model: baseModel } = splitModelVariant(model);
|
|
1903
2949
|
try {
|
|
1904
2950
|
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
1905
2951
|
if (!res.ok) {
|
|
@@ -1916,9 +2962,9 @@ async function getModelAttachmentCapability(port, model) {
|
|
|
1916
2962
|
);
|
|
1917
2963
|
return null;
|
|
1918
2964
|
}
|
|
1919
|
-
const slash =
|
|
1920
|
-
const providerId = slash > 0 ?
|
|
1921
|
-
let modelId = slash > 0 ?
|
|
2965
|
+
const slash = baseModel ? baseModel.indexOf("/") : -1;
|
|
2966
|
+
const providerId = slash > 0 ? baseModel.slice(0, slash) : void 0;
|
|
2967
|
+
let modelId = slash > 0 ? baseModel.slice(slash + 1) : void 0;
|
|
1922
2968
|
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
1923
2969
|
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
1924
2970
|
if (!provider && !providerId) {
|
|
@@ -1998,6 +3044,29 @@ async function buildFileParts(attachments, capable) {
|
|
|
1998
3044
|
}
|
|
1999
3045
|
return { parts, outcomes, capabilityUnknown };
|
|
2000
3046
|
}
|
|
3047
|
+
function splitModelVariant(raw) {
|
|
3048
|
+
const value = raw?.trim();
|
|
3049
|
+
if (!value) return {};
|
|
3050
|
+
const hashIndex = value.indexOf("#");
|
|
3051
|
+
if (hashIndex === -1) return { model: value };
|
|
3052
|
+
const model = value.slice(0, hashIndex).trim() || void 0;
|
|
3053
|
+
const variant = value.slice(hashIndex + 1).trim() || void 0;
|
|
3054
|
+
return { model, variant };
|
|
3055
|
+
}
|
|
3056
|
+
function applyModelOptions(body, options) {
|
|
3057
|
+
if (options?.agent) body.agent = options.agent;
|
|
3058
|
+
const { model, variant } = splitModelVariant(options?.model);
|
|
3059
|
+
if (model) {
|
|
3060
|
+
const slashIndex = model.indexOf("/");
|
|
3061
|
+
if (slashIndex !== -1) {
|
|
3062
|
+
body.model = {
|
|
3063
|
+
providerID: model.substring(0, slashIndex),
|
|
3064
|
+
modelID: model.substring(slashIndex + 1)
|
|
3065
|
+
};
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
if (variant) body.variant = variant;
|
|
3069
|
+
}
|
|
2001
3070
|
function messageText(m) {
|
|
2002
3071
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2003
3072
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
@@ -2022,18 +3091,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2022
3091
|
const body = {
|
|
2023
3092
|
parts
|
|
2024
3093
|
};
|
|
2025
|
-
|
|
2026
|
-
body.agent = options.agent;
|
|
2027
|
-
}
|
|
2028
|
-
if (options?.model) {
|
|
2029
|
-
const slashIndex = options.model.indexOf("/");
|
|
2030
|
-
if (slashIndex !== -1) {
|
|
2031
|
-
body.model = {
|
|
2032
|
-
providerID: options.model.substring(0, slashIndex),
|
|
2033
|
-
modelID: options.model.substring(slashIndex + 1)
|
|
2034
|
-
};
|
|
2035
|
-
}
|
|
2036
|
-
}
|
|
3094
|
+
applyModelOptions(body, options);
|
|
2037
3095
|
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
2038
3096
|
method: "POST",
|
|
2039
3097
|
headers: { "Content-Type": "application/json" },
|
|
@@ -2041,7 +3099,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2041
3099
|
});
|
|
2042
3100
|
if (res.status < 200 || res.status >= 300) {
|
|
2043
3101
|
const text = await res.text().catch(() => "");
|
|
2044
|
-
|
|
3102
|
+
const { variant } = splitModelVariant(options?.model);
|
|
3103
|
+
throw new Error(
|
|
3104
|
+
`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}${variant ? ` (variant: ${variant})` : ""}`
|
|
3105
|
+
);
|
|
2045
3106
|
}
|
|
2046
3107
|
const READ_BACK_ATTEMPTS = 5;
|
|
2047
3108
|
const READ_BACK_DELAY_MS = 150;
|
|
@@ -2065,7 +3126,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2065
3126
|
}
|
|
2066
3127
|
}
|
|
2067
3128
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2068
|
-
await new Promise((
|
|
3129
|
+
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
2069
3130
|
}
|
|
2070
3131
|
}
|
|
2071
3132
|
return null;
|
|
@@ -2111,6 +3172,44 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2111
3172
|
}
|
|
2112
3173
|
return lastOk ?? last;
|
|
2113
3174
|
}
|
|
3175
|
+
function collectSubagentSessions(messages, userMessageId) {
|
|
3176
|
+
if (!messages || messages.length === 0) return [];
|
|
3177
|
+
const byParent = messages.filter(
|
|
3178
|
+
(message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
|
|
3179
|
+
);
|
|
3180
|
+
const assistants = byParent.length > 0 ? byParent : [];
|
|
3181
|
+
if (assistants.length === 0) {
|
|
3182
|
+
const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
|
|
3183
|
+
if (userIndex === -1) return [];
|
|
3184
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
3185
|
+
const message = messages[i];
|
|
3186
|
+
if (roleOf(message) === "user") break;
|
|
3187
|
+
if (roleOf(message) === "assistant") assistants.push(message);
|
|
3188
|
+
}
|
|
3189
|
+
}
|
|
3190
|
+
const refs = [];
|
|
3191
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3192
|
+
for (const message of assistants) {
|
|
3193
|
+
const parts = Array.isArray(message.parts) ? message.parts : [];
|
|
3194
|
+
for (const part of parts) {
|
|
3195
|
+
if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
|
|
3196
|
+
continue;
|
|
3197
|
+
const state = part.state;
|
|
3198
|
+
if (!state || typeof state !== "object") continue;
|
|
3199
|
+
const metadata = state.metadata;
|
|
3200
|
+
if (!metadata || typeof metadata !== "object") continue;
|
|
3201
|
+
const sessionId = metadata.sessionId;
|
|
3202
|
+
if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
|
|
3203
|
+
seen.add(sessionId);
|
|
3204
|
+
const start = state.time?.start;
|
|
3205
|
+
refs.push({
|
|
3206
|
+
sessionId,
|
|
3207
|
+
startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
|
|
3208
|
+
});
|
|
3209
|
+
}
|
|
3210
|
+
}
|
|
3211
|
+
return refs;
|
|
3212
|
+
}
|
|
2114
3213
|
function messageUsage(messages, userMessageId) {
|
|
2115
3214
|
if (!messages || messages.length === 0) return null;
|
|
2116
3215
|
const byParentAll = messages.filter(
|
|
@@ -2196,7 +3295,7 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
2196
3295
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
2197
3296
|
}
|
|
2198
3297
|
function isB2AbandonmentConfirmed(params) {
|
|
2199
|
-
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
3298
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false && params.rootOngoing === false;
|
|
2200
3299
|
}
|
|
2201
3300
|
function isAmbiguousTerminalFinish(m) {
|
|
2202
3301
|
if (completedOf(m) == null) return false;
|
|
@@ -2209,7 +3308,7 @@ function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
|
|
|
2209
3308
|
return isAmbiguousTerminalFinish(reply);
|
|
2210
3309
|
}
|
|
2211
3310
|
function isAmbiguousFinishResolved(params) {
|
|
2212
|
-
return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
|
|
3311
|
+
return params.sessionOngoing === false || params.sessionOngoing !== true && params.pinnedForMs >= params.maxPinnedMs;
|
|
2213
3312
|
}
|
|
2214
3313
|
function messageError(messages, userMessageId) {
|
|
2215
3314
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2239,8 +3338,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2239
3338
|
}
|
|
2240
3339
|
return false;
|
|
2241
3340
|
}
|
|
2242
|
-
function
|
|
2243
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3341
|
+
function classifyReplyAuthError(reply) {
|
|
2244
3342
|
const error2 = errorOf(reply);
|
|
2245
3343
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2246
3344
|
const e = error2;
|
|
@@ -2265,6 +3363,32 @@ function messageFailure(messages, userMessageId) {
|
|
|
2265
3363
|
}
|
|
2266
3364
|
return null;
|
|
2267
3365
|
}
|
|
3366
|
+
function messageFailure(messages, userMessageId) {
|
|
3367
|
+
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
3368
|
+
}
|
|
3369
|
+
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
3370
|
+
if (!messages || messages.length === 0) return null;
|
|
3371
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3372
|
+
const message = messages[i];
|
|
3373
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3374
|
+
const created = createdOf(message);
|
|
3375
|
+
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
3376
|
+
const failure = classifyReplyAuthError(message);
|
|
3377
|
+
if (failure) {
|
|
3378
|
+
if (!failure.providerId) return null;
|
|
3379
|
+
return { providerId: failure.providerId, outcome: "failed", failure };
|
|
3380
|
+
}
|
|
3381
|
+
const providerId = message.info?.providerID;
|
|
3382
|
+
if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
|
|
3383
|
+
return { providerId, outcome: "succeeded" };
|
|
3384
|
+
}
|
|
3385
|
+
return null;
|
|
3386
|
+
}
|
|
3387
|
+
return null;
|
|
3388
|
+
}
|
|
3389
|
+
function findSubagentAuthOutcome(messages, sinceMs) {
|
|
3390
|
+
return findLatestSubagentAuthOutcome(messages, sinceMs);
|
|
3391
|
+
}
|
|
2268
3392
|
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2269
3393
|
if (classified != null) return classified;
|
|
2270
3394
|
if (hasConfiguredProvider !== false) return null;
|
|
@@ -2418,13 +3542,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2418
3542
|
}
|
|
2419
3543
|
|
|
2420
3544
|
// src/lib/opencode/session-db-size.ts
|
|
2421
|
-
import { statSync as
|
|
2422
|
-
import { join as
|
|
3545
|
+
import { statSync as statSync3 } from "node:fs";
|
|
3546
|
+
import { join as join4 } from "node:path";
|
|
2423
3547
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2424
3548
|
function statSessionDbBytes(homeDir) {
|
|
2425
|
-
const dbPath =
|
|
3549
|
+
const dbPath = join4(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2426
3550
|
try {
|
|
2427
|
-
return
|
|
3551
|
+
return statSync3(dbPath).size;
|
|
2428
3552
|
} catch (err) {
|
|
2429
3553
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2430
3554
|
if (!isMissingFile) {
|
|
@@ -2450,11 +3574,15 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
2450
3574
|
}
|
|
2451
3575
|
|
|
2452
3576
|
// src/lib/opencode/session-db-reclaim.ts
|
|
2453
|
-
import { statSync as
|
|
2454
|
-
import { dirname as
|
|
3577
|
+
import { statSync as statSync4, statfsSync } from "node:fs";
|
|
3578
|
+
import { dirname as dirname4 } from "node:path";
|
|
3579
|
+
function errorMessage(error2) {
|
|
3580
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
3581
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
3582
|
+
}
|
|
2455
3583
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2456
3584
|
try {
|
|
2457
|
-
const fsStats = statfsSync(
|
|
3585
|
+
const fsStats = statfsSync(dirname4(dbPath));
|
|
2458
3586
|
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2459
3587
|
if (availableBytes < requiredBytes) {
|
|
2460
3588
|
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
@@ -2477,17 +3605,17 @@ async function probeReclaimAvailability(input) {
|
|
|
2477
3605
|
const { dbPath, requiredBytes } = input;
|
|
2478
3606
|
let sqlite;
|
|
2479
3607
|
try {
|
|
2480
|
-
sqlite = await import("sqlite");
|
|
3608
|
+
sqlite = await import("node:sqlite");
|
|
2481
3609
|
} catch (err) {
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
return "sqlite-unavailable";
|
|
3610
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
3611
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
3612
|
+
return { reason: "sqlite-unavailable", detail };
|
|
2486
3613
|
}
|
|
2487
3614
|
let autoVacuum = null;
|
|
2488
3615
|
try {
|
|
2489
3616
|
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
2490
3617
|
try {
|
|
3618
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
2491
3619
|
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2492
3620
|
} finally {
|
|
2493
3621
|
db.close();
|
|
@@ -2498,23 +3626,25 @@ async function probeReclaimAvailability(input) {
|
|
|
2498
3626
|
);
|
|
2499
3627
|
}
|
|
2500
3628
|
if (autoVacuum !== 0) return null;
|
|
2501
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? "insufficient-disk-space" : null;
|
|
3629
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
2502
3630
|
}
|
|
2503
3631
|
async function reclaimSessionDbSpace(input) {
|
|
2504
3632
|
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
2505
3633
|
let sqlite;
|
|
2506
3634
|
try {
|
|
2507
|
-
sqlite = await import("sqlite");
|
|
3635
|
+
sqlite = await import("node:sqlite");
|
|
2508
3636
|
} catch (err) {
|
|
3637
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
2509
3638
|
console.warn(
|
|
2510
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${
|
|
3639
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
2511
3640
|
);
|
|
2512
|
-
return { ok: false, skipped: "sqlite-unavailable" };
|
|
3641
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
2513
3642
|
}
|
|
2514
3643
|
const { DatabaseSync } = sqlite;
|
|
2515
3644
|
let db;
|
|
2516
3645
|
try {
|
|
2517
3646
|
db = new DatabaseSync(dbPath);
|
|
3647
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
2518
3648
|
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2519
3649
|
if (autoVacuum === 0) {
|
|
2520
3650
|
if (!allowFullVacuum) {
|
|
@@ -2523,7 +3653,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2523
3653
|
);
|
|
2524
3654
|
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2525
3655
|
}
|
|
2526
|
-
const fileBytesForGuard =
|
|
3656
|
+
const fileBytesForGuard = statSync4(dbPath).size;
|
|
2527
3657
|
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2528
3658
|
if (skipReason !== null) {
|
|
2529
3659
|
console.warn(
|
|
@@ -2551,10 +3681,12 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2551
3681
|
);
|
|
2552
3682
|
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
2553
3683
|
} catch (err) {
|
|
2554
|
-
console.error(
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
3684
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
3685
|
+
return {
|
|
3686
|
+
ok: false,
|
|
3687
|
+
skipped: "reclaim-error",
|
|
3688
|
+
detail: errorMessage(err)
|
|
3689
|
+
};
|
|
2558
3690
|
} finally {
|
|
2559
3691
|
db?.close();
|
|
2560
3692
|
}
|
|
@@ -2595,7 +3727,6 @@ var StreamForwarder = class {
|
|
|
2595
3727
|
handleFrame(frame) {
|
|
2596
3728
|
switch (frame.type) {
|
|
2597
3729
|
case "open":
|
|
2598
|
-
this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
|
|
2599
3730
|
void this.handleOpen(frame);
|
|
2600
3731
|
break;
|
|
2601
3732
|
case "req_data":
|
|
@@ -2631,12 +3762,21 @@ var StreamForwarder = class {
|
|
|
2631
3762
|
const { sid, method, path, headers, has_body } = frame;
|
|
2632
3763
|
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
2633
3764
|
const startedAt = Date.now();
|
|
3765
|
+
if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3766
|
+
this.callbacks.onOpen?.(sid, method, path);
|
|
3767
|
+
}
|
|
2634
3768
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
2635
3769
|
this.callbacks.onDrainPing?.();
|
|
2636
3770
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
2637
3771
|
this.send({ type: "res_end", sid });
|
|
2638
3772
|
return;
|
|
2639
3773
|
}
|
|
3774
|
+
if (path === TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3775
|
+
this.callbacks.onUsageRearmPing?.();
|
|
3776
|
+
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
3777
|
+
this.send({ type: "res_end", sid });
|
|
3778
|
+
return;
|
|
3779
|
+
}
|
|
2640
3780
|
if (process.env.DEBUG) {
|
|
2641
3781
|
log("debug", "agent_request", {
|
|
2642
3782
|
correlation_id: correlationId,
|
|
@@ -2651,12 +3791,12 @@ var StreamForwarder = class {
|
|
|
2651
3791
|
let endBody;
|
|
2652
3792
|
if (has_body) {
|
|
2653
3793
|
const chunks = [];
|
|
2654
|
-
bodyPromise = new Promise((
|
|
3794
|
+
bodyPromise = new Promise((resolve4) => {
|
|
2655
3795
|
pushBody = (buf) => {
|
|
2656
3796
|
chunks.push(buf);
|
|
2657
3797
|
};
|
|
2658
3798
|
endBody = () => {
|
|
2659
|
-
|
|
3799
|
+
resolve4(Buffer.concat(chunks));
|
|
2660
3800
|
};
|
|
2661
3801
|
});
|
|
2662
3802
|
}
|
|
@@ -2781,11 +3921,12 @@ function connectTunnel(options) {
|
|
|
2781
3921
|
onResponse,
|
|
2782
3922
|
onInfo,
|
|
2783
3923
|
onWarning,
|
|
2784
|
-
onDrainPing
|
|
3924
|
+
onDrainPing,
|
|
3925
|
+
onUsageRearmPing
|
|
2785
3926
|
} = options;
|
|
2786
3927
|
const tunnelUrl = getTunnelUrlConfig();
|
|
2787
3928
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
2788
|
-
return new Promise((
|
|
3929
|
+
return new Promise((resolve4, reject) => {
|
|
2789
3930
|
const ws = new WebSocket2(url, {
|
|
2790
3931
|
headers: {
|
|
2791
3932
|
Authorization: authHeader
|
|
@@ -2793,7 +3934,8 @@ function connectTunnel(options) {
|
|
|
2793
3934
|
});
|
|
2794
3935
|
const forwarder = new StreamForwarder(ws, port, {
|
|
2795
3936
|
onHead: () => onResponse?.(),
|
|
2796
|
-
onDrainPing: () => onDrainPing?.()
|
|
3937
|
+
onDrainPing: () => onDrainPing?.(),
|
|
3938
|
+
onUsageRearmPing: () => onUsageRearmPing?.()
|
|
2797
3939
|
});
|
|
2798
3940
|
const connectionTimeout = setTimeout(() => {
|
|
2799
3941
|
ws.close();
|
|
@@ -2836,8 +3978,8 @@ function connectTunnel(options) {
|
|
|
2836
3978
|
try {
|
|
2837
3979
|
message = JSON.parse(data.toString());
|
|
2838
3980
|
} catch (error2) {
|
|
2839
|
-
const
|
|
2840
|
-
onError?.(`Failed to handle message: ${
|
|
3981
|
+
const errorMessage3 = error2 instanceof Error ? error2.message : "Unknown error";
|
|
3982
|
+
onError?.(`Failed to handle message: ${errorMessage3}`);
|
|
2841
3983
|
return;
|
|
2842
3984
|
}
|
|
2843
3985
|
if (isStreamFrame(message)) {
|
|
@@ -2849,7 +3991,7 @@ function connectTunnel(options) {
|
|
|
2849
3991
|
clearTimeout(connectionTimeout);
|
|
2850
3992
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
2851
3993
|
onConnected?.(connectedAgentId);
|
|
2852
|
-
|
|
3994
|
+
resolve4({
|
|
2853
3995
|
ws,
|
|
2854
3996
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
2855
3997
|
});
|
|
@@ -2954,6 +4096,7 @@ var RunnerConnection = class {
|
|
|
2954
4096
|
onError: (error2) => events.onError?.(error2),
|
|
2955
4097
|
onResponse: () => events.onResponse?.(),
|
|
2956
4098
|
onDrainPing: () => events.onDrainPing?.(),
|
|
4099
|
+
onUsageRearmPing: () => events.onUsageRearmPing?.(),
|
|
2957
4100
|
onInfo: (message) => events.onInfo?.(message),
|
|
2958
4101
|
onWarning: (message) => events.onWarning?.(message)
|
|
2959
4102
|
});
|
|
@@ -2980,10 +4123,10 @@ var RunnerConnection = class {
|
|
|
2980
4123
|
};
|
|
2981
4124
|
|
|
2982
4125
|
// src/lib/tunnel/ready-marker.ts
|
|
2983
|
-
import { writeFileSync } from "fs";
|
|
4126
|
+
import { writeFileSync as writeFileSync3 } from "node:fs";
|
|
2984
4127
|
function writeTunnelReadyMarker(path, agentId) {
|
|
2985
4128
|
try {
|
|
2986
|
-
|
|
4129
|
+
writeFileSync3(path, `${agentId}
|
|
2987
4130
|
`);
|
|
2988
4131
|
return { ok: true };
|
|
2989
4132
|
} catch (error2) {
|
|
@@ -2991,6 +4134,237 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
2991
4134
|
}
|
|
2992
4135
|
}
|
|
2993
4136
|
|
|
4137
|
+
// src/lib/replication.ts
|
|
4138
|
+
import { spawn as spawn4 } from "node:child_process";
|
|
4139
|
+
function startSessionDbReplication(configPath) {
|
|
4140
|
+
return spawn4("litestream", ["replicate", "-config", configPath], {
|
|
4141
|
+
stdio: "inherit"
|
|
4142
|
+
});
|
|
4143
|
+
}
|
|
4144
|
+
async function stopSessionDbReplication(child, timeoutMs) {
|
|
4145
|
+
return stopProcessAndWait(
|
|
4146
|
+
child,
|
|
4147
|
+
timeoutMs,
|
|
4148
|
+
() => child.kill("SIGTERM"),
|
|
4149
|
+
() => child.kill("SIGKILL")
|
|
4150
|
+
);
|
|
4151
|
+
}
|
|
4152
|
+
|
|
4153
|
+
// src/lib/process-liveness.ts
|
|
4154
|
+
import { readFileSync as readFileSync4 } from "node:fs";
|
|
4155
|
+
function isProcessAlive(pid) {
|
|
4156
|
+
try {
|
|
4157
|
+
process.kill(pid, 0);
|
|
4158
|
+
} catch (error2) {
|
|
4159
|
+
const code = error2.code;
|
|
4160
|
+
if (code === "ESRCH") return false;
|
|
4161
|
+
if (code === "EPERM") return true;
|
|
4162
|
+
console.error(
|
|
4163
|
+
`[process-liveness] could not probe process ${pid}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4164
|
+
);
|
|
4165
|
+
return false;
|
|
4166
|
+
}
|
|
4167
|
+
if (process.platform !== "linux") return true;
|
|
4168
|
+
try {
|
|
4169
|
+
const status2 = readFileSync4(`/proc/${pid}/status`, "utf8");
|
|
4170
|
+
return !/^State:\s+Z(?:\s|$)/m.test(status2);
|
|
4171
|
+
} catch (error2) {
|
|
4172
|
+
console.error(
|
|
4173
|
+
`[process-liveness] could not inspect /proc/${pid}/status: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4174
|
+
);
|
|
4175
|
+
return true;
|
|
4176
|
+
}
|
|
4177
|
+
}
|
|
4178
|
+
|
|
4179
|
+
// src/lib/openai-usage.ts
|
|
4180
|
+
import { readFileSync as readFileSync5 } from "node:fs";
|
|
4181
|
+
import { homedir as homedir3 } from "node:os";
|
|
4182
|
+
import { join as join5 } from "node:path";
|
|
4183
|
+
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
4184
|
+
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
4185
|
+
var OpenAiUsageError = class extends Error {
|
|
4186
|
+
constructor(message, reason) {
|
|
4187
|
+
super(message);
|
|
4188
|
+
this.reason = reason;
|
|
4189
|
+
}
|
|
4190
|
+
};
|
|
4191
|
+
function isLocalCredentialProblem2(err) {
|
|
4192
|
+
return err instanceof OpenAiUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
|
|
4193
|
+
}
|
|
4194
|
+
function readOpenCodeChatGptCredentials() {
|
|
4195
|
+
try {
|
|
4196
|
+
const raw = readFileSync5(join5(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
4197
|
+
let parsed;
|
|
4198
|
+
try {
|
|
4199
|
+
parsed = JSON.parse(raw);
|
|
4200
|
+
} catch {
|
|
4201
|
+
return null;
|
|
4202
|
+
}
|
|
4203
|
+
const entry = parsed.openai;
|
|
4204
|
+
if (entry?.type !== "oauth" || typeof entry.access !== "string" || !entry.access || typeof entry.expires !== "number") {
|
|
4205
|
+
return null;
|
|
4206
|
+
}
|
|
4207
|
+
return { accessToken: entry.access, expiresAt: entry.expires };
|
|
4208
|
+
} catch (err) {
|
|
4209
|
+
const code = err.code;
|
|
4210
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
4211
|
+
console.warn(
|
|
4212
|
+
`readOpenCodeChatGptCredentials: reading auth.json failed (${code ?? "unknown"})`
|
|
4213
|
+
);
|
|
4214
|
+
}
|
|
4215
|
+
return null;
|
|
4216
|
+
}
|
|
4217
|
+
}
|
|
4218
|
+
function parseChatGptIdentity(accessToken) {
|
|
4219
|
+
const segments = accessToken.split(".");
|
|
4220
|
+
if (segments.length !== 3) return null;
|
|
4221
|
+
let payload;
|
|
4222
|
+
try {
|
|
4223
|
+
const parsed = JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8"));
|
|
4224
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
4225
|
+
payload = parsed;
|
|
4226
|
+
} catch {
|
|
4227
|
+
return null;
|
|
4228
|
+
}
|
|
4229
|
+
const profile = payload["https://api.openai.com/profile"];
|
|
4230
|
+
const auth = payload["https://api.openai.com/auth"];
|
|
4231
|
+
const ownerEmail = typeof profile?.email === "string" ? profile.email.trim() || null : null;
|
|
4232
|
+
const planType = typeof auth?.chatgpt_plan_type === "string" ? auth.chatgpt_plan_type.trim() || null : null;
|
|
4233
|
+
return ownerEmail === null && planType === null ? null : { ownerEmail, planType };
|
|
4234
|
+
}
|
|
4235
|
+
function toWindow2(headers, name) {
|
|
4236
|
+
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
4237
|
+
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
4238
|
+
if (utilizationHeader == null || utilizationHeader === "" || windowMinutesHeader == null) {
|
|
4239
|
+
return null;
|
|
4240
|
+
}
|
|
4241
|
+
const utilization = Number(utilizationHeader);
|
|
4242
|
+
const windowMinutes = Number(windowMinutesHeader);
|
|
4243
|
+
if (!Number.isFinite(utilization) || !Number.isFinite(windowMinutes) || windowMinutes <= 0) {
|
|
4244
|
+
return null;
|
|
4245
|
+
}
|
|
4246
|
+
const resetAtHeader = headers.get(`x-codex-${name}-reset-at`);
|
|
4247
|
+
const resetSeconds = resetAtHeader == null || resetAtHeader === "" ? NaN : Number(resetAtHeader);
|
|
4248
|
+
const resetsAt = Number.isFinite(resetSeconds) ? new Date(resetSeconds * 1e3).toISOString() : null;
|
|
4249
|
+
return { utilization: Math.min(100, Math.max(0, utilization)), windowMinutes, resetsAt };
|
|
4250
|
+
}
|
|
4251
|
+
function parseCodexUsageHeaders(headers) {
|
|
4252
|
+
return {
|
|
4253
|
+
primary: toWindow2(headers, "primary"),
|
|
4254
|
+
secondary: toWindow2(headers, "secondary"),
|
|
4255
|
+
hasCredits: headers.has("x-codex-credits-has-credits") ? headers.get("x-codex-credits-has-credits")?.toLowerCase() === "true" : null,
|
|
4256
|
+
creditsUnlimited: headers.has("x-codex-credits-unlimited") ? headers.get("x-codex-credits-unlimited")?.toLowerCase() === "true" : null
|
|
4257
|
+
};
|
|
4258
|
+
}
|
|
4259
|
+
function normalizeProbeModel(model) {
|
|
4260
|
+
return model.endsWith("-fast") ? model.slice(0, -"-fast".length) : model;
|
|
4261
|
+
}
|
|
4262
|
+
async function resolveProbeModels(port) {
|
|
4263
|
+
try {
|
|
4264
|
+
const res = await withRequestTimeout(
|
|
4265
|
+
fetch,
|
|
4266
|
+
REQUEST_TIMEOUT_MS
|
|
4267
|
+
)(`${opencodeBase(port)}/config/providers`);
|
|
4268
|
+
if (!res.ok) {
|
|
4269
|
+
console.error(
|
|
4270
|
+
`[resolveProbeModels] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
4271
|
+
);
|
|
4272
|
+
return [];
|
|
4273
|
+
}
|
|
4274
|
+
const body = await res.json();
|
|
4275
|
+
const provider = body?.providers?.find((candidate) => candidate?.id === "openai");
|
|
4276
|
+
if (!provider || !provider.models || typeof provider.models !== "object") return [];
|
|
4277
|
+
const candidates = [
|
|
4278
|
+
...typeof body?.default?.openai === "string" ? [body.default.openai] : [],
|
|
4279
|
+
...Object.keys(provider.models)
|
|
4280
|
+
].map(normalizeProbeModel);
|
|
4281
|
+
return [...new Set(candidates)].slice(0, 4);
|
|
4282
|
+
} catch (err) {
|
|
4283
|
+
console.error(
|
|
4284
|
+
`[resolveProbeModels] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
4285
|
+
);
|
|
4286
|
+
return [];
|
|
4287
|
+
}
|
|
4288
|
+
}
|
|
4289
|
+
function hasPrimaryHeaders(headers) {
|
|
4290
|
+
return [
|
|
4291
|
+
"x-codex-primary-used-percent",
|
|
4292
|
+
"x-codex-primary-window-minutes",
|
|
4293
|
+
"x-codex-primary-reset-at"
|
|
4294
|
+
].some((name) => headers.has(name));
|
|
4295
|
+
}
|
|
4296
|
+
async function getOpenAiUsage(port) {
|
|
4297
|
+
const credentials2 = readOpenCodeChatGptCredentials();
|
|
4298
|
+
if (!credentials2) {
|
|
4299
|
+
throw new OpenAiUsageError(
|
|
4300
|
+
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
4301
|
+
"no_credentials"
|
|
4302
|
+
);
|
|
4303
|
+
}
|
|
4304
|
+
if (credentials2.expiresAt < Date.now()) {
|
|
4305
|
+
throw new OpenAiUsageError(
|
|
4306
|
+
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
4307
|
+
"credentials_expired"
|
|
4308
|
+
);
|
|
4309
|
+
}
|
|
4310
|
+
const subscription = parseChatGptIdentity(credentials2.accessToken);
|
|
4311
|
+
const models = await resolveProbeModels(port);
|
|
4312
|
+
if (models.length === 0) {
|
|
4313
|
+
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
4314
|
+
}
|
|
4315
|
+
let lastStatus;
|
|
4316
|
+
for (const model of models) {
|
|
4317
|
+
let res;
|
|
4318
|
+
try {
|
|
4319
|
+
res = await withRequestTimeout(fetch, REQUEST_TIMEOUT_MS)(CODEX_RESPONSES_URL, {
|
|
4320
|
+
method: "POST",
|
|
4321
|
+
headers: {
|
|
4322
|
+
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
4323
|
+
"Content-Type": "application/json"
|
|
4324
|
+
},
|
|
4325
|
+
body: JSON.stringify({ model, store: false, stream: true })
|
|
4326
|
+
});
|
|
4327
|
+
} catch (err) {
|
|
4328
|
+
throw new OpenAiUsageError(
|
|
4329
|
+
`OpenAI usage probe request failed: ${err instanceof Error ? err.message : String(err)}.`,
|
|
4330
|
+
"request_failed"
|
|
4331
|
+
);
|
|
4332
|
+
}
|
|
4333
|
+
try {
|
|
4334
|
+
lastStatus = res.status;
|
|
4335
|
+
if (hasPrimaryHeaders(res.headers)) {
|
|
4336
|
+
const usage = parseCodexUsageHeaders(res.headers);
|
|
4337
|
+
if (!usage.primary && !usage.secondary) {
|
|
4338
|
+
throw new OpenAiUsageError(
|
|
4339
|
+
`OpenAI usage probe returned no usable window (HTTP ${res.status}).`,
|
|
4340
|
+
"no_usable_window"
|
|
4341
|
+
);
|
|
4342
|
+
}
|
|
4343
|
+
return { ...usage, subscription };
|
|
4344
|
+
}
|
|
4345
|
+
if (res.status === 401) {
|
|
4346
|
+
throw new OpenAiUsageError(
|
|
4347
|
+
"ChatGPT credentials have expired (HTTP 401).",
|
|
4348
|
+
"credentials_expired"
|
|
4349
|
+
);
|
|
4350
|
+
}
|
|
4351
|
+
if (res.status === 403 || res.status === 429) {
|
|
4352
|
+
throw new OpenAiUsageError(
|
|
4353
|
+
`OpenAI usage probe was blocked (HTTP ${res.status}).`,
|
|
4354
|
+
"probe_blocked"
|
|
4355
|
+
);
|
|
4356
|
+
}
|
|
4357
|
+
} finally {
|
|
4358
|
+
await res.body?.cancel().catch(() => {
|
|
4359
|
+
});
|
|
4360
|
+
}
|
|
4361
|
+
}
|
|
4362
|
+
throw new OpenAiUsageError(
|
|
4363
|
+
`OpenAI usage probe failed: HTTP ${lastStatus ?? "unknown"}.`,
|
|
4364
|
+
"request_failed"
|
|
4365
|
+
);
|
|
4366
|
+
}
|
|
4367
|
+
|
|
2994
4368
|
// src/lib/reporting-schedule.ts
|
|
2995
4369
|
function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
2996
4370
|
const jitterRangeMs = baseMs * jitterFraction;
|
|
@@ -2999,6 +4373,31 @@ function jitteredDelayMs(baseMs, jitterFraction, random = Math.random) {
|
|
|
2999
4373
|
function firstReportDelayMs(random = Math.random) {
|
|
3000
4374
|
return 5e3 + random() * 1e4;
|
|
3001
4375
|
}
|
|
4376
|
+
var VALID_USAGE_REPORTING_MODES = ["auto", "on", "off"];
|
|
4377
|
+
function resolveUsageReportingMode(flagValue, env, names) {
|
|
4378
|
+
const raw = flagValue ?? env[names.envVar];
|
|
4379
|
+
if (raw === void 0 || raw === "") return { mode: "auto", warnings: [] };
|
|
4380
|
+
const normalized = raw.trim().toLowerCase();
|
|
4381
|
+
if (VALID_USAGE_REPORTING_MODES.includes(normalized)) {
|
|
4382
|
+
return { mode: normalized, warnings: [] };
|
|
4383
|
+
}
|
|
4384
|
+
const source = flagValue !== void 0 ? names.flagName : names.envVar;
|
|
4385
|
+
return {
|
|
4386
|
+
mode: "auto",
|
|
4387
|
+
warnings: [
|
|
4388
|
+
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_USAGE_REPORTING_MODES.join(", ")}; using auto`
|
|
4389
|
+
]
|
|
4390
|
+
};
|
|
4391
|
+
}
|
|
4392
|
+
var BASE_USAGE_REPORT_DELAY_MS = 10 * 6e4;
|
|
4393
|
+
var USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
4394
|
+
function usageReportDelayMs(random = Math.random) {
|
|
4395
|
+
return jitteredDelayMs(BASE_USAGE_REPORT_DELAY_MS, USAGE_REPORT_DELAY_JITTER_FRACTION, random);
|
|
4396
|
+
}
|
|
4397
|
+
var USAGE_REPORT_FAILURE_REESCALATION_TICKS = 6;
|
|
4398
|
+
function usageReportFailureLogLevel(consecutiveFailures) {
|
|
4399
|
+
return reportFailureLogLevel(consecutiveFailures, USAGE_REPORT_FAILURE_REESCALATION_TICKS);
|
|
4400
|
+
}
|
|
3002
4401
|
function reportFailureLogLevel(consecutiveFailures, reescalationTicks) {
|
|
3003
4402
|
return consecutiveFailures === 1 || consecutiveFailures % reescalationTicks === 0 ? "warn" : "debug";
|
|
3004
4403
|
}
|
|
@@ -3007,33 +4406,26 @@ function failureStreakSuffix(consecutiveFailures) {
|
|
|
3007
4406
|
}
|
|
3008
4407
|
|
|
3009
4408
|
// src/lib/claude-usage-reporting.ts
|
|
3010
|
-
var VALID_MODES = ["auto", "on", "off"];
|
|
3011
4409
|
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
}
|
|
3016
|
-
const normalized = raw.trim().toLowerCase();
|
|
3017
|
-
if (VALID_MODES.includes(normalized)) {
|
|
3018
|
-
return { mode: normalized, warnings: [] };
|
|
3019
|
-
}
|
|
3020
|
-
const source = flagValue !== void 0 ? "--claude-usage-reporting" : "EVIDENT_CLAUDE_USAGE_REPORTING";
|
|
3021
|
-
return {
|
|
3022
|
-
mode: "auto",
|
|
3023
|
-
warnings: [
|
|
3024
|
-
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_MODES.join(", ")}; using auto`
|
|
3025
|
-
]
|
|
3026
|
-
};
|
|
4410
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
4411
|
+
flagName: "--claude-usage-reporting",
|
|
4412
|
+
envVar: "EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
4413
|
+
});
|
|
3027
4414
|
}
|
|
3028
|
-
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
3029
|
-
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
3030
4415
|
function nextReportDelayMs(random = Math.random) {
|
|
3031
|
-
return
|
|
4416
|
+
return usageReportDelayMs(random);
|
|
3032
4417
|
}
|
|
3033
4418
|
var FIRST_REPORT_DELAY_MS = firstReportDelayMs();
|
|
3034
|
-
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
3035
4419
|
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
3036
|
-
return
|
|
4420
|
+
return usageReportFailureLogLevel(consecutiveFailures);
|
|
4421
|
+
}
|
|
4422
|
+
|
|
4423
|
+
// src/lib/openai-usage-reporting.ts
|
|
4424
|
+
function resolveOpenAiUsageReportingMode(flagValue, env) {
|
|
4425
|
+
return resolveUsageReportingMode(flagValue, env, {
|
|
4426
|
+
flagName: "--openai-usage-reporting",
|
|
4427
|
+
envVar: "EVIDENT_OPENAI_USAGE_REPORTING"
|
|
4428
|
+
});
|
|
3037
4429
|
}
|
|
3038
4430
|
|
|
3039
4431
|
// src/lib/resource-usage-reporting.ts
|
|
@@ -3063,8 +4455,8 @@ function resolveResourceUsageReportingEnabled(flagValue, env) {
|
|
|
3063
4455
|
}
|
|
3064
4456
|
|
|
3065
4457
|
// src/lib/resource-usage.ts
|
|
3066
|
-
import { cpus, totalmem, freemem } from "os";
|
|
3067
|
-
import { statfsSync as statfsSync2 } from "fs";
|
|
4458
|
+
import { cpus, totalmem, freemem } from "node:os";
|
|
4459
|
+
import { statfsSync as statfsSync2 } from "node:fs";
|
|
3068
4460
|
|
|
3069
4461
|
// src/lib/ecs-task-metadata.ts
|
|
3070
4462
|
var ECS_METADATA_TIMEOUT_MS = 2e3;
|
|
@@ -3149,58 +4541,97 @@ function readDisk(homeDir) {
|
|
|
3149
4541
|
};
|
|
3150
4542
|
}
|
|
3151
4543
|
}
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
4544
|
+
var CPU_PEAK_WINDOW_MS = 6e4;
|
|
4545
|
+
var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
|
|
4546
|
+
var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4547
|
+
function createCpuPeakSampler() {
|
|
4548
|
+
const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4549
|
+
sampleHistory[0] = readCpuSample();
|
|
4550
|
+
let nextSampleIndex = 1;
|
|
4551
|
+
let sampleCount = 1;
|
|
4552
|
+
let peak = null;
|
|
4553
|
+
const timer = setInterval(() => {
|
|
3155
4554
|
const current = readCpuSample();
|
|
3156
|
-
const
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
const warnings = [];
|
|
3163
|
-
if (disk.warning) warnings.push(disk.warning);
|
|
3164
|
-
if (ecsWarning) warnings.push(ecsWarning);
|
|
3165
|
-
let cpuPercent = hostCpuPercent;
|
|
3166
|
-
let cpuCount = hostCpuCount;
|
|
3167
|
-
let memoryTotalBytes = totalmem();
|
|
3168
|
-
let memoryAvailableBytes = freemem();
|
|
3169
|
-
if (limits !== null) {
|
|
3170
|
-
cpuCount = limits.cpuCount;
|
|
3171
|
-
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3172
|
-
memoryAvailableBytes = clamp(
|
|
3173
|
-
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3174
|
-
0,
|
|
3175
|
-
limits.memoryTotalBytes
|
|
3176
|
-
);
|
|
3177
|
-
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4555
|
+
const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
|
|
4556
|
+
if (sampleFromWindowAgo !== void 0) {
|
|
4557
|
+
const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
|
|
4558
|
+
if (percentage !== null) {
|
|
4559
|
+
peak = peak === null ? percentage : Math.max(peak, percentage);
|
|
4560
|
+
}
|
|
3178
4561
|
}
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
4562
|
+
sampleHistory[nextSampleIndex] = current;
|
|
4563
|
+
nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4564
|
+
sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4565
|
+
}, CPU_PEAK_SAMPLE_INTERVAL_MS);
|
|
4566
|
+
return {
|
|
4567
|
+
takeAndReset: () => {
|
|
4568
|
+
const currentPeak = peak;
|
|
4569
|
+
peak = null;
|
|
4570
|
+
return currentPeak;
|
|
4571
|
+
},
|
|
4572
|
+
stop: () => clearInterval(timer)
|
|
4573
|
+
};
|
|
4574
|
+
}
|
|
4575
|
+
function createResourceUsageCollector(homeDir) {
|
|
4576
|
+
let previous = readCpuSample();
|
|
4577
|
+
const cpuPeakSampler = createCpuPeakSampler();
|
|
4578
|
+
return {
|
|
4579
|
+
collect: async () => {
|
|
4580
|
+
const current = readCpuSample();
|
|
4581
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
4582
|
+
const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
|
|
4583
|
+
const hostCpuCount = cpus().length;
|
|
4584
|
+
previous = current;
|
|
4585
|
+
const disk = readDisk(homeDir);
|
|
4586
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
4587
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
4588
|
+
const warnings = [];
|
|
4589
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
4590
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
4591
|
+
let cpuPercent = hostCpuPercent;
|
|
4592
|
+
let cpuPeakPercent = hostCpuPeakPercent;
|
|
4593
|
+
let cpuCount = hostCpuCount;
|
|
4594
|
+
let memoryTotalBytes = totalmem();
|
|
4595
|
+
let memoryAvailableBytes = freemem();
|
|
4596
|
+
if (limits !== null) {
|
|
4597
|
+
cpuCount = limits.cpuCount;
|
|
4598
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4599
|
+
memoryAvailableBytes = clamp(
|
|
4600
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4601
|
+
0,
|
|
4602
|
+
limits.memoryTotalBytes
|
|
4603
|
+
);
|
|
4604
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4605
|
+
cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4606
|
+
}
|
|
4607
|
+
return {
|
|
4608
|
+
usage: {
|
|
4609
|
+
cpuPercent,
|
|
4610
|
+
cpuPeakPercent,
|
|
4611
|
+
cpuCount,
|
|
4612
|
+
memoryTotalBytes,
|
|
4613
|
+
memoryAvailableBytes,
|
|
4614
|
+
diskTotalBytes: disk.totalBytes,
|
|
4615
|
+
diskFreeBytes: disk.freeBytes,
|
|
4616
|
+
opencodeDbBytes
|
|
4617
|
+
},
|
|
4618
|
+
warnings
|
|
4619
|
+
};
|
|
4620
|
+
},
|
|
4621
|
+
stop: cpuPeakSampler.stop
|
|
3191
4622
|
};
|
|
3192
4623
|
}
|
|
3193
4624
|
|
|
3194
4625
|
// src/lib/channels/driver.ts
|
|
3195
|
-
import { homedir as
|
|
4626
|
+
import { homedir as homedir4 } from "node:os";
|
|
3196
4627
|
|
|
3197
4628
|
// src/lib/runner-file-sync.ts
|
|
3198
|
-
import { join as
|
|
4629
|
+
import { join as join7 } from "node:path";
|
|
3199
4630
|
|
|
3200
4631
|
// src/lib/file-push.ts
|
|
3201
|
-
import { randomUUID } from "crypto";
|
|
3202
|
-
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
3203
|
-
import { basename, dirname as
|
|
4632
|
+
import { randomUUID } from "node:crypto";
|
|
4633
|
+
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "node:fs/promises";
|
|
4634
|
+
import { basename, dirname as dirname5, isAbsolute, join as join6, relative, resolve as resolve2, sep } from "node:path";
|
|
3204
4635
|
var FILE_MODE = 384;
|
|
3205
4636
|
var DIRECTORY_MODE = 448;
|
|
3206
4637
|
async function writePushedFile(request) {
|
|
@@ -3231,9 +4662,9 @@ async function writePushedFile(request) {
|
|
|
3231
4662
|
}
|
|
3232
4663
|
try {
|
|
3233
4664
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3234
|
-
|
|
4665
|
+
dirname5(candidate)
|
|
3235
4666
|
);
|
|
3236
|
-
const realTarget =
|
|
4667
|
+
const realTarget = join6(existingAncestor, ...missingSegments, basename(candidate));
|
|
3237
4668
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3238
4669
|
if (allowedDirectory === null) {
|
|
3239
4670
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3243,8 +4674,8 @@ async function writePushedFile(request) {
|
|
|
3243
4674
|
}
|
|
3244
4675
|
if (missingSegments.length > 0) {
|
|
3245
4676
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
3246
|
-
const realParent = await realpath(
|
|
3247
|
-
if (realParent !==
|
|
4677
|
+
const realParent = await realpath(dirname5(realTarget));
|
|
4678
|
+
if (realParent !== dirname5(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
3248
4679
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
3249
4680
|
path: realTarget,
|
|
3250
4681
|
bytes,
|
|
@@ -3269,7 +4700,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3269
4700
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3270
4701
|
return null;
|
|
3271
4702
|
}
|
|
3272
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4703
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3273
4704
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3274
4705
|
return null;
|
|
3275
4706
|
}
|
|
@@ -3287,7 +4718,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
3287
4718
|
try {
|
|
3288
4719
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
3289
4720
|
} catch (err) {
|
|
3290
|
-
const parent =
|
|
4721
|
+
const parent = dirname5(current);
|
|
3291
4722
|
if (err.code !== "ENOENT" || parent === current) {
|
|
3292
4723
|
throw err;
|
|
3293
4724
|
}
|
|
@@ -3342,13 +4773,13 @@ function contains(realDirectory, realTarget) {
|
|
|
3342
4773
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3343
4774
|
let current = existingAncestor;
|
|
3344
4775
|
for (const segment of missingSegments) {
|
|
3345
|
-
current =
|
|
4776
|
+
current = join6(current, segment);
|
|
3346
4777
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3347
4778
|
await chmod(current, DIRECTORY_MODE);
|
|
3348
4779
|
}
|
|
3349
4780
|
}
|
|
3350
4781
|
async function writeAtomically(realTarget, content) {
|
|
3351
|
-
const temporaryPath =
|
|
4782
|
+
const temporaryPath = join6(dirname5(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3352
4783
|
let handle;
|
|
3353
4784
|
try {
|
|
3354
4785
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -3390,20 +4821,28 @@ async function syncPendingRunnerFiles(options) {
|
|
|
3390
4821
|
for (const id of options.ackFailures.keys()) {
|
|
3391
4822
|
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
3392
4823
|
}
|
|
3393
|
-
if (pending.length === 0)
|
|
4824
|
+
if (pending.length === 0) {
|
|
4825
|
+
return { applied: 0, claudeCredentialApplied: false, opencodeAuthApplied: false };
|
|
4826
|
+
}
|
|
3394
4827
|
options.log({
|
|
3395
4828
|
level: "info",
|
|
3396
4829
|
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
3397
4830
|
});
|
|
3398
4831
|
let applied = 0;
|
|
3399
4832
|
let claudeCredentialApplied = false;
|
|
4833
|
+
let opencodeAuthApplied = false;
|
|
3400
4834
|
for (const file of pending) {
|
|
3401
4835
|
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
3402
4836
|
const outcome = await applyOne(options, file);
|
|
3403
4837
|
if (outcome.applied) applied += 1;
|
|
3404
4838
|
if (outcome.claudeCredentialApplied) claudeCredentialApplied = true;
|
|
4839
|
+
if (outcome.opencodeAuthApplied) opencodeAuthApplied = true;
|
|
3405
4840
|
}
|
|
3406
|
-
return {
|
|
4841
|
+
return {
|
|
4842
|
+
applied,
|
|
4843
|
+
claudeCredentialApplied,
|
|
4844
|
+
opencodeAuthApplied
|
|
4845
|
+
};
|
|
3407
4846
|
}
|
|
3408
4847
|
async function listPendingFiles(options) {
|
|
3409
4848
|
let res;
|
|
@@ -3464,10 +4903,18 @@ function asPendingFile(entry) {
|
|
|
3464
4903
|
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
3465
4904
|
return { id, path, size };
|
|
3466
4905
|
}
|
|
3467
|
-
var NOT_APPLIED = {
|
|
4906
|
+
var NOT_APPLIED = {
|
|
4907
|
+
applied: false,
|
|
4908
|
+
claudeCredentialApplied: false,
|
|
4909
|
+
opencodeAuthApplied: false
|
|
4910
|
+
};
|
|
3468
4911
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3469
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3470
|
-
return expanded ===
|
|
4912
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4913
|
+
return expanded === join7(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
4914
|
+
}
|
|
4915
|
+
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
4916
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4917
|
+
return expanded === join7(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
3471
4918
|
}
|
|
3472
4919
|
async function applyOne(options, file) {
|
|
3473
4920
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -3523,7 +4970,8 @@ async function applyOne(options, file) {
|
|
|
3523
4970
|
await ack(options, file, "applied");
|
|
3524
4971
|
return {
|
|
3525
4972
|
applied: true,
|
|
3526
|
-
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir)
|
|
4973
|
+
claudeCredentialApplied: isClaudeCredentialPath(file.path, options.homeDir),
|
|
4974
|
+
opencodeAuthApplied: isOpenCodeAuthPath(file.path, options.homeDir)
|
|
3527
4975
|
};
|
|
3528
4976
|
}
|
|
3529
4977
|
function durableDownloadCode(status2) {
|
|
@@ -3977,6 +5425,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3977
5425
|
* that way rather than "fixing" it into a count.
|
|
3978
5426
|
*/
|
|
3979
5427
|
claudeCredentialApplyCount = 0;
|
|
5428
|
+
opencodeAuthApplyCount = 0;
|
|
3980
5429
|
/**
|
|
3981
5430
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
3982
5431
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -3992,6 +5441,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3992
5441
|
* and stops opencode.
|
|
3993
5442
|
*/
|
|
3994
5443
|
stopped = false;
|
|
5444
|
+
recycleRequestedFlag = false;
|
|
3995
5445
|
constructor(config) {
|
|
3996
5446
|
this.agentId = config.agentId;
|
|
3997
5447
|
this.port = config.port;
|
|
@@ -4011,7 +5461,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4011
5461
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
4012
5462
|
this.now = config.now ?? (() => Date.now());
|
|
4013
5463
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
4014
|
-
this.homeDir = config.homeDir ??
|
|
5464
|
+
this.homeDir = config.homeDir ?? homedir4();
|
|
4015
5465
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4016
5466
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4017
5467
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
@@ -4081,6 +5531,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4081
5531
|
});
|
|
4082
5532
|
this.appliedFileCount += result.applied;
|
|
4083
5533
|
if (result.claudeCredentialApplied) this.claudeCredentialApplyCount += 1;
|
|
5534
|
+
if (result.opencodeAuthApplied) this.opencodeAuthApplyCount += 1;
|
|
4084
5535
|
return result.applied;
|
|
4085
5536
|
} catch (err) {
|
|
4086
5537
|
this.log({
|
|
@@ -4096,6 +5547,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4096
5547
|
let dispatched = 0;
|
|
4097
5548
|
try {
|
|
4098
5549
|
const conversations = await this.getPendingConversations();
|
|
5550
|
+
if (this.recycleRequestedFlag) {
|
|
5551
|
+
this.stop();
|
|
5552
|
+
}
|
|
4099
5553
|
if (conversations.length > 0) {
|
|
4100
5554
|
const total = conversations.reduce((sum, c) => sum + (c.pending_message_count ?? 0), 0);
|
|
4101
5555
|
this.log({
|
|
@@ -4188,7 +5642,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4188
5642
|
return {
|
|
4189
5643
|
appliedFiles: this.appliedFileCount,
|
|
4190
5644
|
inFlight: this.syncingFiles,
|
|
4191
|
-
claudeCredentialApplies: this.claudeCredentialApplyCount
|
|
5645
|
+
claudeCredentialApplies: this.claudeCredentialApplyCount,
|
|
5646
|
+
opencodeAuthApplyCount: this.opencodeAuthApplyCount
|
|
4192
5647
|
};
|
|
4193
5648
|
}
|
|
4194
5649
|
/**
|
|
@@ -4223,6 +5678,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4223
5678
|
stop() {
|
|
4224
5679
|
this.stopped = true;
|
|
4225
5680
|
}
|
|
5681
|
+
/**
|
|
5682
|
+
* The server clears this request when a new MicroVM identity is recorded, so a
|
|
5683
|
+
* same-VM tunnel reconnect does not consume it. This is a plain read rather
|
|
5684
|
+
* than a consume; `run.ts` guards the action once-only.
|
|
5685
|
+
*/
|
|
5686
|
+
get recycleRequested() {
|
|
5687
|
+
return this.recycleRequestedFlag;
|
|
5688
|
+
}
|
|
4226
5689
|
/**
|
|
4227
5690
|
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
4228
5691
|
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
@@ -4360,7 +5823,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4360
5823
|
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
4361
5824
|
break;
|
|
4362
5825
|
}
|
|
4363
|
-
const
|
|
5826
|
+
const errorMessage3 = err instanceof Error ? err.message : String(err);
|
|
4364
5827
|
this.sessions.delete(conv.id);
|
|
4365
5828
|
this.supersede(conv.id, sessionId);
|
|
4366
5829
|
this.log({
|
|
@@ -4369,7 +5832,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4369
5832
|
conversation_id: conv.id,
|
|
4370
5833
|
message_id: message.id
|
|
4371
5834
|
});
|
|
4372
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5835
|
+
await this.markFailed(conv.id, message.id, null, errorMessage3).catch((markErr) => {
|
|
4373
5836
|
this.log({
|
|
4374
5837
|
level: "warn",
|
|
4375
5838
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4380,7 +5843,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4380
5843
|
});
|
|
4381
5844
|
this.log({
|
|
4382
5845
|
level: "error",
|
|
4383
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
5846
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage3}`,
|
|
4384
5847
|
conversation_id: conv.id,
|
|
4385
5848
|
message_id: message.id
|
|
4386
5849
|
});
|
|
@@ -4401,14 +5864,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4401
5864
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4402
5865
|
this.sessions.delete(conv.id);
|
|
4403
5866
|
this.supersede(conv.id, sessionId);
|
|
4404
|
-
const
|
|
5867
|
+
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
4405
5868
|
this.log({
|
|
4406
5869
|
level: "error",
|
|
4407
|
-
message:
|
|
5870
|
+
message: errorMessage3,
|
|
4408
5871
|
conversation_id: conv.id,
|
|
4409
5872
|
message_id: message.id
|
|
4410
5873
|
});
|
|
4411
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5874
|
+
await this.markFailed(conv.id, message.id, null, errorMessage3).catch((markErr) => {
|
|
4412
5875
|
this.log({
|
|
4413
5876
|
level: "warn",
|
|
4414
5877
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4742,6 +6205,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4742
6205
|
});
|
|
4743
6206
|
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
4744
6207
|
}
|
|
6208
|
+
if (ocId !== null) {
|
|
6209
|
+
await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
|
|
6210
|
+
}
|
|
4745
6211
|
} catch (err) {
|
|
4746
6212
|
if (err instanceof ChannelAuthError) throw err;
|
|
4747
6213
|
this.log({
|
|
@@ -5276,6 +6742,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5276
6742
|
deliveryDeadlineAnchored: false,
|
|
5277
6743
|
b2PinnedSinceMs: 0,
|
|
5278
6744
|
b2LastDescendantCheckMs: 0,
|
|
6745
|
+
b2RootOngoingHeldLogged: false,
|
|
5279
6746
|
b2AbandonedSignalled: false,
|
|
5280
6747
|
ambiguousPinnedSinceMs: 0,
|
|
5281
6748
|
ambiguousResolved: false
|
|
@@ -5370,6 +6837,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5370
6837
|
deliveryDeadlineAnchored: false,
|
|
5371
6838
|
b2PinnedSinceMs: 0,
|
|
5372
6839
|
b2LastDescendantCheckMs: 0,
|
|
6840
|
+
b2RootOngoingHeldLogged: false,
|
|
5373
6841
|
b2AbandonedSignalled: false,
|
|
5374
6842
|
ambiguousPinnedSinceMs: 0,
|
|
5375
6843
|
ambiguousResolved: false
|
|
@@ -5704,6 +7172,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5704
7172
|
return;
|
|
5705
7173
|
}
|
|
5706
7174
|
inFlight.done = true;
|
|
7175
|
+
await this.reportSubagentAuthFailures(
|
|
7176
|
+
watcher.conv.id,
|
|
7177
|
+
inFlight.opencodeMessageId,
|
|
7178
|
+
inFlight.evidentMessageId,
|
|
7179
|
+
messages
|
|
7180
|
+
);
|
|
5707
7181
|
}
|
|
5708
7182
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
5709
7183
|
return;
|
|
@@ -5723,6 +7197,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5723
7197
|
if (snapshotReadable) {
|
|
5724
7198
|
inFlight.b2PinnedSinceMs = 0;
|
|
5725
7199
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
7200
|
+
inFlight.b2RootOngoingHeldLogged = false;
|
|
5726
7201
|
inFlight.b2AbandonedSignalled = false;
|
|
5727
7202
|
}
|
|
5728
7203
|
} else {
|
|
@@ -5734,11 +7209,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5734
7209
|
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
5735
7210
|
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
5736
7211
|
inFlight.b2LastDescendantCheckMs = this.now();
|
|
5737
|
-
const descendantOngoing = await
|
|
7212
|
+
const [descendantOngoing, rootOngoing] = await Promise.all([
|
|
7213
|
+
this.isAnyDescendantSessionOngoing(sessionId),
|
|
7214
|
+
isSessionOngoing(this.port, sessionId)
|
|
7215
|
+
]);
|
|
5738
7216
|
if (isB2AbandonmentConfirmed({
|
|
5739
7217
|
pinnedForMs,
|
|
5740
7218
|
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
5741
|
-
descendantOngoing
|
|
7219
|
+
descendantOngoing,
|
|
7220
|
+
rootOngoing
|
|
5742
7221
|
})) {
|
|
5743
7222
|
inFlight.b2AbandonedSignalled = true;
|
|
5744
7223
|
this.log({
|
|
@@ -5747,12 +7226,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5747
7226
|
conversation_id: conv.id,
|
|
5748
7227
|
message_id: id
|
|
5749
7228
|
});
|
|
7229
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
5750
7230
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
5751
|
-
watched_for_ms: pinnedForMs
|
|
7231
|
+
watched_for_ms: pinnedForMs,
|
|
7232
|
+
finish: reply?.info?.finish ?? reply?.finish,
|
|
7233
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
7234
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
7235
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
5752
7236
|
});
|
|
5753
7237
|
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
5754
7238
|
return;
|
|
5755
7239
|
}
|
|
7240
|
+
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
7241
|
+
inFlight.b2RootOngoingHeldLogged = true;
|
|
7242
|
+
this.log({
|
|
7243
|
+
level: "warn",
|
|
7244
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with root_ongoing=${rootOngoing} and descendant_ongoing=${descendantOngoing} \u2014 holding until OpenCode confirms the root is idle`,
|
|
7245
|
+
conversation_id: conv.id,
|
|
7246
|
+
message_id: id
|
|
7247
|
+
});
|
|
7248
|
+
}
|
|
5756
7249
|
}
|
|
5757
7250
|
}
|
|
5758
7251
|
const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
|
|
@@ -5927,6 +7420,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5927
7420
|
return;
|
|
5928
7421
|
}
|
|
5929
7422
|
inFlight.done = true;
|
|
7423
|
+
await this.reportSubagentAuthFailures(
|
|
7424
|
+
watcher.conv.id,
|
|
7425
|
+
inFlight.opencodeMessageId,
|
|
7426
|
+
inFlight.evidentMessageId,
|
|
7427
|
+
messages
|
|
7428
|
+
);
|
|
5930
7429
|
}
|
|
5931
7430
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
5932
7431
|
}
|
|
@@ -6097,6 +7596,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6097
7596
|
});
|
|
6098
7597
|
return;
|
|
6099
7598
|
}
|
|
7599
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
6100
7600
|
this.dontRedispatch.delete(row.id);
|
|
6101
7601
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
6102
7602
|
return;
|
|
@@ -6258,6 +7758,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6258
7758
|
});
|
|
6259
7759
|
return;
|
|
6260
7760
|
}
|
|
7761
|
+
if (ocId !== null) {
|
|
7762
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
|
|
7763
|
+
}
|
|
6261
7764
|
this.dontRedispatch.delete(row.id);
|
|
6262
7765
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
6263
7766
|
}
|
|
@@ -6351,14 +7854,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6351
7854
|
this.unconfirmedDispatchFailures.delete(row.id);
|
|
6352
7855
|
this.sessions.delete(readoptConv.id);
|
|
6353
7856
|
this.supersede(readoptConv.id, sessionId);
|
|
6354
|
-
const
|
|
7857
|
+
const errorMessage3 = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
6355
7858
|
this.log({
|
|
6356
7859
|
level: "error",
|
|
6357
|
-
message:
|
|
7860
|
+
message: errorMessage3,
|
|
6358
7861
|
conversation_id: row.conversation_id,
|
|
6359
7862
|
message_id: row.id
|
|
6360
7863
|
});
|
|
6361
|
-
await this.markFailed(row.conversation_id, row.id, null,
|
|
7864
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
6362
7865
|
this.log({
|
|
6363
7866
|
level: "warn",
|
|
6364
7867
|
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -6973,6 +8476,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6973
8476
|
throw new Error(`Failed to get pending conversations: HTTP ${res.status}`);
|
|
6974
8477
|
}
|
|
6975
8478
|
const data = await res.json();
|
|
8479
|
+
this.recycleRequestedFlag = data.recycle_requested === true;
|
|
6976
8480
|
let conversations = data.conversations;
|
|
6977
8481
|
if (this.conversationFilter) {
|
|
6978
8482
|
conversations = conversations.filter((c) => c.id === this.conversationFilter);
|
|
@@ -7208,6 +8712,111 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7208
8712
|
reply?.info?.modelID ?? null
|
|
7209
8713
|
);
|
|
7210
8714
|
}
|
|
8715
|
+
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
8716
|
+
const providerId = failure.providerId ?? "(unknown)";
|
|
8717
|
+
try {
|
|
8718
|
+
const res = await this.fetchImpl(
|
|
8719
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
|
|
8720
|
+
{
|
|
8721
|
+
method: "POST",
|
|
8722
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
8723
|
+
body: JSON.stringify({
|
|
8724
|
+
provider_id: failure.providerId,
|
|
8725
|
+
model_id: failure.modelId,
|
|
8726
|
+
reason: failure.reason
|
|
8727
|
+
})
|
|
8728
|
+
}
|
|
8729
|
+
);
|
|
8730
|
+
if (!res.ok) {
|
|
8731
|
+
this.log({
|
|
8732
|
+
level: "warn",
|
|
8733
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8734
|
+
conversation_id: conversationId,
|
|
8735
|
+
message_id: messageId
|
|
8736
|
+
});
|
|
8737
|
+
}
|
|
8738
|
+
} catch (err) {
|
|
8739
|
+
this.log({
|
|
8740
|
+
level: "warn",
|
|
8741
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8742
|
+
conversation_id: conversationId,
|
|
8743
|
+
message_id: messageId
|
|
8744
|
+
});
|
|
8745
|
+
}
|
|
8746
|
+
}
|
|
8747
|
+
async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
|
|
8748
|
+
try {
|
|
8749
|
+
const res = await this.fetchImpl(
|
|
8750
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
|
|
8751
|
+
{
|
|
8752
|
+
method: "DELETE",
|
|
8753
|
+
headers: { Authorization: this.getAuthHeader() }
|
|
8754
|
+
}
|
|
8755
|
+
);
|
|
8756
|
+
if (!res.ok) {
|
|
8757
|
+
this.log({
|
|
8758
|
+
level: "warn",
|
|
8759
|
+
message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8760
|
+
conversation_id: conversationId,
|
|
8761
|
+
message_id: messageId
|
|
8762
|
+
});
|
|
8763
|
+
}
|
|
8764
|
+
} catch (err) {
|
|
8765
|
+
this.log({
|
|
8766
|
+
level: "warn",
|
|
8767
|
+
message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8768
|
+
conversation_id: conversationId,
|
|
8769
|
+
message_id: messageId
|
|
8770
|
+
});
|
|
8771
|
+
}
|
|
8772
|
+
}
|
|
8773
|
+
async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
|
|
8774
|
+
const refs = collectSubagentSessions(messages, opencodeMessageId);
|
|
8775
|
+
if (refs.length === 0) return;
|
|
8776
|
+
const failedProviders = /* @__PURE__ */ new Map();
|
|
8777
|
+
const succeededProviders = /* @__PURE__ */ new Set();
|
|
8778
|
+
for (const ref of refs) {
|
|
8779
|
+
try {
|
|
8780
|
+
const childMessages = await getSessionMessages(this.port, ref.sessionId);
|
|
8781
|
+
if (childMessages === null) {
|
|
8782
|
+
this.log({
|
|
8783
|
+
level: "debug",
|
|
8784
|
+
message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
|
|
8785
|
+
conversation_id: conversationId,
|
|
8786
|
+
message_id: evidentMessageId
|
|
8787
|
+
});
|
|
8788
|
+
continue;
|
|
8789
|
+
}
|
|
8790
|
+
const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
|
|
8791
|
+
if (!outcome) continue;
|
|
8792
|
+
if (outcome.outcome === "failed") {
|
|
8793
|
+
failedProviders.set(outcome.providerId, outcome.failure);
|
|
8794
|
+
} else {
|
|
8795
|
+
succeededProviders.add(outcome.providerId);
|
|
8796
|
+
}
|
|
8797
|
+
} catch (err) {
|
|
8798
|
+
this.log({
|
|
8799
|
+
level: "warn",
|
|
8800
|
+
message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8801
|
+
conversation_id: conversationId,
|
|
8802
|
+
message_id: evidentMessageId
|
|
8803
|
+
});
|
|
8804
|
+
}
|
|
8805
|
+
}
|
|
8806
|
+
for (const [providerId, failure] of failedProviders) {
|
|
8807
|
+
this.log({
|
|
8808
|
+
level: "warn",
|
|
8809
|
+
message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
|
|
8810
|
+
conversation_id: conversationId,
|
|
8811
|
+
message_id: evidentMessageId
|
|
8812
|
+
});
|
|
8813
|
+
await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
|
|
8814
|
+
}
|
|
8815
|
+
for (const providerId of succeededProviders) {
|
|
8816
|
+
if (failedProviders.has(providerId)) continue;
|
|
8817
|
+
await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
|
|
8818
|
+
}
|
|
8819
|
+
}
|
|
7211
8820
|
/**
|
|
7212
8821
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
7213
8822
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -7361,6 +8970,13 @@ import chalk5 from "chalk";
|
|
|
7361
8970
|
import ora2 from "ora";
|
|
7362
8971
|
import { select as select2 } from "@inquirer/prompts";
|
|
7363
8972
|
var INTERACTIVE_START_TIMEOUT_MS = 3e4;
|
|
8973
|
+
function checkNonInteractivePortConflict(port, isPortInUseFn) {
|
|
8974
|
+
if (isPortInUseFn(port)) {
|
|
8975
|
+
throw new Error(
|
|
8976
|
+
`Port ${port} is already in use by a non-OpenCode process. Free it or pass --port.`
|
|
8977
|
+
);
|
|
8978
|
+
}
|
|
8979
|
+
}
|
|
7364
8980
|
async function ensureOpenCodeRunning(ctx) {
|
|
7365
8981
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
7366
8982
|
if (healthCheck.healthy) {
|
|
@@ -7408,8 +9024,9 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
7408
9024
|
}
|
|
7409
9025
|
}
|
|
7410
9026
|
if (!ctx.interactive) {
|
|
9027
|
+
checkNonInteractivePortConflict(ctx.port, isPortInUse);
|
|
7411
9028
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
7412
|
-
const proc = await startOpenCode(ctx.port);
|
|
9029
|
+
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
7413
9030
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
7414
9031
|
if (!health.healthy) {
|
|
7415
9032
|
return {
|
|
@@ -7446,47 +9063,702 @@ Port ${port} is already in use.`));
|
|
|
7446
9063
|
throw new Error(`Port ${ctx.port} is in use`);
|
|
7447
9064
|
}
|
|
7448
9065
|
}
|
|
7449
|
-
}
|
|
7450
|
-
const action = await select2({
|
|
7451
|
-
message: "OpenCode is not running. What would you like to do?",
|
|
7452
|
-
choices: [
|
|
7453
|
-
{
|
|
7454
|
-
name: "Start OpenCode for me",
|
|
7455
|
-
value: "start",
|
|
7456
|
-
description: `Run 'opencode serve --port ${port}'`
|
|
7457
|
-
},
|
|
7458
|
-
{
|
|
7459
|
-
name: "Show me the command",
|
|
7460
|
-
value: "manual",
|
|
7461
|
-
description: "Display the command to run manually"
|
|
7462
|
-
},
|
|
7463
|
-
{
|
|
7464
|
-
name: "Continue without OpenCode",
|
|
7465
|
-
value: "continue",
|
|
7466
|
-
description: "Requests will fail until OpenCode starts"
|
|
9066
|
+
}
|
|
9067
|
+
const action = await select2({
|
|
9068
|
+
message: "OpenCode is not running. What would you like to do?",
|
|
9069
|
+
choices: [
|
|
9070
|
+
{
|
|
9071
|
+
name: "Start OpenCode for me",
|
|
9072
|
+
value: "start",
|
|
9073
|
+
description: `Run 'opencode serve --port ${port}'`
|
|
9074
|
+
},
|
|
9075
|
+
{
|
|
9076
|
+
name: "Show me the command",
|
|
9077
|
+
value: "manual",
|
|
9078
|
+
description: "Display the command to run manually"
|
|
9079
|
+
},
|
|
9080
|
+
{
|
|
9081
|
+
name: "Continue without OpenCode",
|
|
9082
|
+
value: "continue",
|
|
9083
|
+
description: "Requests will fail until OpenCode starts"
|
|
9084
|
+
}
|
|
9085
|
+
]
|
|
9086
|
+
});
|
|
9087
|
+
if (action === "manual") {
|
|
9088
|
+
blank();
|
|
9089
|
+
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
9090
|
+
blank();
|
|
9091
|
+
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
9092
|
+
blank();
|
|
9093
|
+
throw new Error("Please start OpenCode manually");
|
|
9094
|
+
}
|
|
9095
|
+
if (action === "start") {
|
|
9096
|
+
const spinner = ora2("Starting OpenCode...").start();
|
|
9097
|
+
const proc = await startOpenCode(port, { inheritStdio: ctx.inheritStdio });
|
|
9098
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
9099
|
+
if (!health.healthy) {
|
|
9100
|
+
spinner.fail("Failed to start OpenCode");
|
|
9101
|
+
throw new Error("OpenCode failed to start");
|
|
9102
|
+
}
|
|
9103
|
+
spinner.stop();
|
|
9104
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
9105
|
+
}
|
|
9106
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
9107
|
+
}
|
|
9108
|
+
|
|
9109
|
+
// src/commands/ensure-opencode-v2.ts
|
|
9110
|
+
import chalk6 from "chalk";
|
|
9111
|
+
import { select as select3 } from "@inquirer/prompts";
|
|
9112
|
+
async function probeOpenCode2WithoutPassword(port) {
|
|
9113
|
+
try {
|
|
9114
|
+
const response = await fetch(`http://127.0.0.1:${port}/global/health`, {
|
|
9115
|
+
signal: AbortSignal.timeout(2e3)
|
|
9116
|
+
});
|
|
9117
|
+
if (response.status === 401) {
|
|
9118
|
+
return { healthy: false, authFailed: true, error: "HTTP 401" };
|
|
9119
|
+
}
|
|
9120
|
+
if (!response.ok) {
|
|
9121
|
+
return { healthy: false, error: `HTTP ${response.status}` };
|
|
9122
|
+
}
|
|
9123
|
+
return { healthy: true };
|
|
9124
|
+
} catch (error2) {
|
|
9125
|
+
return {
|
|
9126
|
+
healthy: false,
|
|
9127
|
+
error: error2 instanceof Error ? error2.message : "Unknown error"
|
|
9128
|
+
};
|
|
9129
|
+
}
|
|
9130
|
+
}
|
|
9131
|
+
function unknownPasswordError(port) {
|
|
9132
|
+
return new Error(
|
|
9133
|
+
`OpenCode V2 (opencode2) is already running on port ${port} with a password this runner doesn't know. Free the port or pass --port.`
|
|
9134
|
+
);
|
|
9135
|
+
}
|
|
9136
|
+
function v2SessionSupportIncompleteError() {
|
|
9137
|
+
return new Error(
|
|
9138
|
+
"OpenCode V2 (opencode2) session support is incomplete \u2014 see https://github.com/sroze/evident/issues/2075. Not starting a new opencode2 process."
|
|
9139
|
+
);
|
|
9140
|
+
}
|
|
9141
|
+
async function ensureOpenCode2Running(ctx) {
|
|
9142
|
+
const initialHealth = await probeOpenCode2WithoutPassword(ctx.port);
|
|
9143
|
+
if (initialHealth.authFailed) {
|
|
9144
|
+
throw unknownPasswordError(ctx.port);
|
|
9145
|
+
}
|
|
9146
|
+
if (initialHealth.healthy) {
|
|
9147
|
+
return {
|
|
9148
|
+
port: ctx.port,
|
|
9149
|
+
process: null,
|
|
9150
|
+
version: null,
|
|
9151
|
+
notReadyReason: null,
|
|
9152
|
+
password: null
|
|
9153
|
+
};
|
|
9154
|
+
}
|
|
9155
|
+
if (!isOpenCode2Installed()) {
|
|
9156
|
+
throw new Error(
|
|
9157
|
+
"OpenCode V2 (opencode2) is not installed. Install it with: npm install -g @opencode-ai/cli@beta"
|
|
9158
|
+
);
|
|
9159
|
+
}
|
|
9160
|
+
let port = ctx.port;
|
|
9161
|
+
if (!ctx.interactive) {
|
|
9162
|
+
checkNonInteractivePortConflict(port, isPortInUse);
|
|
9163
|
+
} else if (isPortInUse(port)) {
|
|
9164
|
+
console.log(chalk6.yellow(`
|
|
9165
|
+
Port ${port} is already in use.`));
|
|
9166
|
+
const alternativePort = findAvailablePort(port + 1);
|
|
9167
|
+
if (alternativePort) {
|
|
9168
|
+
const useAlternative = await select3({
|
|
9169
|
+
message: `Use port ${alternativePort} instead?`,
|
|
9170
|
+
choices: [
|
|
9171
|
+
{ name: `Yes, use port ${alternativePort}`, value: "yes" },
|
|
9172
|
+
{ name: "No, I will free the port manually", value: "no" }
|
|
9173
|
+
]
|
|
9174
|
+
});
|
|
9175
|
+
if (useAlternative === "yes") {
|
|
9176
|
+
port = alternativePort;
|
|
9177
|
+
} else {
|
|
9178
|
+
throw new Error(`Port ${ctx.port} is in use`);
|
|
9179
|
+
}
|
|
9180
|
+
}
|
|
9181
|
+
}
|
|
9182
|
+
if (!ctx.interactive) {
|
|
9183
|
+
throw v2SessionSupportIncompleteError();
|
|
9184
|
+
}
|
|
9185
|
+
console.log(chalk6.yellow(`
|
|
9186
|
+
${v2SessionSupportIncompleteError().message}`));
|
|
9187
|
+
const action = await select3({
|
|
9188
|
+
message: "OpenCode V2 is not running. What would you like to do?",
|
|
9189
|
+
choices: [
|
|
9190
|
+
{
|
|
9191
|
+
name: "Show me the command",
|
|
9192
|
+
value: "manual",
|
|
9193
|
+
description: "Display the command to run manually"
|
|
9194
|
+
},
|
|
9195
|
+
{
|
|
9196
|
+
name: "Continue without OpenCode V2",
|
|
9197
|
+
value: "continue",
|
|
9198
|
+
description: "Requests will fail until OpenCode V2 starts"
|
|
9199
|
+
}
|
|
9200
|
+
]
|
|
9201
|
+
});
|
|
9202
|
+
if (action === "manual") {
|
|
9203
|
+
blank();
|
|
9204
|
+
console.log(chalk6.bold("Run this command in another terminal:"));
|
|
9205
|
+
blank();
|
|
9206
|
+
console.log(` ${chalk6.cyan(`opencode2 serve --port ${port}`)}`);
|
|
9207
|
+
blank();
|
|
9208
|
+
throw new Error("Please start OpenCode V2 manually");
|
|
9209
|
+
}
|
|
9210
|
+
return {
|
|
9211
|
+
port,
|
|
9212
|
+
process: null,
|
|
9213
|
+
version: null,
|
|
9214
|
+
notReadyReason: "you chose to continue without OpenCode V2",
|
|
9215
|
+
password: null
|
|
9216
|
+
};
|
|
9217
|
+
}
|
|
9218
|
+
|
|
9219
|
+
// src/lib/runner-credentials.ts
|
|
9220
|
+
import { chmodSync as chmodSync2, writeFileSync as writeFileSync4 } from "node:fs";
|
|
9221
|
+
import { spawn as spawn5 } from "node:child_process";
|
|
9222
|
+
var RUNNER_SECRET_FETCH_TIMEOUT_MS = 6e4;
|
|
9223
|
+
var CREDENTIAL_RESTORE_TIMEOUT_MS = 6e4;
|
|
9224
|
+
var GITHUB_PROBE_TIMEOUT_MS = 1e4;
|
|
9225
|
+
var GIT_CONFIG_GLOBAL = "/tmp/gitconfig";
|
|
9226
|
+
var GIT_CREDENTIAL_HELPER = "/tmp/git-credential-helper.sh";
|
|
9227
|
+
function commandError2(result) {
|
|
9228
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
9229
|
+
}
|
|
9230
|
+
var runCommand2 = (command, args, opts) => {
|
|
9231
|
+
return new Promise((resolve4) => {
|
|
9232
|
+
let child;
|
|
9233
|
+
let stdout = "";
|
|
9234
|
+
let stderr = "";
|
|
9235
|
+
let settled = false;
|
|
9236
|
+
const timer = {};
|
|
9237
|
+
const finish = (result) => {
|
|
9238
|
+
if (settled) return;
|
|
9239
|
+
settled = true;
|
|
9240
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
9241
|
+
resolve4(result);
|
|
9242
|
+
};
|
|
9243
|
+
try {
|
|
9244
|
+
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
9245
|
+
} catch (error2) {
|
|
9246
|
+
finish({
|
|
9247
|
+
code: null,
|
|
9248
|
+
stdout,
|
|
9249
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
9250
|
+
timedOut: false
|
|
9251
|
+
});
|
|
9252
|
+
return;
|
|
9253
|
+
}
|
|
9254
|
+
child.stdout?.setEncoding("utf8");
|
|
9255
|
+
child.stdout?.on("data", (chunk) => {
|
|
9256
|
+
stdout += chunk;
|
|
9257
|
+
});
|
|
9258
|
+
child.stderr?.setEncoding("utf8");
|
|
9259
|
+
child.stderr?.on("data", (chunk) => {
|
|
9260
|
+
stderr += chunk;
|
|
9261
|
+
});
|
|
9262
|
+
child.once("error", (error2) => {
|
|
9263
|
+
finish({
|
|
9264
|
+
code: null,
|
|
9265
|
+
stdout,
|
|
9266
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
9267
|
+
${error2.message}`,
|
|
9268
|
+
timedOut: false
|
|
9269
|
+
});
|
|
9270
|
+
});
|
|
9271
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
9272
|
+
timer.handle = setTimeout(
|
|
9273
|
+
() => {
|
|
9274
|
+
child.kill("SIGKILL");
|
|
9275
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
9276
|
+
},
|
|
9277
|
+
Math.max(0, opts.timeoutMs)
|
|
9278
|
+
);
|
|
9279
|
+
});
|
|
9280
|
+
};
|
|
9281
|
+
function isEnvironmentObject(value) {
|
|
9282
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9283
|
+
}
|
|
9284
|
+
function secretFailure(marker, detail, log3) {
|
|
9285
|
+
const message = `${marker}: ${detail}`;
|
|
9286
|
+
log3(message, "error");
|
|
9287
|
+
return new Error(message);
|
|
9288
|
+
}
|
|
9289
|
+
async function installRunnerSecret({
|
|
9290
|
+
env,
|
|
9291
|
+
log: log3,
|
|
9292
|
+
commandRunner
|
|
9293
|
+
}) {
|
|
9294
|
+
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
9295
|
+
if (!arn) {
|
|
9296
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
9297
|
+
return false;
|
|
9298
|
+
}
|
|
9299
|
+
const result = await (commandRunner ?? runCommand2)(
|
|
9300
|
+
"aws",
|
|
9301
|
+
[
|
|
9302
|
+
"secretsmanager",
|
|
9303
|
+
"get-secret-value",
|
|
9304
|
+
"--secret-id",
|
|
9305
|
+
arn,
|
|
9306
|
+
"--query",
|
|
9307
|
+
"SecretString",
|
|
9308
|
+
"--output",
|
|
9309
|
+
"text"
|
|
9310
|
+
],
|
|
9311
|
+
{ env, timeoutMs: RUNNER_SECRET_FETCH_TIMEOUT_MS }
|
|
9312
|
+
);
|
|
9313
|
+
if (result.timedOut) {
|
|
9314
|
+
throw secretFailure(
|
|
9315
|
+
"CREDENTIAL-RESTORE-TIMEOUT",
|
|
9316
|
+
`runner-secret did not finish within ${RUNNER_SECRET_FETCH_TIMEOUT_MS}ms`,
|
|
9317
|
+
log3
|
|
9318
|
+
);
|
|
9319
|
+
}
|
|
9320
|
+
if (result.code !== 0) {
|
|
9321
|
+
throw secretFailure("RUNNER-SECRET-UNREADABLE", commandError2(result), log3);
|
|
9322
|
+
}
|
|
9323
|
+
let payload;
|
|
9324
|
+
try {
|
|
9325
|
+
payload = JSON.parse(result.stdout);
|
|
9326
|
+
} catch (error2) {
|
|
9327
|
+
log3(
|
|
9328
|
+
`RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9329
|
+
"warn"
|
|
9330
|
+
);
|
|
9331
|
+
return false;
|
|
9332
|
+
}
|
|
9333
|
+
if (!isEnvironmentObject(payload)) {
|
|
9334
|
+
log3("RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object", "warn");
|
|
9335
|
+
return false;
|
|
9336
|
+
}
|
|
9337
|
+
let populated = 0;
|
|
9338
|
+
let skipped = 0;
|
|
9339
|
+
let githubTokenPopulated = false;
|
|
9340
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
9341
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
9342
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
9343
|
+
log3(
|
|
9344
|
+
`RUNNER-SECRET-KEY-SKIPPED: ${JSON.stringify(key)} is not a valid environment variable name`,
|
|
9345
|
+
"warn"
|
|
9346
|
+
);
|
|
9347
|
+
skipped += 1;
|
|
9348
|
+
continue;
|
|
9349
|
+
}
|
|
9350
|
+
env[key] = value;
|
|
9351
|
+
populated += 1;
|
|
9352
|
+
if (key === "GH_TOKEN") githubTokenPopulated = true;
|
|
9353
|
+
}
|
|
9354
|
+
if (populated === 0) {
|
|
9355
|
+
log3(
|
|
9356
|
+
"RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md",
|
|
9357
|
+
"warn"
|
|
9358
|
+
);
|
|
9359
|
+
} else {
|
|
9360
|
+
log3(
|
|
9361
|
+
`RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names`
|
|
9362
|
+
);
|
|
9363
|
+
}
|
|
9364
|
+
return githubTokenPopulated;
|
|
9365
|
+
}
|
|
9366
|
+
function restoreFailure(operation, result, log3) {
|
|
9367
|
+
const message = `CREDENTIAL-RESTORE-FAILED: ${operation} failed: ${commandError2(result)}`;
|
|
9368
|
+
log3(message, "error");
|
|
9369
|
+
return new Error(message);
|
|
9370
|
+
}
|
|
9371
|
+
async function runCredentialRestore(args, operation, log3, synchroniserRunner) {
|
|
9372
|
+
const result = await synchroniserRunner(args, { timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS });
|
|
9373
|
+
if (result.timedOut) {
|
|
9374
|
+
log3(
|
|
9375
|
+
`CREDENTIAL-RESTORE-TIMEOUT: ${operation} did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9376
|
+
"warn"
|
|
9377
|
+
);
|
|
9378
|
+
return result;
|
|
9379
|
+
}
|
|
9380
|
+
if (result.code !== 0) throw restoreFailure(operation, result, log3);
|
|
9381
|
+
return result;
|
|
9382
|
+
}
|
|
9383
|
+
async function restoreCredentialStores({
|
|
9384
|
+
env,
|
|
9385
|
+
log: log3,
|
|
9386
|
+
synchroniserRunner = runSynchroniser
|
|
9387
|
+
}) {
|
|
9388
|
+
await runCredentialRestore(["restore", "claude"], "restore-claude", log3, synchroniserRunner);
|
|
9389
|
+
await runCredentialRestore(["restore", "opencode"], "restore-opencode", log3, synchroniserRunner);
|
|
9390
|
+
const result = await synchroniserRunner(["model-auth-ready"], {
|
|
9391
|
+
timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS
|
|
9392
|
+
});
|
|
9393
|
+
if (result.timedOut) {
|
|
9394
|
+
log3(
|
|
9395
|
+
`CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9396
|
+
"warn"
|
|
9397
|
+
);
|
|
9398
|
+
return;
|
|
9399
|
+
}
|
|
9400
|
+
switch (result.code) {
|
|
9401
|
+
case 0:
|
|
9402
|
+
return;
|
|
9403
|
+
case 10:
|
|
9404
|
+
log3(
|
|
9405
|
+
`no model credentials under s3://${env.LITESTREAM_BUCKET ?? ""}/${env.LITESTREAM_PREFIX ?? ""}/ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs a model provider fails until one is connected. See 'Seeding a credential store' in infrastructure/evident-runner/MICROVM.md.`,
|
|
9406
|
+
"warn"
|
|
9407
|
+
);
|
|
9408
|
+
return;
|
|
9409
|
+
default:
|
|
9410
|
+
log3("could not determine whether this VM has model credentials", "warn");
|
|
9411
|
+
}
|
|
9412
|
+
}
|
|
9413
|
+
var GIT_CREDENTIAL_HELPER_CONTENT = [
|
|
9414
|
+
"#!/usr/bin/env bash",
|
|
9415
|
+
'[ "$1" = get ] || exit 0',
|
|
9416
|
+
"echo username=x-access-token",
|
|
9417
|
+
'echo "password=${GH_TOKEN}"',
|
|
9418
|
+
""
|
|
9419
|
+
].join("\n");
|
|
9420
|
+
async function probeGitHubAccess({ env, log: log3 }) {
|
|
9421
|
+
const auth = await runCommand2("gh", ["api", "user", "--jq", ".login"], {
|
|
9422
|
+
env,
|
|
9423
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9424
|
+
});
|
|
9425
|
+
if (auth.timedOut) {
|
|
9426
|
+
log3(`GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`, "warn");
|
|
9427
|
+
return;
|
|
9428
|
+
}
|
|
9429
|
+
if (auth.code !== 0) {
|
|
9430
|
+
log3(`GITHUB-AUTH-REJECTED: ${commandError2(auth)}`, "warn");
|
|
9431
|
+
return;
|
|
9432
|
+
}
|
|
9433
|
+
log3(`GITHUB-AUTH-OK: ${auth.stdout.trim()}`, "debug");
|
|
9434
|
+
const remote = await runCommand2("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
|
9435
|
+
env,
|
|
9436
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9437
|
+
});
|
|
9438
|
+
if (remote.code !== 0 || remote.timedOut) return;
|
|
9439
|
+
const repo = remote.stdout.trim().replace(/^(?:https:\/\/github\.com\/|git@github\.com:)/, "").replace(/\.git$/, "");
|
|
9440
|
+
if (!repo) return;
|
|
9441
|
+
const repository = await runCommand2("gh", ["api", `repos/${repo}`, "--jq", ".full_name"], {
|
|
9442
|
+
env,
|
|
9443
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9444
|
+
});
|
|
9445
|
+
if (repository.timedOut) {
|
|
9446
|
+
log3(
|
|
9447
|
+
`GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`,
|
|
9448
|
+
"warn"
|
|
9449
|
+
);
|
|
9450
|
+
} else if (repository.code !== 0) {
|
|
9451
|
+
log3(`GITHUB-REPO-INACCESSIBLE: ${repo}: ${commandError2(repository)}`, "warn");
|
|
9452
|
+
}
|
|
9453
|
+
}
|
|
9454
|
+
async function configureGitHubAccess({ env, log: log3 }) {
|
|
9455
|
+
if (!env.GH_TOKEN) {
|
|
9456
|
+
log3("GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above", "warn");
|
|
9457
|
+
return;
|
|
9458
|
+
}
|
|
9459
|
+
try {
|
|
9460
|
+
env.GIT_CONFIG_GLOBAL = GIT_CONFIG_GLOBAL;
|
|
9461
|
+
writeFileSync4(GIT_CONFIG_GLOBAL, "");
|
|
9462
|
+
writeFileSync4(GIT_CREDENTIAL_HELPER, GIT_CREDENTIAL_HELPER_CONTENT, { mode: 448 });
|
|
9463
|
+
chmodSync2(GIT_CREDENTIAL_HELPER, 448);
|
|
9464
|
+
const config = [
|
|
9465
|
+
["user.name", env.GIT_USER_NAME ?? "evident-bot"],
|
|
9466
|
+
["user.email", env.GIT_USER_EMAIL ?? "evident-bot@users.noreply.github.com"],
|
|
9467
|
+
["init.defaultBranch", "main"],
|
|
9468
|
+
["credential.https://github.com.helper", GIT_CREDENTIAL_HELPER]
|
|
9469
|
+
];
|
|
9470
|
+
for (const [key, value] of config) {
|
|
9471
|
+
const result = await runCommand2("git", ["config", "--global", key, value], {
|
|
9472
|
+
env,
|
|
9473
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9474
|
+
});
|
|
9475
|
+
if (result.timedOut || result.code !== 0) throw new Error(commandError2(result));
|
|
9476
|
+
}
|
|
9477
|
+
} catch (error2) {
|
|
9478
|
+
log3(
|
|
9479
|
+
`GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9480
|
+
"warn"
|
|
9481
|
+
);
|
|
9482
|
+
return;
|
|
9483
|
+
}
|
|
9484
|
+
void probeGitHubAccess({ env, log: log3 }).catch((error2) => {
|
|
9485
|
+
log3(
|
|
9486
|
+
`GITHUB-SETUP-FAILED: GitHub access probe failed: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9487
|
+
"warn"
|
|
9488
|
+
);
|
|
9489
|
+
});
|
|
9490
|
+
}
|
|
9491
|
+
|
|
9492
|
+
// src/lib/opencode/config-overlay.ts
|
|
9493
|
+
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
9494
|
+
import { copyFileSync, existsSync as existsSync2, statSync as statSync5 } from "node:fs";
|
|
9495
|
+
import { isAbsolute as isAbsolute2, join as join8, resolve as resolve3 } from "node:path";
|
|
9496
|
+
function isFile(filePath) {
|
|
9497
|
+
return existsSync2(filePath) && statSync5(filePath).isFile();
|
|
9498
|
+
}
|
|
9499
|
+
function applyRunnerOpenCodeConfig({
|
|
9500
|
+
overlayPath,
|
|
9501
|
+
cwd = process.cwd(),
|
|
9502
|
+
log: log3
|
|
9503
|
+
}) {
|
|
9504
|
+
if (!overlayPath) {
|
|
9505
|
+
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
9506
|
+
return;
|
|
9507
|
+
}
|
|
9508
|
+
const source = isAbsolute2(overlayPath) ? overlayPath : resolve3(cwd, overlayPath);
|
|
9509
|
+
const target = isFile(join8(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
9510
|
+
if (!isFile(source)) {
|
|
9511
|
+
log3(
|
|
9512
|
+
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
9513
|
+
"error"
|
|
9514
|
+
);
|
|
9515
|
+
return;
|
|
9516
|
+
}
|
|
9517
|
+
copyFileSync(source, join8(cwd, target));
|
|
9518
|
+
try {
|
|
9519
|
+
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
9520
|
+
stdio: "ignore"
|
|
9521
|
+
});
|
|
9522
|
+
} catch (error2) {
|
|
9523
|
+
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
9524
|
+
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
9525
|
+
}
|
|
9526
|
+
log3(`Applied runner OpenCode config ${source} to ${join8(cwd, target)}`);
|
|
9527
|
+
}
|
|
9528
|
+
|
|
9529
|
+
// src/lib/credential-sync.ts
|
|
9530
|
+
import { renameSync, writeFileSync as writeFileSync5 } from "node:fs";
|
|
9531
|
+
var CREDENTIAL_SYNC_TICK_TIMEOUT_MS = 3e4;
|
|
9532
|
+
var CREDENTIAL_FLUSH_DEADLINE_MS = 8e3;
|
|
9533
|
+
var CREDENTIAL_FLUSH_ABORT_GRACE_MS = 500;
|
|
9534
|
+
var DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS = 60;
|
|
9535
|
+
var STORES = ["claude", "opencode"];
|
|
9536
|
+
var MAX_FLUSH_PASSES = 2;
|
|
9537
|
+
function outcomesWith(outcome) {
|
|
9538
|
+
return { claude: outcome, opencode: outcome };
|
|
9539
|
+
}
|
|
9540
|
+
function errorMessage2(error2) {
|
|
9541
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
9542
|
+
}
|
|
9543
|
+
function waitForSettlement(promise, timeoutMs) {
|
|
9544
|
+
return new Promise((resolve4) => {
|
|
9545
|
+
let settled = false;
|
|
9546
|
+
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
9547
|
+
const finish = (value) => {
|
|
9548
|
+
if (settled) return;
|
|
9549
|
+
settled = true;
|
|
9550
|
+
clearTimeout(timer);
|
|
9551
|
+
resolve4(value);
|
|
9552
|
+
};
|
|
9553
|
+
promise.then(
|
|
9554
|
+
() => finish(true),
|
|
9555
|
+
() => finish(true)
|
|
9556
|
+
);
|
|
9557
|
+
});
|
|
9558
|
+
}
|
|
9559
|
+
function writeMarker(markerPath, outcomes, log3) {
|
|
9560
|
+
const body = `${STORES.map((store) => `${store}=${outcomes[store]}`).join("\n")}
|
|
9561
|
+
`;
|
|
9562
|
+
const temporaryPath = `${markerPath}.tmp`;
|
|
9563
|
+
try {
|
|
9564
|
+
writeFileSync5(temporaryPath, body, { mode: 384 });
|
|
9565
|
+
renameSync(temporaryPath, markerPath);
|
|
9566
|
+
} catch (error2) {
|
|
9567
|
+
log3(`CREDENTIAL-FLUSH-MARKER-UNWRITABLE: ${markerPath}: ${errorMessage2(error2)}`, "warn");
|
|
9568
|
+
}
|
|
9569
|
+
}
|
|
9570
|
+
function intervalSeconds(env, log3) {
|
|
9571
|
+
const raw = env.CREDS_SYNC_INTERVAL;
|
|
9572
|
+
if (raw === void 0 || /^[1-9][0-9]*$/.test(raw) && Number.isSafeInteger(Number(raw))) {
|
|
9573
|
+
return raw === void 0 ? DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS : Number(raw);
|
|
9574
|
+
}
|
|
9575
|
+
log3(
|
|
9576
|
+
`CREDENTIAL-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${raw}' is not a positive integer; using ${DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS}s`,
|
|
9577
|
+
"warn"
|
|
9578
|
+
);
|
|
9579
|
+
return DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS;
|
|
9580
|
+
}
|
|
9581
|
+
async function runFlushStore(store, env, synchroniserRunner, log3, deadlineAt) {
|
|
9582
|
+
const remainingMs = deadlineAt - Date.now();
|
|
9583
|
+
if (remainingMs <= 0) return { outcome: "timeout", orphaned: false };
|
|
9584
|
+
const controller = new AbortController();
|
|
9585
|
+
let result;
|
|
9586
|
+
let failed = false;
|
|
9587
|
+
const completion = Promise.resolve().then(
|
|
9588
|
+
() => synchroniserRunner(["sync-once", store], {
|
|
9589
|
+
timeoutMs: remainingMs,
|
|
9590
|
+
env,
|
|
9591
|
+
signal: controller.signal
|
|
9592
|
+
})
|
|
9593
|
+
).then(
|
|
9594
|
+
(value) => {
|
|
9595
|
+
result = value;
|
|
9596
|
+
},
|
|
9597
|
+
(error2) => {
|
|
9598
|
+
failed = true;
|
|
9599
|
+
log3(`CREDENTIAL-FLUSH-ERROR: ${store}: ${errorMessage2(error2)}`, "warn");
|
|
9600
|
+
}
|
|
9601
|
+
);
|
|
9602
|
+
const abortTimer = setTimeout(() => controller.abort(), remainingMs);
|
|
9603
|
+
const settledBeforeDeadline = await waitForSettlement(completion, remainingMs);
|
|
9604
|
+
clearTimeout(abortTimer);
|
|
9605
|
+
if (!settledBeforeDeadline) {
|
|
9606
|
+
controller.abort();
|
|
9607
|
+
const settledAfterAbort = await waitForSettlement(completion, CREDENTIAL_FLUSH_ABORT_GRACE_MS);
|
|
9608
|
+
if (!settledAfterAbort) return { outcome: "timeout", orphaned: true };
|
|
9609
|
+
return { outcome: "timeout", orphaned: false };
|
|
9610
|
+
}
|
|
9611
|
+
if (failed || !result) return { outcome: "failed", orphaned: false };
|
|
9612
|
+
if (result.timedOut || Date.now() >= deadlineAt) {
|
|
9613
|
+
return { outcome: "timeout", orphaned: false };
|
|
9614
|
+
}
|
|
9615
|
+
return { outcome: result.code === 0 ? "ok" : "failed", orphaned: false };
|
|
9616
|
+
}
|
|
9617
|
+
function createCredentialSync({
|
|
9618
|
+
markerPath,
|
|
9619
|
+
env,
|
|
9620
|
+
log: log3,
|
|
9621
|
+
synchroniserRunner = runSynchroniser
|
|
9622
|
+
}) {
|
|
9623
|
+
const persistenceDisabled = !env.PERSISTENCE_BUCKET;
|
|
9624
|
+
let disabled = persistenceDisabled;
|
|
9625
|
+
let armed = false;
|
|
9626
|
+
let stopped = false;
|
|
9627
|
+
let timer;
|
|
9628
|
+
let inFlight;
|
|
9629
|
+
let activeTickAbort;
|
|
9630
|
+
let lastTickFailed;
|
|
9631
|
+
let flushPromise;
|
|
9632
|
+
const scheduleTick = (intervalMs, startTick2) => {
|
|
9633
|
+
if (stopped) return;
|
|
9634
|
+
timer = setTimeout(() => {
|
|
9635
|
+
timer = void 0;
|
|
9636
|
+
startTick2();
|
|
9637
|
+
}, intervalMs);
|
|
9638
|
+
};
|
|
9639
|
+
const startTick = (intervalMs) => {
|
|
9640
|
+
if (stopped) return;
|
|
9641
|
+
const controller = new AbortController();
|
|
9642
|
+
activeTickAbort = controller;
|
|
9643
|
+
const tick = (async () => {
|
|
9644
|
+
const outcomes = {
|
|
9645
|
+
claude: "failed",
|
|
9646
|
+
opencode: "failed"
|
|
9647
|
+
};
|
|
9648
|
+
for (const store of STORES) {
|
|
9649
|
+
if (controller.signal.aborted) break;
|
|
9650
|
+
try {
|
|
9651
|
+
const result = await synchroniserRunner(["sync-once", store], {
|
|
9652
|
+
timeoutMs: CREDENTIAL_SYNC_TICK_TIMEOUT_MS,
|
|
9653
|
+
env,
|
|
9654
|
+
signal: controller.signal
|
|
9655
|
+
});
|
|
9656
|
+
outcomes[store] = !result.timedOut && result.code === 0 ? "ok" : "failed";
|
|
9657
|
+
} catch (error2) {
|
|
9658
|
+
outcomes[store] = "failed";
|
|
9659
|
+
log3(`CREDENTIAL-SYNC-TICK-ERROR: ${store}: ${errorMessage2(error2)}`, "debug");
|
|
9660
|
+
}
|
|
9661
|
+
}
|
|
9662
|
+
const failed = STORES.some((store) => outcomes[store] === "failed");
|
|
9663
|
+
log3(
|
|
9664
|
+
`CREDENTIAL-SYNC-TICK: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
9665
|
+
"debug"
|
|
9666
|
+
);
|
|
9667
|
+
if (failed && lastTickFailed !== true) {
|
|
9668
|
+
log3(
|
|
9669
|
+
"CREDENTIAL-SYNC-FAILED: an interval credential sync failed; retrying on the next tick",
|
|
9670
|
+
"warn"
|
|
9671
|
+
);
|
|
9672
|
+
} else if (!failed && lastTickFailed === true) {
|
|
9673
|
+
log3("CREDENTIAL-SYNC-RECOVERED: interval credential sync succeeded again", "warn");
|
|
9674
|
+
}
|
|
9675
|
+
lastTickFailed = failed;
|
|
9676
|
+
})().finally(() => {
|
|
9677
|
+
if (activeTickAbort === controller) activeTickAbort = void 0;
|
|
9678
|
+
if (inFlight === tick) inFlight = void 0;
|
|
9679
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9680
|
+
});
|
|
9681
|
+
inFlight = tick;
|
|
9682
|
+
};
|
|
9683
|
+
const performFlush = async () => {
|
|
9684
|
+
stopped = true;
|
|
9685
|
+
if (timer) {
|
|
9686
|
+
clearTimeout(timer);
|
|
9687
|
+
timer = void 0;
|
|
9688
|
+
}
|
|
9689
|
+
const deadlineAt = Date.now() + CREDENTIAL_FLUSH_DEADLINE_MS;
|
|
9690
|
+
if (inFlight) {
|
|
9691
|
+
const settled = await waitForSettlement(inFlight, CREDENTIAL_FLUSH_DEADLINE_MS);
|
|
9692
|
+
if (!settled) {
|
|
9693
|
+
activeTickAbort?.abort();
|
|
9694
|
+
const settledAfterAbort = await waitForSettlement(
|
|
9695
|
+
inFlight,
|
|
9696
|
+
CREDENTIAL_FLUSH_ABORT_GRACE_MS
|
|
9697
|
+
);
|
|
9698
|
+
if (!settledAfterAbort) {
|
|
9699
|
+
log3(
|
|
9700
|
+
"CREDENTIAL-FLUSH-ORPHANED-TICK: a sync-once child could not be proven settled; skipping the boundary flush rather than becoming a second writer",
|
|
9701
|
+
"warn"
|
|
9702
|
+
);
|
|
9703
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9704
|
+
}
|
|
9705
|
+
}
|
|
9706
|
+
}
|
|
9707
|
+
if (disabled) return { outcomes: outcomesWith("skipped"), orphaned: false };
|
|
9708
|
+
const outcomes = outcomesWith("timeout");
|
|
9709
|
+
for (const store of STORES) {
|
|
9710
|
+
const result = await runFlushStore(store, env, synchroniserRunner, log3, deadlineAt);
|
|
9711
|
+
if (result.orphaned) {
|
|
9712
|
+
log3(
|
|
9713
|
+
"CREDENTIAL-FLUSH-ORPHANED-CHILD: a flush sync-once child could not be proven settled; refusing another flush pass rather than becoming a second writer",
|
|
9714
|
+
"warn"
|
|
9715
|
+
);
|
|
9716
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
7467
9717
|
}
|
|
7468
|
-
|
|
7469
|
-
});
|
|
7470
|
-
if (action === "manual") {
|
|
7471
|
-
blank();
|
|
7472
|
-
console.log(chalk5.bold("Run this command in another terminal:"));
|
|
7473
|
-
blank();
|
|
7474
|
-
console.log(` ${chalk5.cyan(`opencode serve --port ${port}`)}`);
|
|
7475
|
-
blank();
|
|
7476
|
-
throw new Error("Please start OpenCode manually");
|
|
7477
|
-
}
|
|
7478
|
-
if (action === "start") {
|
|
7479
|
-
const spinner = ora2("Starting OpenCode...").start();
|
|
7480
|
-
const proc = await startOpenCode(port);
|
|
7481
|
-
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
7482
|
-
if (!health.healthy) {
|
|
7483
|
-
spinner.fail("Failed to start OpenCode");
|
|
7484
|
-
throw new Error("OpenCode failed to start");
|
|
9718
|
+
outcomes[store] = result.outcome;
|
|
7485
9719
|
}
|
|
7486
|
-
|
|
7487
|
-
|
|
7488
|
-
|
|
7489
|
-
|
|
9720
|
+
return { outcomes, orphaned: false };
|
|
9721
|
+
};
|
|
9722
|
+
let flushPasses = 0;
|
|
9723
|
+
let lastFlush;
|
|
9724
|
+
return {
|
|
9725
|
+
arm() {
|
|
9726
|
+
if (stopped || armed) return;
|
|
9727
|
+
armed = true;
|
|
9728
|
+
if (persistenceDisabled) {
|
|
9729
|
+
disabled = true;
|
|
9730
|
+
log3(
|
|
9731
|
+
"CREDENTIAL-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot",
|
|
9732
|
+
"warn"
|
|
9733
|
+
);
|
|
9734
|
+
return;
|
|
9735
|
+
}
|
|
9736
|
+
disabled = false;
|
|
9737
|
+
const intervalMs = intervalSeconds(env, log3) * 1e3;
|
|
9738
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9739
|
+
},
|
|
9740
|
+
async stopAndFlush(publish) {
|
|
9741
|
+
let result;
|
|
9742
|
+
const runningFlush = flushPromise;
|
|
9743
|
+
if (runningFlush) {
|
|
9744
|
+
result = await runningFlush;
|
|
9745
|
+
} else if (flushPasses >= MAX_FLUSH_PASSES || lastFlush?.orphaned) {
|
|
9746
|
+
result = lastFlush ?? { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9747
|
+
} else {
|
|
9748
|
+
flushPasses++;
|
|
9749
|
+
const currentFlush = performFlush();
|
|
9750
|
+
flushPromise = currentFlush;
|
|
9751
|
+
try {
|
|
9752
|
+
result = await currentFlush;
|
|
9753
|
+
lastFlush = result;
|
|
9754
|
+
} finally {
|
|
9755
|
+
if (flushPromise === currentFlush) flushPromise = void 0;
|
|
9756
|
+
}
|
|
9757
|
+
}
|
|
9758
|
+
if (publish) writeMarker(markerPath, result.outcomes, log3);
|
|
9759
|
+
return result.outcomes;
|
|
9760
|
+
}
|
|
9761
|
+
};
|
|
7490
9762
|
}
|
|
7491
9763
|
|
|
7492
9764
|
// src/commands/run.ts
|
|
@@ -7495,6 +9767,7 @@ var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_
|
|
|
7495
9767
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
7496
9768
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
7497
9769
|
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
9770
|
+
var CHILD_STOP_TIMEOUT_MS = 1e4;
|
|
7498
9771
|
function resolveLogLevel(options) {
|
|
7499
9772
|
const accepted = Object.keys(LOG_LEVELS);
|
|
7500
9773
|
const validate = (value, source) => {
|
|
@@ -7525,11 +9798,11 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
7525
9798
|
if (trimmed === "") {
|
|
7526
9799
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
7527
9800
|
}
|
|
7528
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7529
|
-
if (!
|
|
9801
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join9(homeDir, trimmed.slice(2)) : trimmed;
|
|
9802
|
+
if (!isAbsolute3(expanded)) {
|
|
7530
9803
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
7531
9804
|
}
|
|
7532
|
-
const normalized =
|
|
9805
|
+
const normalized = resolvePath2(expanded);
|
|
7533
9806
|
if (parse(normalized).root === normalized) {
|
|
7534
9807
|
throw new Error(
|
|
7535
9808
|
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
@@ -7549,6 +9822,28 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
7549
9822
|
var DEFAULT_OPENCODE_START_TIMEOUT_SECONDS = 180;
|
|
7550
9823
|
var MAX_OPENCODE_START_TIMEOUT_SECONDS = 3600;
|
|
7551
9824
|
var OPENCODE_START_TIMEOUT_ENV = "EVIDENT_OPENCODE_START_TIMEOUT";
|
|
9825
|
+
var OPENCODE_VERSION_ENV = "EVIDENT_OPENCODE_VERSION";
|
|
9826
|
+
function resolveOpenCodeVersion(options, env = process.env) {
|
|
9827
|
+
let raw;
|
|
9828
|
+
let source;
|
|
9829
|
+
if (options.opencodeVersion !== void 0) {
|
|
9830
|
+
raw = options.opencodeVersion;
|
|
9831
|
+
source = "--opencode-version";
|
|
9832
|
+
} else if (env[OPENCODE_VERSION_ENV] !== void 0 && env[OPENCODE_VERSION_ENV] !== "") {
|
|
9833
|
+
raw = env[OPENCODE_VERSION_ENV];
|
|
9834
|
+
source = OPENCODE_VERSION_ENV;
|
|
9835
|
+
} else {
|
|
9836
|
+
return { version: "v1", warnings: [] };
|
|
9837
|
+
}
|
|
9838
|
+
const normalized = raw.trim().toLowerCase();
|
|
9839
|
+
if (normalized !== "v1" && normalized !== "v2") {
|
|
9840
|
+
return {
|
|
9841
|
+
version: "v1",
|
|
9842
|
+
warnings: [`Ignoring invalid ${source} "${raw}": expected v1 or v2; using the default v1`]
|
|
9843
|
+
};
|
|
9844
|
+
}
|
|
9845
|
+
return { version: normalized, warnings: [] };
|
|
9846
|
+
}
|
|
7552
9847
|
function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
7553
9848
|
const defaultMs = DEFAULT_OPENCODE_START_TIMEOUT_SECONDS * 1e3;
|
|
7554
9849
|
let raw;
|
|
@@ -7615,7 +9910,7 @@ function log2(state, message, level = "info") {
|
|
|
7615
9910
|
})
|
|
7616
9911
|
);
|
|
7617
9912
|
} else if (!state.interactive) {
|
|
7618
|
-
const prefix = level === "error" ?
|
|
9913
|
+
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
7619
9914
|
console.log(`${prefix} ${message}`);
|
|
7620
9915
|
}
|
|
7621
9916
|
}
|
|
@@ -7623,7 +9918,7 @@ function logActivity(state, entry) {
|
|
|
7623
9918
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
7624
9919
|
if (!meetsThreshold(state, level)) return;
|
|
7625
9920
|
forwardRunnerActivity(
|
|
7626
|
-
{ level, message: entry.message, error: entry.error },
|
|
9921
|
+
{ level, message: entry.message, error: entry.error, metadata: entry.metadata },
|
|
7627
9922
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
7628
9923
|
);
|
|
7629
9924
|
const fullEntry = {
|
|
@@ -7643,21 +9938,51 @@ function logActivity(state, entry) {
|
|
|
7643
9938
|
}
|
|
7644
9939
|
}
|
|
7645
9940
|
}
|
|
9941
|
+
function reportSessionDbRecovery(state) {
|
|
9942
|
+
try {
|
|
9943
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir5(), env: process.env });
|
|
9944
|
+
for (const record of report.records) {
|
|
9945
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9946
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9947
|
+
logActivity(state, {
|
|
9948
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9949
|
+
level: activity.level,
|
|
9950
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9951
|
+
metadata: activity.metadata
|
|
9952
|
+
});
|
|
9953
|
+
}
|
|
9954
|
+
acknowledgeSessionDbRecoveryReport(report.path);
|
|
9955
|
+
} catch (error2) {
|
|
9956
|
+
console.error(
|
|
9957
|
+
`[run] could not report session-DB recovery activity: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
9958
|
+
);
|
|
9959
|
+
}
|
|
9960
|
+
}
|
|
9961
|
+
function reportSessionDbRecoveryRecord(state, record) {
|
|
9962
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9963
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9964
|
+
logActivity(state, {
|
|
9965
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9966
|
+
level: activity.level,
|
|
9967
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9968
|
+
metadata: activity.metadata
|
|
9969
|
+
});
|
|
9970
|
+
}
|
|
7646
9971
|
function displayStatus(state) {
|
|
7647
9972
|
if (!state.interactive) return;
|
|
7648
9973
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
7649
|
-
const tunnel = state.connected ?
|
|
7650
|
-
const opencode = state.opencodeConnected ?
|
|
7651
|
-
const messages = state.messageCount > 0 ?
|
|
9974
|
+
const tunnel = state.connected ? chalk7.green("tunnel: connected") : attempt > 0 ? chalk7.yellow(`tunnel: reconnecting (#${attempt})`) : chalk7.yellow("tunnel: connecting");
|
|
9975
|
+
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
9976
|
+
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
7652
9977
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
7653
|
-
const detail = last ?
|
|
9978
|
+
const detail = last ? chalk7.dim(` \xB7 ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`) : "";
|
|
7654
9979
|
const agent = state.agentName ?? state.agentId;
|
|
7655
9980
|
console.log(
|
|
7656
|
-
`${
|
|
9981
|
+
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
7657
9982
|
);
|
|
7658
9983
|
}
|
|
7659
9984
|
async function promptForLogin(promptMessage, successMessage) {
|
|
7660
|
-
const action = await
|
|
9985
|
+
const action = await select4({
|
|
7661
9986
|
message: promptMessage,
|
|
7662
9987
|
choices: [
|
|
7663
9988
|
{
|
|
@@ -7673,7 +9998,7 @@ async function promptForLogin(promptMessage, successMessage) {
|
|
|
7673
9998
|
]
|
|
7674
9999
|
});
|
|
7675
10000
|
if (action === "exit") {
|
|
7676
|
-
console.log(
|
|
10001
|
+
console.log(chalk7.dim(`
|
|
7677
10002
|
You can log in later by running: ${getCliName()} login`));
|
|
7678
10003
|
process.exit(0);
|
|
7679
10004
|
}
|
|
@@ -7684,7 +10009,7 @@ You can log in later by running: ${getCliName()} login`));
|
|
|
7684
10009
|
process.exit(1);
|
|
7685
10010
|
}
|
|
7686
10011
|
blank();
|
|
7687
|
-
console.log(
|
|
10012
|
+
console.log(chalk7.green(successMessage));
|
|
7688
10013
|
blank();
|
|
7689
10014
|
return { token: credentials2.token, authType: "bearer", user: credentials2.user };
|
|
7690
10015
|
}
|
|
@@ -7697,12 +10022,12 @@ async function handleAuthError(state, error2) {
|
|
|
7697
10022
|
if (state.interactive) displayStatus(state);
|
|
7698
10023
|
if (!state.interactive) {
|
|
7699
10024
|
blank();
|
|
7700
|
-
console.log(
|
|
7701
|
-
console.log(
|
|
10025
|
+
console.log(chalk7.red("Authentication expired"));
|
|
10026
|
+
console.log(chalk7.dim("Your authentication token is no longer valid."));
|
|
7702
10027
|
blank();
|
|
7703
|
-
console.log(
|
|
7704
|
-
console.log(
|
|
7705
|
-
console.log(
|
|
10028
|
+
console.log(chalk7.dim("To fix this:"));
|
|
10029
|
+
console.log(chalk7.dim(` 1. Run '${getCliName()} login' to re-authenticate`));
|
|
10030
|
+
console.log(chalk7.dim(" 2. Restart this command"));
|
|
7706
10031
|
blank();
|
|
7707
10032
|
await cleanup(state);
|
|
7708
10033
|
await shutdownTelemetry();
|
|
@@ -7710,7 +10035,7 @@ async function handleAuthError(state, error2) {
|
|
|
7710
10035
|
return { success: false };
|
|
7711
10036
|
}
|
|
7712
10037
|
blank();
|
|
7713
|
-
console.log(
|
|
10038
|
+
console.log(chalk7.yellow("Your authentication has expired."));
|
|
7714
10039
|
blank();
|
|
7715
10040
|
try {
|
|
7716
10041
|
const credentials2 = await promptForLogin(
|
|
@@ -7733,6 +10058,7 @@ async function driveChannels(state, driver) {
|
|
|
7733
10058
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7734
10059
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
7735
10060
|
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
10061
|
+
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
7736
10062
|
while (state.running) {
|
|
7737
10063
|
const cycleStartedAtMs = performance.now();
|
|
7738
10064
|
let idleThisCycle = false;
|
|
@@ -7754,6 +10080,10 @@ async function driveChannels(state, driver) {
|
|
|
7754
10080
|
consecutiveDrainFailures = 0;
|
|
7755
10081
|
unreachableMs = 0;
|
|
7756
10082
|
state.messageCount += processed;
|
|
10083
|
+
if (driver.recycleRequested) {
|
|
10084
|
+
await beginGracefulShutdown(state, "recycle");
|
|
10085
|
+
return;
|
|
10086
|
+
}
|
|
7757
10087
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
7758
10088
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
7759
10089
|
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
@@ -7765,6 +10095,10 @@ async function driveChannels(state, driver) {
|
|
|
7765
10095
|
const claudeCredentialApplied = claudeCredentialApplies !== lastSeenClaudeApplies;
|
|
7766
10096
|
lastSeenClaudeApplies = claudeCredentialApplies;
|
|
7767
10097
|
if (claudeCredentialApplied) state.claudeUsageRearm?.();
|
|
10098
|
+
const opencodeAuthApplies = fileActivitySnapshot.opencodeAuthApplyCount;
|
|
10099
|
+
const opencodeAuthApplied = opencodeAuthApplies !== lastSeenOpencodeAuthApplies;
|
|
10100
|
+
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
10101
|
+
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
7768
10102
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
7769
10103
|
idlePolls = 0;
|
|
7770
10104
|
idleMs = 0;
|
|
@@ -7792,8 +10126,8 @@ async function driveChannels(state, driver) {
|
|
|
7792
10126
|
state.running = false;
|
|
7793
10127
|
break;
|
|
7794
10128
|
}
|
|
7795
|
-
const
|
|
7796
|
-
logActivity(state, { type: "error", error: `Channel processing error: ${
|
|
10129
|
+
const errorMessage3 = error2 instanceof Error ? error2.message : String(error2);
|
|
10130
|
+
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage3}` });
|
|
7797
10131
|
if (state.interactive) displayStatus(state);
|
|
7798
10132
|
if (driver.hasInFlightWatchers()) {
|
|
7799
10133
|
consecutiveDrainFailures = 0;
|
|
@@ -7810,7 +10144,7 @@ async function driveChannels(state, driver) {
|
|
|
7810
10144
|
}
|
|
7811
10145
|
}
|
|
7812
10146
|
}
|
|
7813
|
-
await new Promise((
|
|
10147
|
+
await new Promise((resolve4) => setTimeout(resolve4, CHANNEL_POLL_INTERVAL_MS));
|
|
7814
10148
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
7815
10149
|
if (idleThisCycle) idleMs += cycleMs;
|
|
7816
10150
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -7831,9 +10165,54 @@ async function driveChannels(state, driver) {
|
|
|
7831
10165
|
}
|
|
7832
10166
|
}
|
|
7833
10167
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7834
|
-
var SESSION_DB_RECLAIM_MAX_PAGES =
|
|
10168
|
+
var SESSION_DB_RECLAIM_MAX_PAGES = 2e4;
|
|
10169
|
+
function shouldWarnForReclaimSkip(reason) {
|
|
10170
|
+
if (reason !== "sqlite-unavailable") return false;
|
|
10171
|
+
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
10172
|
+
if (!version2) return false;
|
|
10173
|
+
const major = Number(version2[1]);
|
|
10174
|
+
const minor = Number(version2[2]);
|
|
10175
|
+
const patch = Number(version2[3]);
|
|
10176
|
+
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
10177
|
+
}
|
|
7835
10178
|
function sessionDbPath() {
|
|
7836
|
-
return
|
|
10179
|
+
return join9(homedir5(), ".local", "share", "opencode", "opencode.db");
|
|
10180
|
+
}
|
|
10181
|
+
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
10182
|
+
const record = {
|
|
10183
|
+
v: 1,
|
|
10184
|
+
event: "session_db_recovery",
|
|
10185
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10186
|
+
stage: "verify",
|
|
10187
|
+
outcome: "schema_provenance_mismatch",
|
|
10188
|
+
severity: "error",
|
|
10189
|
+
reason: provenance.reason ?? "schema-provenance-mismatch",
|
|
10190
|
+
litestream_exit_code: null,
|
|
10191
|
+
attempt: null,
|
|
10192
|
+
replica_objects: null,
|
|
10193
|
+
replica_bytes: null,
|
|
10194
|
+
quarantine_destination: null,
|
|
10195
|
+
quarantined_objects: null,
|
|
10196
|
+
quarantine_failed_objects: null,
|
|
10197
|
+
quarantined_bytes: null,
|
|
10198
|
+
verified_restore_point: null,
|
|
10199
|
+
restore_points_tried: null,
|
|
10200
|
+
provenance_reason: provenance.reason,
|
|
10201
|
+
provenance_migration_delta: provenance.migrationDelta,
|
|
10202
|
+
replication_suspended: false,
|
|
10203
|
+
dbPath: sessionDbPath(),
|
|
10204
|
+
recorded_version: provenance.recordedVersion,
|
|
10205
|
+
current_version: currentVersion,
|
|
10206
|
+
provenance_pre_boot_migration_count: preBootMigrationCount
|
|
10207
|
+
};
|
|
10208
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10209
|
+
if (!activity) throw new Error("could not map session-DB provenance activity");
|
|
10210
|
+
logActivity(state, {
|
|
10211
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10212
|
+
level: activity.level,
|
|
10213
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10214
|
+
metadata: activity.metadata
|
|
10215
|
+
});
|
|
7837
10216
|
}
|
|
7838
10217
|
async function runSweep(state, driver, config) {
|
|
7839
10218
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -7880,7 +10259,7 @@ async function runSweep(state, driver, config) {
|
|
|
7880
10259
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
7881
10260
|
dbPath: sessionDbPath(),
|
|
7882
10261
|
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
7883
|
-
allowFullVacuum: protectedNow.size === 0
|
|
10262
|
+
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
7884
10263
|
});
|
|
7885
10264
|
if (reclaimResult.ok) {
|
|
7886
10265
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
@@ -7893,7 +10272,7 @@ async function runSweep(state, driver, config) {
|
|
|
7893
10272
|
} else {
|
|
7894
10273
|
logActivity(state, {
|
|
7895
10274
|
type: "info",
|
|
7896
|
-
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})`
|
|
10275
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : "")
|
|
7897
10276
|
});
|
|
7898
10277
|
}
|
|
7899
10278
|
} catch (error2) {
|
|
@@ -7916,13 +10295,20 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7916
10295
|
for (const warning2 of config.warnings) {
|
|
7917
10296
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
7918
10297
|
}
|
|
7919
|
-
const dbBytes = statSessionDbBytes(
|
|
10298
|
+
const dbBytes = statSessionDbBytes(homedir5());
|
|
7920
10299
|
void (async () => {
|
|
7921
|
-
const
|
|
10300
|
+
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
10301
|
+
if (reclaimAvailability !== null) {
|
|
10302
|
+
logActivity(state, {
|
|
10303
|
+
type: "info",
|
|
10304
|
+
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
10305
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
10306
|
+
});
|
|
10307
|
+
}
|
|
7922
10308
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
7923
10309
|
dbBytes,
|
|
7924
10310
|
cleanupEnabled: config.enabled,
|
|
7925
|
-
reclaimSkipReason
|
|
10311
|
+
reclaimSkipReason: reclaimAvailability?.reason ?? null
|
|
7926
10312
|
});
|
|
7927
10313
|
if (sizeWarning !== null) {
|
|
7928
10314
|
logActivity(state, { type: "info", level: "warn", message: sizeWarning });
|
|
@@ -7944,23 +10330,20 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
7944
10330
|
);
|
|
7945
10331
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
7946
10332
|
}
|
|
7947
|
-
function
|
|
7948
|
-
const { mode, warnings } =
|
|
7949
|
-
options.claudeUsageReporting,
|
|
7950
|
-
process.env
|
|
7951
|
-
);
|
|
10333
|
+
function scheduleUsageReporting(state, params) {
|
|
10334
|
+
const { mode, warnings } = params.resolved;
|
|
7952
10335
|
for (const warning2 of warnings) {
|
|
7953
10336
|
logActivity(state, {
|
|
7954
10337
|
type: "info",
|
|
7955
10338
|
level: "warn",
|
|
7956
|
-
message:
|
|
10339
|
+
message: `${params.label} usage reporting: ${warning2}`
|
|
7957
10340
|
});
|
|
7958
10341
|
}
|
|
7959
10342
|
if (mode === "off") {
|
|
7960
10343
|
logActivity(state, {
|
|
7961
10344
|
type: "info",
|
|
7962
10345
|
level: "debug",
|
|
7963
|
-
message:
|
|
10346
|
+
message: `${params.label} usage reporting is off (${params.offFlagHint})`
|
|
7964
10347
|
});
|
|
7965
10348
|
return null;
|
|
7966
10349
|
}
|
|
@@ -7969,7 +10352,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7969
10352
|
let rearmRequested = false;
|
|
7970
10353
|
const armProbe = () => {
|
|
7971
10354
|
phase = "probe-pending";
|
|
7972
|
-
|
|
10355
|
+
params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
|
|
7973
10356
|
};
|
|
7974
10357
|
const scheduleNextTick = () => {
|
|
7975
10358
|
if (rearmRequested) {
|
|
@@ -7978,7 +10361,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7978
10361
|
return;
|
|
7979
10362
|
}
|
|
7980
10363
|
phase = "steady-pending";
|
|
7981
|
-
|
|
10364
|
+
params.setTimer(setTimeout(() => void tick(false), params.nextDelayMs()));
|
|
7982
10365
|
};
|
|
7983
10366
|
const rearm = () => {
|
|
7984
10367
|
switch (phase) {
|
|
@@ -7988,9 +10371,9 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
7988
10371
|
case "probe-pending":
|
|
7989
10372
|
return;
|
|
7990
10373
|
case "steady-pending":
|
|
7991
|
-
if (
|
|
7992
|
-
clearTimeout(
|
|
7993
|
-
|
|
10374
|
+
if (params.getTimer()) {
|
|
10375
|
+
clearTimeout(params.getTimer());
|
|
10376
|
+
params.setTimer(null);
|
|
7994
10377
|
}
|
|
7995
10378
|
rearmRequested = false;
|
|
7996
10379
|
armProbe();
|
|
@@ -8004,45 +10387,45 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8004
10387
|
const tick = async (isProbe) => {
|
|
8005
10388
|
phase = "tick-in-flight";
|
|
8006
10389
|
try {
|
|
8007
|
-
const usage = await
|
|
8008
|
-
const result = await
|
|
10390
|
+
const usage = await params.fetchUsage();
|
|
10391
|
+
const result = await params.report(usage);
|
|
8009
10392
|
if (result.ok) {
|
|
8010
10393
|
if (consecutiveFailures > 0) {
|
|
8011
10394
|
logActivity(state, {
|
|
8012
10395
|
type: "info",
|
|
8013
10396
|
level: "info",
|
|
8014
|
-
message:
|
|
10397
|
+
message: `${params.label} usage reporting recovered`
|
|
8015
10398
|
});
|
|
8016
10399
|
}
|
|
8017
10400
|
consecutiveFailures = 0;
|
|
8018
10401
|
logActivity(state, {
|
|
8019
10402
|
type: "info",
|
|
8020
10403
|
level: "debug",
|
|
8021
|
-
message:
|
|
10404
|
+
message: `Reported ${params.label} usage to Evident`
|
|
8022
10405
|
});
|
|
8023
10406
|
} else {
|
|
8024
10407
|
consecutiveFailures++;
|
|
8025
10408
|
logActivity(state, {
|
|
8026
10409
|
type: "info",
|
|
8027
|
-
level:
|
|
8028
|
-
message: `Failed to report
|
|
10410
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
10411
|
+
message: `Failed to report ${params.label} usage: ${result.error}${failureStreakSuffix(consecutiveFailures)}`
|
|
8029
10412
|
});
|
|
8030
10413
|
}
|
|
8031
10414
|
scheduleNextTick();
|
|
8032
10415
|
} catch (error2) {
|
|
8033
|
-
if (
|
|
10416
|
+
if (params.isLocalCredentialProblem(error2)) {
|
|
8034
10417
|
if (mode === "on") {
|
|
8035
10418
|
logActivity(state, {
|
|
8036
10419
|
type: "info",
|
|
8037
10420
|
level: "warn",
|
|
8038
|
-
message:
|
|
10421
|
+
message: `${params.label} usage reporting is forced on but no usable login was found \u2014 ${params.forcedOnHint}; reporting will keep retrying`
|
|
8039
10422
|
});
|
|
8040
10423
|
scheduleNextTick();
|
|
8041
10424
|
} else if (isProbe) {
|
|
8042
10425
|
logActivity(state, {
|
|
8043
10426
|
type: "info",
|
|
8044
10427
|
level: "debug",
|
|
8045
|
-
message:
|
|
10428
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
8046
10429
|
});
|
|
8047
10430
|
phase = "dormant";
|
|
8048
10431
|
if (rearmRequested) rearm();
|
|
@@ -8050,7 +10433,7 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8050
10433
|
logActivity(state, {
|
|
8051
10434
|
type: "info",
|
|
8052
10435
|
level: "debug",
|
|
8053
|
-
message:
|
|
10436
|
+
message: `${params.label} usage reporting: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
8054
10437
|
});
|
|
8055
10438
|
scheduleNextTick();
|
|
8056
10439
|
}
|
|
@@ -8059,8 +10442,8 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8059
10442
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8060
10443
|
logActivity(state, {
|
|
8061
10444
|
type: "info",
|
|
8062
|
-
level:
|
|
8063
|
-
message:
|
|
10445
|
+
level: params.failureLogLevel(consecutiveFailures),
|
|
10446
|
+
message: `${params.label} usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
8064
10447
|
});
|
|
8065
10448
|
scheduleNextTick();
|
|
8066
10449
|
}
|
|
@@ -8069,6 +10452,34 @@ function scheduleClaudeUsageReporting(state, options) {
|
|
|
8069
10452
|
armProbe();
|
|
8070
10453
|
return rearm;
|
|
8071
10454
|
}
|
|
10455
|
+
function scheduleClaudeUsageReporting(state, options) {
|
|
10456
|
+
return scheduleUsageReporting(state, {
|
|
10457
|
+
label: "Claude",
|
|
10458
|
+
resolved: resolveClaudeUsageReportingMode(options.claudeUsageReporting, process.env),
|
|
10459
|
+
offFlagHint: "--claude-usage-reporting off",
|
|
10460
|
+
getTimer: () => state.claudeUsageTimer,
|
|
10461
|
+
setTimer: (timer) => {
|
|
10462
|
+
state.claudeUsageTimer = timer;
|
|
10463
|
+
},
|
|
10464
|
+
fetchUsage: async () => {
|
|
10465
|
+
const usage = await getClaudeUsage();
|
|
10466
|
+
if (usage.ownerLookupError) {
|
|
10467
|
+
logActivity(state, {
|
|
10468
|
+
type: "info",
|
|
10469
|
+
level: "debug",
|
|
10470
|
+
message: `Claude usage owner lookup failed: ${usage.ownerLookupError}`
|
|
10471
|
+
});
|
|
10472
|
+
}
|
|
10473
|
+
return usage;
|
|
10474
|
+
},
|
|
10475
|
+
report: (usage) => reportClaudeUsage(state.agentId, state.authHeader, usage),
|
|
10476
|
+
isLocalCredentialProblem,
|
|
10477
|
+
forcedOnHint: "run `claude` to sign in",
|
|
10478
|
+
firstDelayMs: () => FIRST_REPORT_DELAY_MS,
|
|
10479
|
+
nextDelayMs: nextReportDelayMs,
|
|
10480
|
+
failureLogLevel: claudeUsageFailureLogLevel
|
|
10481
|
+
});
|
|
10482
|
+
}
|
|
8072
10483
|
var RESOURCE_USAGE_BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
8073
10484
|
var RESOURCE_USAGE_REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
8074
10485
|
var RESOURCE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
@@ -8092,7 +10503,8 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
8092
10503
|
});
|
|
8093
10504
|
return;
|
|
8094
10505
|
}
|
|
8095
|
-
const collect = createResourceUsageCollector(
|
|
10506
|
+
const { collect, stop } = createResourceUsageCollector(homedir5());
|
|
10507
|
+
state.stopResourceUsageSampling = stop;
|
|
8096
10508
|
let consecutiveFailures = 0;
|
|
8097
10509
|
const tick = async () => {
|
|
8098
10510
|
try {
|
|
@@ -8193,25 +10605,50 @@ async function cleanup(state, opts = {}) {
|
|
|
8193
10605
|
state.claudeUsageTimer = null;
|
|
8194
10606
|
}
|
|
8195
10607
|
state.claudeUsageRearm = null;
|
|
10608
|
+
if (state.openaiUsageTimer) {
|
|
10609
|
+
clearTimeout(state.openaiUsageTimer);
|
|
10610
|
+
state.openaiUsageTimer = null;
|
|
10611
|
+
}
|
|
10612
|
+
state.openaiUsageRearm = null;
|
|
8196
10613
|
if (state.resourceUsageTimer) {
|
|
8197
10614
|
clearTimeout(state.resourceUsageTimer);
|
|
8198
10615
|
state.resourceUsageTimer = null;
|
|
8199
10616
|
}
|
|
10617
|
+
state.stopResourceUsageSampling?.();
|
|
10618
|
+
state.stopResourceUsageSampling = null;
|
|
10619
|
+
const credentialSync = state.credentialSync;
|
|
10620
|
+
const flushCredentials = credentialSync ? async (phase, publish) => {
|
|
10621
|
+
await timeShutdownPhase(state, durations, phase, async () => {
|
|
10622
|
+
const outcomes = await credentialSync.stopAndFlush(publish);
|
|
10623
|
+
const level = Object.values(outcomes).every((outcome) => outcome === "ok") ? "info" : "warn";
|
|
10624
|
+
log2(
|
|
10625
|
+
state,
|
|
10626
|
+
`Credential flush: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
10627
|
+
level
|
|
10628
|
+
);
|
|
10629
|
+
});
|
|
10630
|
+
} : void 0;
|
|
10631
|
+
let drainSettled = true;
|
|
8200
10632
|
if (opts.graceful && state.channelDriver) {
|
|
8201
10633
|
state.channelDriver.stop();
|
|
10634
|
+
}
|
|
10635
|
+
if (flushCredentials) {
|
|
10636
|
+
await flushCredentials("credential_flush", !(opts.graceful && state.channelDriver));
|
|
10637
|
+
}
|
|
10638
|
+
if (opts.graceful && state.channelDriver) {
|
|
8202
10639
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
8203
10640
|
if (state.interactive) {
|
|
8204
10641
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
8205
10642
|
displayStatus(state);
|
|
8206
10643
|
}
|
|
8207
10644
|
const driver = state.channelDriver;
|
|
8208
|
-
|
|
10645
|
+
drainSettled = await timeShutdownPhase(
|
|
8209
10646
|
state,
|
|
8210
10647
|
durations,
|
|
8211
10648
|
"drain",
|
|
8212
10649
|
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
8213
10650
|
);
|
|
8214
|
-
if (!
|
|
10651
|
+
if (!drainSettled) {
|
|
8215
10652
|
logActivity(state, {
|
|
8216
10653
|
type: "info",
|
|
8217
10654
|
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
@@ -8219,6 +10656,9 @@ async function cleanup(state, opts = {}) {
|
|
|
8219
10656
|
if (state.interactive) displayStatus(state);
|
|
8220
10657
|
}
|
|
8221
10658
|
}
|
|
10659
|
+
if (opts.graceful && state.channelDriver && flushCredentials && drainSettled) {
|
|
10660
|
+
await flushCredentials("credential_flush_final", true);
|
|
10661
|
+
}
|
|
8222
10662
|
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
8223
10663
|
if (state.connection) {
|
|
8224
10664
|
const connection = state.connection;
|
|
@@ -8227,24 +10667,83 @@ async function cleanup(state, opts = {}) {
|
|
|
8227
10667
|
}
|
|
8228
10668
|
if (state.opencodeProcess) {
|
|
8229
10669
|
const opencodeProcess = state.opencodeProcess;
|
|
8230
|
-
|
|
10670
|
+
const result = await timeShutdownPhase(
|
|
10671
|
+
state,
|
|
10672
|
+
durations,
|
|
10673
|
+
"opencode_stop",
|
|
10674
|
+
() => stopOpenCodeAndWait(opencodeProcess, CHILD_STOP_TIMEOUT_MS)
|
|
10675
|
+
);
|
|
8231
10676
|
if (state.interactive) {
|
|
8232
|
-
logActivity(state, { type: "info", message:
|
|
10677
|
+
logActivity(state, { type: "info", message: `Stopped OpenCode process (${result.outcome})` });
|
|
8233
10678
|
displayStatus(state);
|
|
8234
10679
|
} else {
|
|
8235
|
-
log2(state,
|
|
10680
|
+
log2(state, `Stopped OpenCode process (${result.outcome})`);
|
|
8236
10681
|
}
|
|
8237
10682
|
state.opencodeProcess = null;
|
|
8238
10683
|
}
|
|
10684
|
+
if (state.litestreamProcess) {
|
|
10685
|
+
const litestreamProcess = state.litestreamProcess;
|
|
10686
|
+
const result = await timeShutdownPhase(
|
|
10687
|
+
state,
|
|
10688
|
+
durations,
|
|
10689
|
+
"litestream_stop",
|
|
10690
|
+
() => stopSessionDbReplication(litestreamProcess, CHILD_STOP_TIMEOUT_MS)
|
|
10691
|
+
);
|
|
10692
|
+
log2(state, `Stopped litestream replication (${result.outcome})`);
|
|
10693
|
+
state.litestreamProcess = null;
|
|
10694
|
+
}
|
|
8239
10695
|
return durations;
|
|
8240
10696
|
}
|
|
10697
|
+
async function beginGracefulShutdown(state, trigger) {
|
|
10698
|
+
if (state.shuttingDown) return;
|
|
10699
|
+
state.shuttingDown = true;
|
|
10700
|
+
const shutdownStartedAt = Date.now();
|
|
10701
|
+
const shutdownMessage = trigger === "recycle" ? "Recycle requested \u2014 finishing in-flight work, then exiting" : "Shutting down...";
|
|
10702
|
+
if (state.interactive) {
|
|
10703
|
+
logActivity(state, { type: "info", message: shutdownMessage });
|
|
10704
|
+
displayStatus(state);
|
|
10705
|
+
} else {
|
|
10706
|
+
log2(state, shutdownMessage);
|
|
10707
|
+
}
|
|
10708
|
+
const durations = await cleanup(state, { graceful: true });
|
|
10709
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
10710
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
10711
|
+
let timer;
|
|
10712
|
+
const flushed = shutdownTelemetry().then(
|
|
10713
|
+
() => true,
|
|
10714
|
+
(error2) => {
|
|
10715
|
+
log2(
|
|
10716
|
+
state,
|
|
10717
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10718
|
+
"warn"
|
|
10719
|
+
);
|
|
10720
|
+
return true;
|
|
10721
|
+
}
|
|
10722
|
+
);
|
|
10723
|
+
const timedOut = new Promise((resolve4) => {
|
|
10724
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
10725
|
+
});
|
|
10726
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
10727
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
10728
|
+
}
|
|
10729
|
+
clearTimeout(timer);
|
|
10730
|
+
});
|
|
10731
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
10732
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
10733
|
+
process.exit(0);
|
|
10734
|
+
}
|
|
8241
10735
|
async function run(options) {
|
|
8242
10736
|
const interactive = isInteractive(options.json);
|
|
8243
10737
|
let logLevel;
|
|
8244
10738
|
let fileSyncDirectories;
|
|
8245
10739
|
try {
|
|
8246
10740
|
logLevel = resolveLogLevel(options);
|
|
8247
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
10741
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir5());
|
|
10742
|
+
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
10743
|
+
throw new Error(
|
|
10744
|
+
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
10745
|
+
);
|
|
10746
|
+
}
|
|
8248
10747
|
} catch (error2) {
|
|
8249
10748
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8250
10749
|
if (options.json) {
|
|
@@ -8268,7 +10767,9 @@ async function run(options) {
|
|
|
8268
10767
|
connected: false,
|
|
8269
10768
|
opencodeConnected: false,
|
|
8270
10769
|
opencodeVersion: null,
|
|
10770
|
+
sessionDbProvenanceAnomaly: false,
|
|
8271
10771
|
opencodeProcess: null,
|
|
10772
|
+
litestreamProcess: null,
|
|
8272
10773
|
connection: null,
|
|
8273
10774
|
channelDriver: null,
|
|
8274
10775
|
running: true,
|
|
@@ -8279,10 +10780,27 @@ async function run(options) {
|
|
|
8279
10780
|
sessionCleanupTimers: [],
|
|
8280
10781
|
claudeUsageTimer: null,
|
|
8281
10782
|
claudeUsageRearm: null,
|
|
10783
|
+
openaiUsageTimer: null,
|
|
10784
|
+
openaiUsageRearm: null,
|
|
8282
10785
|
resourceUsageTimer: null,
|
|
10786
|
+
stopResourceUsageSampling: null,
|
|
10787
|
+
credentialSync: null,
|
|
8283
10788
|
authHeader: ""
|
|
8284
10789
|
};
|
|
8285
10790
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
10791
|
+
if (options.credentialSyncMarker) {
|
|
10792
|
+
state.credentialSync = createCredentialSync({
|
|
10793
|
+
markerPath: options.credentialSyncMarker,
|
|
10794
|
+
env: process.env,
|
|
10795
|
+
log: (message, level = "info") => {
|
|
10796
|
+
if (level === "error") {
|
|
10797
|
+
logActivity(state, { type: "error", error: message });
|
|
10798
|
+
} else {
|
|
10799
|
+
logActivity(state, { type: "info", level, message });
|
|
10800
|
+
}
|
|
10801
|
+
}
|
|
10802
|
+
});
|
|
10803
|
+
}
|
|
8286
10804
|
if (fileSyncDirectories.length > 0) {
|
|
8287
10805
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
8288
10806
|
} else {
|
|
@@ -8308,43 +10826,7 @@ async function run(options) {
|
|
|
8308
10826
|
"warn"
|
|
8309
10827
|
);
|
|
8310
10828
|
}
|
|
8311
|
-
const handleSignal =
|
|
8312
|
-
if (state.shuttingDown) return;
|
|
8313
|
-
state.shuttingDown = true;
|
|
8314
|
-
const shutdownStartedAt = Date.now();
|
|
8315
|
-
if (state.interactive) {
|
|
8316
|
-
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
8317
|
-
displayStatus(state);
|
|
8318
|
-
} else {
|
|
8319
|
-
log2(state, "Shutting down...");
|
|
8320
|
-
}
|
|
8321
|
-
const durations = await cleanup(state, { graceful: true });
|
|
8322
|
-
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
8323
|
-
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
8324
|
-
let timer;
|
|
8325
|
-
const flushed = shutdownTelemetry().then(
|
|
8326
|
-
() => true,
|
|
8327
|
-
(error2) => {
|
|
8328
|
-
log2(
|
|
8329
|
-
state,
|
|
8330
|
-
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8331
|
-
"warn"
|
|
8332
|
-
);
|
|
8333
|
-
return true;
|
|
8334
|
-
}
|
|
8335
|
-
);
|
|
8336
|
-
const timedOut = new Promise((resolve3) => {
|
|
8337
|
-
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
8338
|
-
});
|
|
8339
|
-
if (!await Promise.race([flushed, timedOut])) {
|
|
8340
|
-
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
8341
|
-
}
|
|
8342
|
-
clearTimeout(timer);
|
|
8343
|
-
});
|
|
8344
|
-
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
8345
|
-
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
8346
|
-
process.exit(0);
|
|
8347
|
-
};
|
|
10829
|
+
const handleSignal = () => beginGracefulShutdown(state, "signal");
|
|
8348
10830
|
process.on("SIGINT", handleSignal);
|
|
8349
10831
|
process.on("SIGTERM", handleSignal);
|
|
8350
10832
|
try {
|
|
@@ -8354,15 +10836,15 @@ async function run(options) {
|
|
|
8354
10836
|
printError("Authentication required");
|
|
8355
10837
|
blank();
|
|
8356
10838
|
console.log(
|
|
8357
|
-
|
|
10839
|
+
chalk7.dim("Set EVIDENT_RUNNER_KEY (or EVIDENT_AGENT_KEY) environment variable for CI")
|
|
8358
10840
|
);
|
|
8359
|
-
console.log(
|
|
10841
|
+
console.log(chalk7.dim("Or run `evident login` for interactive authentication"));
|
|
8360
10842
|
blank();
|
|
8361
10843
|
process.exit(1);
|
|
8362
10844
|
return;
|
|
8363
10845
|
}
|
|
8364
10846
|
blank();
|
|
8365
|
-
console.log(
|
|
10847
|
+
console.log(chalk7.yellow("You are not logged in to Evident."));
|
|
8366
10848
|
blank();
|
|
8367
10849
|
credentials2 = await promptForLogin(
|
|
8368
10850
|
"Would you like to log in now?",
|
|
@@ -8412,7 +10894,7 @@ async function run(options) {
|
|
|
8412
10894
|
);
|
|
8413
10895
|
blank();
|
|
8414
10896
|
console.log(
|
|
8415
|
-
|
|
10897
|
+
chalk7.dim(
|
|
8416
10898
|
"Either provide --runner/--agent <id> or set EVIDENT_RUNNER_KEY/EVIDENT_AGENT_KEY"
|
|
8417
10899
|
)
|
|
8418
10900
|
);
|
|
@@ -8435,15 +10917,15 @@ async function run(options) {
|
|
|
8435
10917
|
);
|
|
8436
10918
|
if (interactive && !state.json) {
|
|
8437
10919
|
blank();
|
|
8438
|
-
console.log(
|
|
8439
|
-
console.log(
|
|
10920
|
+
console.log(chalk7.bold("Evident Run"));
|
|
10921
|
+
console.log(chalk7.dim("-".repeat(40)));
|
|
8440
10922
|
}
|
|
8441
10923
|
const spinner = interactive && !state.json ? ora3("Validating runner...").start() : null;
|
|
8442
10924
|
let validation = await getAgentInfo(state.agentId, state.authHeader);
|
|
8443
10925
|
if (!validation.valid && validation.authFailed && interactive) {
|
|
8444
10926
|
spinner?.fail("Authentication failed");
|
|
8445
10927
|
blank();
|
|
8446
|
-
console.log(
|
|
10928
|
+
console.log(chalk7.yellow("Your authentication token is invalid or expired."));
|
|
8447
10929
|
blank();
|
|
8448
10930
|
credentials2 = await promptForLogin(
|
|
8449
10931
|
"Would you like to log in again?",
|
|
@@ -8474,26 +10956,133 @@ async function run(options) {
|
|
|
8474
10956
|
} else {
|
|
8475
10957
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
8476
10958
|
}
|
|
10959
|
+
if (options.restoreRunnerCredentials) {
|
|
10960
|
+
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
10961
|
+
const credentialContext = {
|
|
10962
|
+
env: process.env,
|
|
10963
|
+
log: (message, level = "info") => {
|
|
10964
|
+
if (level === "error") {
|
|
10965
|
+
logActivity(state, { type: "error", error: message });
|
|
10966
|
+
} else {
|
|
10967
|
+
logActivity(state, { type: "info", level, message });
|
|
10968
|
+
}
|
|
10969
|
+
}
|
|
10970
|
+
};
|
|
10971
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
10972
|
+
await restoreCredentialStores(credentialContext);
|
|
10973
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
10974
|
+
}
|
|
10975
|
+
state.credentialSync?.arm();
|
|
10976
|
+
let sessionDbVerifyFatal = false;
|
|
10977
|
+
if (!options.restoreSessionDb) {
|
|
10978
|
+
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
10979
|
+
} else {
|
|
10980
|
+
const health = await checkOpenCodeHealth(state.port);
|
|
10981
|
+
if (health.healthy) {
|
|
10982
|
+
log2(
|
|
10983
|
+
state,
|
|
10984
|
+
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
10985
|
+
"debug"
|
|
10986
|
+
);
|
|
10987
|
+
} else {
|
|
10988
|
+
const result = await restoreAndVerifySessionDb({
|
|
10989
|
+
dbPath: sessionDbPath(),
|
|
10990
|
+
litestreamConfig: options.litestreamConfig,
|
|
10991
|
+
noReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
10992
|
+
env: process.env,
|
|
10993
|
+
log: (message, level = "info") => {
|
|
10994
|
+
if (level === "error") {
|
|
10995
|
+
logActivity(state, { type: "error", error: message });
|
|
10996
|
+
} else {
|
|
10997
|
+
logActivity(state, { type: "info", level, message });
|
|
10998
|
+
}
|
|
10999
|
+
},
|
|
11000
|
+
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
11001
|
+
});
|
|
11002
|
+
sessionDbVerifyFatal = result.verifyFatal;
|
|
11003
|
+
}
|
|
11004
|
+
}
|
|
11005
|
+
reportSessionDbRecovery(state);
|
|
11006
|
+
if (sessionDbVerifyFatal) {
|
|
11007
|
+
throw new Error(
|
|
11008
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
11009
|
+
);
|
|
11010
|
+
}
|
|
11011
|
+
applyRunnerOpenCodeConfig({
|
|
11012
|
+
overlayPath: options.opencodeConfigOverlay,
|
|
11013
|
+
log: (message, level = "info") => {
|
|
11014
|
+
if (level === "error") {
|
|
11015
|
+
logActivity(state, { type: "error", error: message });
|
|
11016
|
+
} else {
|
|
11017
|
+
logActivity(state, { type: "info", level, message });
|
|
11018
|
+
}
|
|
11019
|
+
}
|
|
11020
|
+
});
|
|
8477
11021
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
8478
11022
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
8479
11023
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
8480
11024
|
}
|
|
11025
|
+
const { version: opencodeVersion, warnings: opencodeVersionWarnings } = resolveOpenCodeVersion(
|
|
11026
|
+
options,
|
|
11027
|
+
process.env
|
|
11028
|
+
);
|
|
11029
|
+
for (const warning2 of opencodeVersionWarnings) {
|
|
11030
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
11031
|
+
}
|
|
8481
11032
|
const { value: maxActiveSessions, warnings: maxActiveSessionsWarnings } = resolveMaxActiveSessions(options, process.env);
|
|
8482
11033
|
for (const warning2 of maxActiveSessionsWarnings) {
|
|
8483
11034
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
8484
11035
|
}
|
|
11036
|
+
const preBootMigrationIds = readSessionDbMigrationIds(sessionDbPath());
|
|
8485
11037
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
8486
11038
|
try {
|
|
8487
|
-
const oc = await
|
|
11039
|
+
const oc = opencodeVersion === "v2" ? await ensureOpenCode2Running({
|
|
11040
|
+
port: state.port,
|
|
11041
|
+
interactive: state.interactive,
|
|
11042
|
+
agentId: state.agentId,
|
|
11043
|
+
log: (message) => log2(state, message),
|
|
11044
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
11045
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
11046
|
+
}) : await ensureOpenCodeRunning({
|
|
8488
11047
|
port: state.port,
|
|
8489
11048
|
interactive: state.interactive,
|
|
8490
11049
|
agentId: state.agentId,
|
|
8491
11050
|
log: (message) => log2(state, message),
|
|
8492
|
-
startTimeoutMs: opencodeStartTimeoutMs
|
|
11051
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
11052
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
8493
11053
|
});
|
|
8494
11054
|
state.port = oc.port;
|
|
8495
|
-
state.opencodeProcess = oc.process;
|
|
11055
|
+
state.opencodeProcess = options.opencodePidFile ? null : oc.process;
|
|
8496
11056
|
state.opencodeVersion = oc.version;
|
|
11057
|
+
if (options.opencodePidFile && oc.process?.pid !== void 0) {
|
|
11058
|
+
try {
|
|
11059
|
+
writeFileSync6(options.opencodePidFile, `${oc.process.pid}
|
|
11060
|
+
`, { mode: 384 });
|
|
11061
|
+
chmodSync3(options.opencodePidFile, 384);
|
|
11062
|
+
} catch (error2) {
|
|
11063
|
+
logActivity(state, {
|
|
11064
|
+
type: "error",
|
|
11065
|
+
error: `Failed to write OpenCode pid file ${options.opencodePidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11066
|
+
});
|
|
11067
|
+
}
|
|
11068
|
+
}
|
|
11069
|
+
if (state.opencodeVersion !== null) {
|
|
11070
|
+
const provenance = checkSessionDbProvenance({
|
|
11071
|
+
dbPath: sessionDbPath(),
|
|
11072
|
+
currentVersion: state.opencodeVersion,
|
|
11073
|
+
homeDir: homedir5(),
|
|
11074
|
+
env: process.env
|
|
11075
|
+
});
|
|
11076
|
+
if (provenance.anomaly) {
|
|
11077
|
+
state.sessionDbProvenanceAnomaly = true;
|
|
11078
|
+
logSessionDbProvenanceMismatch(
|
|
11079
|
+
state,
|
|
11080
|
+
provenance,
|
|
11081
|
+
state.opencodeVersion,
|
|
11082
|
+
preBootMigrationIds?.length ?? null
|
|
11083
|
+
);
|
|
11084
|
+
}
|
|
11085
|
+
}
|
|
8497
11086
|
state.opencodeConnected = oc.notReadyReason === null;
|
|
8498
11087
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
8499
11088
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
@@ -8516,10 +11105,10 @@ async function run(options) {
|
|
|
8516
11105
|
if (state.interactive && !state.json) {
|
|
8517
11106
|
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
8518
11107
|
blank();
|
|
8519
|
-
console.log(
|
|
11108
|
+
console.log(chalk7.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
8520
11109
|
console.log(
|
|
8521
|
-
|
|
8522
|
-
`Run ${
|
|
11110
|
+
chalk7.dim(
|
|
11111
|
+
`Run ${chalk7.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
8523
11112
|
)
|
|
8524
11113
|
);
|
|
8525
11114
|
blank();
|
|
@@ -8530,6 +11119,108 @@ async function run(options) {
|
|
|
8530
11119
|
ocSpinner?.fail(error2.message);
|
|
8531
11120
|
throw error2;
|
|
8532
11121
|
}
|
|
11122
|
+
if (options.litestreamPidFile) {
|
|
11123
|
+
if (options.sessionDbNoReplicateMarker && existsSync3(options.sessionDbNoReplicateMarker)) {
|
|
11124
|
+
log2(
|
|
11125
|
+
state,
|
|
11126
|
+
`Skipping Litestream replication because ${options.sessionDbNoReplicateMarker} marks this session database unsafe to replicate`
|
|
11127
|
+
);
|
|
11128
|
+
} else if (!options.litestreamConfig) {
|
|
11129
|
+
logActivity(state, {
|
|
11130
|
+
type: "info",
|
|
11131
|
+
level: "warn",
|
|
11132
|
+
message: "Skipping Litestream replication because no configuration file was provided"
|
|
11133
|
+
});
|
|
11134
|
+
} else {
|
|
11135
|
+
let existingPid;
|
|
11136
|
+
if (existsSync3(options.litestreamPidFile)) {
|
|
11137
|
+
try {
|
|
11138
|
+
const rawPid = readFileSync6(options.litestreamPidFile, "utf8").trim();
|
|
11139
|
+
const parsedPid = Number(rawPid);
|
|
11140
|
+
if (/^\d+$/.test(rawPid) && Number.isSafeInteger(parsedPid) && parsedPid > 0) {
|
|
11141
|
+
existingPid = parsedPid;
|
|
11142
|
+
}
|
|
11143
|
+
} catch (error2) {
|
|
11144
|
+
logActivity(state, {
|
|
11145
|
+
type: "info",
|
|
11146
|
+
level: "warn",
|
|
11147
|
+
message: `Could not read Litestream pid file ${options.litestreamPidFile}; checking for a new process: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11148
|
+
});
|
|
11149
|
+
}
|
|
11150
|
+
}
|
|
11151
|
+
if (existingPid !== void 0 && isProcessAlive(existingPid)) {
|
|
11152
|
+
log2(state, `Litestream replication is already running with pid ${existingPid}`);
|
|
11153
|
+
} else {
|
|
11154
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
11155
|
+
state.litestreamProcess = null;
|
|
11156
|
+
let failureHandled = false;
|
|
11157
|
+
const reportImageOwnedReplicationFailure = (message) => {
|
|
11158
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
11159
|
+
failureHandled = true;
|
|
11160
|
+
logActivity(state, { type: "error", error: message });
|
|
11161
|
+
if (state.interactive) displayStatus(state);
|
|
11162
|
+
};
|
|
11163
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
11164
|
+
reportImageOwnedReplicationFailure(
|
|
11165
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
11166
|
+
);
|
|
11167
|
+
});
|
|
11168
|
+
litestreamProcess.on("error", (error2) => {
|
|
11169
|
+
reportImageOwnedReplicationFailure(
|
|
11170
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11171
|
+
);
|
|
11172
|
+
});
|
|
11173
|
+
try {
|
|
11174
|
+
if (litestreamProcess.pid !== void 0) {
|
|
11175
|
+
writeFileSync6(options.litestreamPidFile, `${litestreamProcess.pid}
|
|
11176
|
+
`, {
|
|
11177
|
+
mode: 384
|
|
11178
|
+
});
|
|
11179
|
+
chmodSync3(options.litestreamPidFile, 384);
|
|
11180
|
+
}
|
|
11181
|
+
} catch (error2) {
|
|
11182
|
+
logActivity(state, {
|
|
11183
|
+
type: "error",
|
|
11184
|
+
error: `Failed to write Litestream pid file ${options.litestreamPidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11185
|
+
});
|
|
11186
|
+
}
|
|
11187
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
11188
|
+
}
|
|
11189
|
+
}
|
|
11190
|
+
} else if (options.litestreamConfig) {
|
|
11191
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
11192
|
+
state.litestreamProcess = litestreamProcess;
|
|
11193
|
+
let failureHandled = false;
|
|
11194
|
+
const failRunForReplication = (message) => {
|
|
11195
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
11196
|
+
failureHandled = true;
|
|
11197
|
+
state.shuttingDown = true;
|
|
11198
|
+
logActivity(state, { type: "error", error: message });
|
|
11199
|
+
if (state.interactive) displayStatus(state);
|
|
11200
|
+
void (async () => {
|
|
11201
|
+
try {
|
|
11202
|
+
await cleanup(state);
|
|
11203
|
+
await shutdownTelemetry();
|
|
11204
|
+
} catch (error2) {
|
|
11205
|
+
console.error(
|
|
11206
|
+
`[run] replication failure cleanup failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11207
|
+
);
|
|
11208
|
+
}
|
|
11209
|
+
process.exit(1);
|
|
11210
|
+
})();
|
|
11211
|
+
};
|
|
11212
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
11213
|
+
failRunForReplication(
|
|
11214
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
11215
|
+
);
|
|
11216
|
+
});
|
|
11217
|
+
litestreamProcess.on("error", (error2) => {
|
|
11218
|
+
failRunForReplication(
|
|
11219
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11220
|
+
);
|
|
11221
|
+
});
|
|
11222
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
11223
|
+
}
|
|
8533
11224
|
const tunnelSpinner = interactive && !state.json ? ora3("Connecting tunnel...").start() : null;
|
|
8534
11225
|
const channelDriver = new ChannelDriver({
|
|
8535
11226
|
agentId: state.agentId,
|
|
@@ -8541,7 +11232,7 @@ async function run(options) {
|
|
|
8541
11232
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
8542
11233
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
8543
11234
|
fileSyncDirectories,
|
|
8544
|
-
homeDir:
|
|
11235
|
+
homeDir: homedir5(),
|
|
8545
11236
|
maxActiveSessions,
|
|
8546
11237
|
log: (entry) => (
|
|
8547
11238
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -8667,6 +11358,18 @@ async function run(options) {
|
|
|
8667
11358
|
if (state.interactive) displayStatus(state);
|
|
8668
11359
|
});
|
|
8669
11360
|
},
|
|
11361
|
+
// Both loops are rearmed because `rearm()` is idempotent for the
|
|
11362
|
+
// provider that did not just connect, and is a no-op when reporting is off.
|
|
11363
|
+
onUsageRearmPing: () => {
|
|
11364
|
+
if (!state.running) return;
|
|
11365
|
+
logActivity(state, {
|
|
11366
|
+
type: "info",
|
|
11367
|
+
level: "debug",
|
|
11368
|
+
message: "Usage rearm ping received"
|
|
11369
|
+
});
|
|
11370
|
+
state.claudeUsageRearm?.();
|
|
11371
|
+
state.openaiUsageRearm?.();
|
|
11372
|
+
},
|
|
8670
11373
|
onInfo: (message) => logActivity(state, { type: "info", message })
|
|
8671
11374
|
}
|
|
8672
11375
|
});
|
|
@@ -8679,6 +11382,32 @@ async function run(options) {
|
|
|
8679
11382
|
}
|
|
8680
11383
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
8681
11384
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
11385
|
+
state.openaiUsageRearm = scheduleUsageReporting(state, {
|
|
11386
|
+
label: "OpenAI",
|
|
11387
|
+
resolved: resolveOpenAiUsageReportingMode(options.openaiUsageReporting, process.env),
|
|
11388
|
+
offFlagHint: "--openai-usage-reporting off",
|
|
11389
|
+
getTimer: () => state.openaiUsageTimer,
|
|
11390
|
+
setTimer: (timer) => {
|
|
11391
|
+
state.openaiUsageTimer = timer;
|
|
11392
|
+
},
|
|
11393
|
+
fetchUsage: async () => {
|
|
11394
|
+
const usage = await getOpenAiUsage(state.port);
|
|
11395
|
+
if (usage.subscription === null) {
|
|
11396
|
+
logActivity(state, {
|
|
11397
|
+
type: "info",
|
|
11398
|
+
level: "debug",
|
|
11399
|
+
message: "OpenAI usage subscription could not be identified from the local credential"
|
|
11400
|
+
});
|
|
11401
|
+
}
|
|
11402
|
+
return usage;
|
|
11403
|
+
},
|
|
11404
|
+
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
11405
|
+
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
11406
|
+
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
11407
|
+
firstDelayMs: firstReportDelayMs,
|
|
11408
|
+
nextDelayMs: usageReportDelayMs,
|
|
11409
|
+
failureLogLevel: usageReportFailureLogLevel
|
|
11410
|
+
});
|
|
8682
11411
|
scheduleResourceUsageReporting(state, options);
|
|
8683
11412
|
if (!interactive || state.json) {
|
|
8684
11413
|
log2(state, "Driving channel messages...");
|
|
@@ -8717,7 +11446,7 @@ async function run(options) {
|
|
|
8717
11446
|
}
|
|
8718
11447
|
|
|
8719
11448
|
// src/index.ts
|
|
8720
|
-
var { version } =
|
|
11449
|
+
var { version } = createRequire2(import.meta.url)("../package.json");
|
|
8721
11450
|
var program = new Command();
|
|
8722
11451
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
8723
11452
|
"--endpoint <url>",
|
|
@@ -8745,6 +11474,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8745
11474
|
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
8746
11475
|
"--opencode-start-timeout <seconds>",
|
|
8747
11476
|
"Seconds to wait for OpenCode to become healthy when the runner starts it (default: 180). Env: EVIDENT_OPENCODE_START_TIMEOUT"
|
|
11477
|
+
).option(
|
|
11478
|
+
"--opencode-version <v1|v2>",
|
|
11479
|
+
"Which OpenCode major version to launch: v1 or v2 (default: v1). Env: EVIDENT_OPENCODE_VERSION"
|
|
8748
11480
|
).option("--json", "Output in JSON format").option(
|
|
8749
11481
|
"--session-cleanup-max-age <duration>",
|
|
8750
11482
|
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
@@ -8760,6 +11492,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8760
11492
|
).option(
|
|
8761
11493
|
"--claude-usage-reporting <mode>",
|
|
8762
11494
|
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
11495
|
+
).option(
|
|
11496
|
+
"--openai-usage-reporting <mode>",
|
|
11497
|
+
"Report OpenAI plan usage to Evident: auto | on | off (default: auto). Env: EVIDENT_OPENAI_USAGE_REPORTING"
|
|
8763
11498
|
).option(
|
|
8764
11499
|
"--no-resource-usage-reporting",
|
|
8765
11500
|
"Don't report this machine's CPU and memory usage to Evident (reporting is on by default). Env: EVIDENT_RESOURCE_USAGE_REPORTING=off"
|
|
@@ -8771,6 +11506,30 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8771
11506
|
).option(
|
|
8772
11507
|
"--tunnel-ready-file <path>",
|
|
8773
11508
|
"Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
|
|
11509
|
+
).option(
|
|
11510
|
+
"--litestream-config <path>",
|
|
11511
|
+
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
11512
|
+
).option(
|
|
11513
|
+
"--opencode-pid-file <path>",
|
|
11514
|
+
"Record the OpenCode pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11515
|
+
).option(
|
|
11516
|
+
"--litestream-pid-file <path>",
|
|
11517
|
+
"Record the Litestream pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11518
|
+
).option(
|
|
11519
|
+
"--session-db-no-replicate-marker <path>",
|
|
11520
|
+
"Read this marker as a gate before starting Litestream; the session-DB restore writes it when this boot's database is not safe to replicate."
|
|
11521
|
+
).option(
|
|
11522
|
+
"--restore-session-db",
|
|
11523
|
+
"Restore and verify the OpenCode session database before starting OpenCode; requires --litestream-config for the replica and --session-db-no-replicate-marker for give-up records."
|
|
11524
|
+
).option(
|
|
11525
|
+
"--restore-runner-credentials",
|
|
11526
|
+
"Restore the hosted runner secret and persisted credential stores before starting OpenCode."
|
|
11527
|
+
).option(
|
|
11528
|
+
"--opencode-config-overlay <path>",
|
|
11529
|
+
"Apply this runner-provided OpenCode config before starting OpenCode."
|
|
11530
|
+
).option(
|
|
11531
|
+
"--credential-sync-marker <path>",
|
|
11532
|
+
"Own the interval credential sync and write this marker once the shutdown flush has finished, so the runner image's lifecycle hooks can wait on it."
|
|
8774
11533
|
).action(
|
|
8775
11534
|
(options) => {
|
|
8776
11535
|
run({
|
|
@@ -8786,6 +11545,7 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8786
11545
|
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
8787
11546
|
// resolveOpenCodeStartTimeoutMs (flag > EVIDENT_OPENCODE_START_TIMEOUT > 180s default).
|
|
8788
11547
|
opencodeStartTimeout: options.opencodeStartTimeout,
|
|
11548
|
+
opencodeVersion: options.opencodeVersion,
|
|
8789
11549
|
json: options.json,
|
|
8790
11550
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
8791
11551
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
@@ -8795,13 +11555,22 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
8795
11555
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
8796
11556
|
// (resolveClaudeUsageReportingMode).
|
|
8797
11557
|
claudeUsageReporting: options.claudeUsageReporting,
|
|
11558
|
+
openaiUsageReporting: options.openaiUsageReporting,
|
|
8798
11559
|
// Raw value — resolution is single-sourced in run.ts's
|
|
8799
11560
|
// resolveResourceUsageReportingEnabled.
|
|
8800
11561
|
resourceUsageReporting: options.resourceUsageReporting,
|
|
8801
11562
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
8802
11563
|
// resolveFileSyncDirectories.
|
|
8803
11564
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
8804
|
-
tunnelReadyFile: options.tunnelReadyFile
|
|
11565
|
+
tunnelReadyFile: options.tunnelReadyFile,
|
|
11566
|
+
litestreamConfig: options.litestreamConfig,
|
|
11567
|
+
opencodePidFile: options.opencodePidFile,
|
|
11568
|
+
litestreamPidFile: options.litestreamPidFile,
|
|
11569
|
+
sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
11570
|
+
restoreSessionDb: options.restoreSessionDb,
|
|
11571
|
+
restoreRunnerCredentials: options.restoreRunnerCredentials,
|
|
11572
|
+
opencodeConfigOverlay: options.opencodeConfigOverlay,
|
|
11573
|
+
credentialSyncMarker: options.credentialSyncMarker
|
|
8805
11574
|
});
|
|
8806
11575
|
}
|
|
8807
11576
|
);
|