@evident-ai/cli 3.4.1-dev.2f1b44b → 3.4.1-dev.325472f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/index.js +2010 -174
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire } from "module";
|
|
4
|
+
import { createRequire as createRequire2 } from "module";
|
|
5
5
|
import { Command } from "commander";
|
|
6
6
|
|
|
7
7
|
// src/commands/login.ts
|
|
@@ -371,14 +371,14 @@ function blank() {
|
|
|
371
371
|
console.log();
|
|
372
372
|
}
|
|
373
373
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
374
|
-
return new Promise((
|
|
374
|
+
return new Promise((resolve4) => {
|
|
375
375
|
process.stdout.write(chalk.dim(prompt));
|
|
376
376
|
const handler = () => {
|
|
377
377
|
process.stdin.removeListener("data", handler);
|
|
378
378
|
process.stdin.setRawMode?.(false);
|
|
379
379
|
process.stdin.pause();
|
|
380
380
|
console.log();
|
|
381
|
-
|
|
381
|
+
resolve4();
|
|
382
382
|
};
|
|
383
383
|
if (process.stdin.isTTY) {
|
|
384
384
|
process.stdin.setRawMode?.(true);
|
|
@@ -388,7 +388,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
388
388
|
});
|
|
389
389
|
}
|
|
390
390
|
function sleep(ms) {
|
|
391
|
-
return new Promise((
|
|
391
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
392
392
|
}
|
|
393
393
|
|
|
394
394
|
// src/commands/login.ts
|
|
@@ -466,19 +466,19 @@ async function tokenLogin() {
|
|
|
466
466
|
);
|
|
467
467
|
blank();
|
|
468
468
|
process.stdout.write("Paste token: ");
|
|
469
|
-
const token = await new Promise((
|
|
469
|
+
const token = await new Promise((resolve4) => {
|
|
470
470
|
let data = "";
|
|
471
471
|
process.stdin.setEncoding("utf8");
|
|
472
472
|
process.stdin.on("data", (chunk) => {
|
|
473
473
|
data += chunk;
|
|
474
474
|
});
|
|
475
475
|
process.stdin.on("end", () => {
|
|
476
|
-
|
|
476
|
+
resolve4(data.trim());
|
|
477
477
|
});
|
|
478
478
|
if (process.stdin.isTTY) {
|
|
479
479
|
process.stdin.once("data", (chunk) => {
|
|
480
480
|
process.stdin.pause();
|
|
481
|
-
|
|
481
|
+
resolve4(chunk.toString().trim());
|
|
482
482
|
});
|
|
483
483
|
process.stdin.resume();
|
|
484
484
|
}
|
|
@@ -763,6 +763,13 @@ function toReportedOpenAiWindow(window) {
|
|
|
763
763
|
resets_at: window.resetsAt
|
|
764
764
|
};
|
|
765
765
|
}
|
|
766
|
+
function toReportedOpenAiSubscription(snapshot) {
|
|
767
|
+
if (!snapshot.subscription) return null;
|
|
768
|
+
return {
|
|
769
|
+
owner_email: snapshot.subscription.ownerEmail,
|
|
770
|
+
plan_type: snapshot.subscription.planType
|
|
771
|
+
};
|
|
772
|
+
}
|
|
766
773
|
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
767
774
|
try {
|
|
768
775
|
const apiUrl = getApiUrlConfig();
|
|
@@ -773,7 +780,8 @@ async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
|
773
780
|
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
774
781
|
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
775
782
|
has_credits: snapshot.hasCredits,
|
|
776
|
-
credits_unlimited: snapshot.creditsUnlimited
|
|
783
|
+
credits_unlimited: snapshot.creditsUnlimited,
|
|
784
|
+
subscription: toReportedOpenAiSubscription(snapshot)
|
|
777
785
|
}),
|
|
778
786
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
779
787
|
});
|
|
@@ -797,6 +805,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
797
805
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
798
806
|
body: JSON.stringify({
|
|
799
807
|
cpu_percent: usage.cpuPercent,
|
|
808
|
+
cpu_peak_percent: usage.cpuPeakPercent,
|
|
800
809
|
cpu_count: usage.cpuCount,
|
|
801
810
|
memory_total_bytes: usage.memoryTotalBytes,
|
|
802
811
|
memory_available_bytes: usage.memoryAvailableBytes,
|
|
@@ -1183,8 +1192,9 @@ async function claudeUsage() {
|
|
|
1183
1192
|
}
|
|
1184
1193
|
|
|
1185
1194
|
// src/commands/run.ts
|
|
1186
|
-
import {
|
|
1187
|
-
import {
|
|
1195
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "fs";
|
|
1196
|
+
import { homedir as homedir5 } from "os";
|
|
1197
|
+
import { isAbsolute as isAbsolute3, join as join9, parse, resolve as resolvePath2 } from "path";
|
|
1188
1198
|
import chalk6 from "chalk";
|
|
1189
1199
|
|
|
1190
1200
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1206,6 +1216,7 @@ var TelemetryEventTypes = {
|
|
|
1206
1216
|
// ../../packages/types/src/tunnel/index.ts
|
|
1207
1217
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1208
1218
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1219
|
+
var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
|
|
1209
1220
|
|
|
1210
1221
|
// ../../packages/types/src/runner-files.ts
|
|
1211
1222
|
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
@@ -1524,7 +1535,14 @@ function drainSessionDbRecoveryReport({
|
|
|
1524
1535
|
skippedLines++;
|
|
1525
1536
|
return [];
|
|
1526
1537
|
}
|
|
1527
|
-
return [
|
|
1538
|
+
return [
|
|
1539
|
+
{
|
|
1540
|
+
...value,
|
|
1541
|
+
provenance_reason: value.provenance_reason ?? null,
|
|
1542
|
+
provenance_migration_delta: value.provenance_migration_delta ?? null,
|
|
1543
|
+
replication_suspended: value.replication_suspended ?? false
|
|
1544
|
+
}
|
|
1545
|
+
];
|
|
1528
1546
|
} catch (error2) {
|
|
1529
1547
|
skippedLines++;
|
|
1530
1548
|
console.error(
|
|
@@ -1627,6 +1645,12 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1627
1645
|
metadata: withoutContractFields(record),
|
|
1628
1646
|
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1629
1647
|
};
|
|
1648
|
+
case "schema_provenance_mismatch":
|
|
1649
|
+
return {
|
|
1650
|
+
level,
|
|
1651
|
+
metadata: withoutContractFields(record),
|
|
1652
|
+
message: `Session database schema provenance mismatch for ${record.dbPath ?? record.db_path ?? "unknown"}: recorded version=${record.recorded_version ?? "unknown"}, current version=${record.current_version ?? "unknown"}, reason=${record.provenance_reason ?? "unknown"}, migration delta=${record.provenance_migration_delta ?? "unknown"}. Inspect the session database and runner backup before continuing.`
|
|
1653
|
+
};
|
|
1630
1654
|
default:
|
|
1631
1655
|
return null;
|
|
1632
1656
|
}
|
|
@@ -1641,7 +1665,8 @@ var OUTCOMES = /* @__PURE__ */ new Set([
|
|
|
1641
1665
|
"fresh_session_db",
|
|
1642
1666
|
"history_rolled_back",
|
|
1643
1667
|
"restore_misconfigured",
|
|
1644
|
-
"session_db_boot_refused"
|
|
1668
|
+
"session_db_boot_refused",
|
|
1669
|
+
"schema_provenance_mismatch"
|
|
1645
1670
|
]);
|
|
1646
1671
|
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1647
1672
|
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
@@ -1659,7 +1684,7 @@ var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"]
|
|
|
1659
1684
|
function isSessionDbRecoveryRecord(value) {
|
|
1660
1685
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1661
1686
|
const record = value;
|
|
1662
|
-
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1687
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && (record.provenance_reason === void 0 || record.provenance_reason === null || typeof record.provenance_reason === "string") && (record.provenance_migration_delta === void 0 || record.provenance_migration_delta === null || Number.isInteger(record.provenance_migration_delta)) && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1663
1688
|
(field) => record[field] === null || typeof record[field] === "string"
|
|
1664
1689
|
);
|
|
1665
1690
|
}
|
|
@@ -1688,11 +1713,667 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1688
1713
|
if (health.healthy) {
|
|
1689
1714
|
return health;
|
|
1690
1715
|
}
|
|
1691
|
-
await new Promise((
|
|
1716
|
+
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
1692
1717
|
}
|
|
1693
1718
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1694
1719
|
}
|
|
1695
1720
|
|
|
1721
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1722
|
+
import { spawn as spawn2 } from "child_process";
|
|
1723
|
+
import { mkdirSync, statSync as statSync2, unlinkSync as unlinkSync2, writeFileSync } from "fs";
|
|
1724
|
+
import { homedir as homedir2 } from "os";
|
|
1725
|
+
import { dirname as dirname2, resolve as resolvePath } from "path";
|
|
1726
|
+
|
|
1727
|
+
// src/lib/runner-synchroniser.ts
|
|
1728
|
+
import { spawn } from "child_process";
|
|
1729
|
+
function appendError(stderr, error2) {
|
|
1730
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1731
|
+
return stderr === "" ? message : `${stderr}
|
|
1732
|
+
${message}`;
|
|
1733
|
+
}
|
|
1734
|
+
function runSynchroniser(args, opts) {
|
|
1735
|
+
return new Promise((resolve4) => {
|
|
1736
|
+
let child;
|
|
1737
|
+
let stdout = "";
|
|
1738
|
+
let stderr = "";
|
|
1739
|
+
let settled = false;
|
|
1740
|
+
const timer = {};
|
|
1741
|
+
let abortListener;
|
|
1742
|
+
let spawnListener;
|
|
1743
|
+
const finish = (result) => {
|
|
1744
|
+
if (settled) return;
|
|
1745
|
+
settled = true;
|
|
1746
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
1747
|
+
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
1748
|
+
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
1749
|
+
resolve4(result);
|
|
1750
|
+
};
|
|
1751
|
+
try {
|
|
1752
|
+
child = spawn("runner-synchroniser", args, {
|
|
1753
|
+
env: opts.env ?? process.env,
|
|
1754
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1755
|
+
});
|
|
1756
|
+
} catch (error2) {
|
|
1757
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
child.stdout?.setEncoding("utf8");
|
|
1761
|
+
child.stdout?.on("data", (chunk) => {
|
|
1762
|
+
stdout += chunk;
|
|
1763
|
+
});
|
|
1764
|
+
child.stderr?.setEncoding("utf8");
|
|
1765
|
+
child.stderr?.on("data", (chunk) => {
|
|
1766
|
+
stderr += chunk;
|
|
1767
|
+
});
|
|
1768
|
+
child.once("error", (error2) => {
|
|
1769
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1770
|
+
});
|
|
1771
|
+
child.once("close", (code) => {
|
|
1772
|
+
finish({ code, stdout, stderr, timedOut: false });
|
|
1773
|
+
});
|
|
1774
|
+
if (opts.signal) {
|
|
1775
|
+
const killChild = () => {
|
|
1776
|
+
if (child.pid === void 0) {
|
|
1777
|
+
if (!spawnListener) {
|
|
1778
|
+
spawnListener = killChild;
|
|
1779
|
+
child.once("spawn", spawnListener);
|
|
1780
|
+
}
|
|
1781
|
+
return;
|
|
1782
|
+
}
|
|
1783
|
+
child.kill("SIGKILL");
|
|
1784
|
+
};
|
|
1785
|
+
abortListener = killChild;
|
|
1786
|
+
if (opts.signal.aborted) {
|
|
1787
|
+
abortListener();
|
|
1788
|
+
} else {
|
|
1789
|
+
opts.signal.addEventListener("abort", abortListener, { once: true });
|
|
1790
|
+
if (opts.signal.aborted) abortListener();
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
timer.handle = setTimeout(
|
|
1794
|
+
() => {
|
|
1795
|
+
child.kill("SIGKILL");
|
|
1796
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1797
|
+
},
|
|
1798
|
+
Math.max(0, opts.timeoutMs)
|
|
1799
|
+
);
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1804
|
+
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
1805
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
|
|
1806
|
+
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
1807
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
|
|
1808
|
+
function commandError(result) {
|
|
1809
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
1810
|
+
}
|
|
1811
|
+
function reportRecord(stage, outcome, reason, litestreamExitCode, options) {
|
|
1812
|
+
options.reportRecovery({
|
|
1813
|
+
v: 1,
|
|
1814
|
+
event: "session_db_recovery",
|
|
1815
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1816
|
+
stage,
|
|
1817
|
+
outcome,
|
|
1818
|
+
severity: "error",
|
|
1819
|
+
reason,
|
|
1820
|
+
litestream_exit_code: litestreamExitCode,
|
|
1821
|
+
attempt: null,
|
|
1822
|
+
replica_objects: null,
|
|
1823
|
+
replica_bytes: null,
|
|
1824
|
+
quarantine_destination: null,
|
|
1825
|
+
quarantined_objects: null,
|
|
1826
|
+
quarantine_failed_objects: null,
|
|
1827
|
+
quarantined_bytes: null,
|
|
1828
|
+
verified_restore_point: null,
|
|
1829
|
+
restore_points_tried: null,
|
|
1830
|
+
provenance_reason: null,
|
|
1831
|
+
provenance_migration_delta: null,
|
|
1832
|
+
replication_suspended: stage === "restore"
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
function clearMarker(options) {
|
|
1836
|
+
if (!options.noReplicateMarker) return;
|
|
1837
|
+
try {
|
|
1838
|
+
unlinkSync2(options.noReplicateMarker);
|
|
1839
|
+
} catch (error2) {
|
|
1840
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1841
|
+
options.log(
|
|
1842
|
+
`Could not clear the previous session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1843
|
+
"warn"
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
function markNoReplicate(options, message) {
|
|
1848
|
+
if (options.noReplicateMarker) {
|
|
1849
|
+
try {
|
|
1850
|
+
mkdirSync(dirname2(options.noReplicateMarker), { recursive: true });
|
|
1851
|
+
writeFileSync(options.noReplicateMarker, "");
|
|
1852
|
+
} catch (error2) {
|
|
1853
|
+
options.log(
|
|
1854
|
+
`Could not write session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1855
|
+
"error"
|
|
1856
|
+
);
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
options.log(`SESSION-DB-NO-REPLICATE: ${message}`, "warn");
|
|
1860
|
+
}
|
|
1861
|
+
function discardSessionDbDebris(options) {
|
|
1862
|
+
for (const path of [options.dbPath, `${options.dbPath}-wal`, `${options.dbPath}-shm`]) {
|
|
1863
|
+
try {
|
|
1864
|
+
unlinkSync2(path);
|
|
1865
|
+
} catch (error2) {
|
|
1866
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") continue;
|
|
1867
|
+
options.log(
|
|
1868
|
+
`Could not remove session-DB debris ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1869
|
+
"warn"
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
function splitDiagnostics(text) {
|
|
1875
|
+
return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
1876
|
+
}
|
|
1877
|
+
function logSynchroniserDiagnostics(result, options) {
|
|
1878
|
+
for (const line of splitDiagnostics(result.stderr)) options.log(line, "warn");
|
|
1879
|
+
}
|
|
1880
|
+
function parseSingleQuotedAssignment(line) {
|
|
1881
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
1882
|
+
if (!match || !match[2].startsWith("'")) return null;
|
|
1883
|
+
const valueSource = match[2];
|
|
1884
|
+
let value = "";
|
|
1885
|
+
for (let index = 1; index < valueSource.length; index++) {
|
|
1886
|
+
const character = valueSource[index];
|
|
1887
|
+
if (character !== "'") {
|
|
1888
|
+
value += character;
|
|
1889
|
+
continue;
|
|
1890
|
+
}
|
|
1891
|
+
if (index === valueSource.length - 1) return [match[1], value];
|
|
1892
|
+
if (valueSource.slice(index + 1, index + 4) !== "\\''") return null;
|
|
1893
|
+
value += "'";
|
|
1894
|
+
index += 3;
|
|
1895
|
+
}
|
|
1896
|
+
return null;
|
|
1897
|
+
}
|
|
1898
|
+
function parseSynchroniserEnv(stdout) {
|
|
1899
|
+
const values = {};
|
|
1900
|
+
for (const line of stdout.split("\n")) {
|
|
1901
|
+
if (line.trim() === "") continue;
|
|
1902
|
+
const assignment = parseSingleQuotedAssignment(line);
|
|
1903
|
+
if (!assignment) return null;
|
|
1904
|
+
values[assignment[0]] = assignment[1];
|
|
1905
|
+
}
|
|
1906
|
+
return values;
|
|
1907
|
+
}
|
|
1908
|
+
function runCommand(command, args, options) {
|
|
1909
|
+
return new Promise((resolve4) => {
|
|
1910
|
+
let child;
|
|
1911
|
+
let stdout = "";
|
|
1912
|
+
let stderr = "";
|
|
1913
|
+
let settled = false;
|
|
1914
|
+
const finish = (result) => {
|
|
1915
|
+
if (settled) return;
|
|
1916
|
+
settled = true;
|
|
1917
|
+
if (timer) clearTimeout(timer);
|
|
1918
|
+
resolve4(result);
|
|
1919
|
+
};
|
|
1920
|
+
try {
|
|
1921
|
+
child = spawn2(command, args, {
|
|
1922
|
+
env: options.env,
|
|
1923
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1924
|
+
});
|
|
1925
|
+
} catch (error2) {
|
|
1926
|
+
resolve4({
|
|
1927
|
+
code: null,
|
|
1928
|
+
stdout,
|
|
1929
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
1930
|
+
timedOut: false
|
|
1931
|
+
});
|
|
1932
|
+
return;
|
|
1933
|
+
}
|
|
1934
|
+
child.stdout?.setEncoding("utf8");
|
|
1935
|
+
child.stdout?.on("data", (chunk) => {
|
|
1936
|
+
stdout += chunk;
|
|
1937
|
+
});
|
|
1938
|
+
child.stderr?.setEncoding("utf8");
|
|
1939
|
+
child.stderr?.on("data", (chunk) => {
|
|
1940
|
+
stderr += chunk;
|
|
1941
|
+
});
|
|
1942
|
+
child.once("error", (error2) => {
|
|
1943
|
+
finish({
|
|
1944
|
+
code: null,
|
|
1945
|
+
stdout,
|
|
1946
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
1947
|
+
${error2.message}`,
|
|
1948
|
+
timedOut: false
|
|
1949
|
+
});
|
|
1950
|
+
});
|
|
1951
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
1952
|
+
const timer = setTimeout(
|
|
1953
|
+
() => {
|
|
1954
|
+
child.kill("SIGKILL");
|
|
1955
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1956
|
+
},
|
|
1957
|
+
Math.max(0, options.timeoutMs)
|
|
1958
|
+
);
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
async function ensureLitestreamConfig(options, env) {
|
|
1962
|
+
const configPath = options.litestreamConfig;
|
|
1963
|
+
if (!configPath) {
|
|
1964
|
+
markNoReplicate(options, "no Litestream configuration path was provided");
|
|
1965
|
+
reportRecord(
|
|
1966
|
+
"restore",
|
|
1967
|
+
"restore_misconfigured",
|
|
1968
|
+
"litestream_config_unavailable",
|
|
1969
|
+
null,
|
|
1970
|
+
options
|
|
1971
|
+
);
|
|
1972
|
+
return null;
|
|
1973
|
+
}
|
|
1974
|
+
try {
|
|
1975
|
+
if (statSync2(configPath).size > 0) return configPath;
|
|
1976
|
+
} catch (error2) {
|
|
1977
|
+
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
1978
|
+
options.log(
|
|
1979
|
+
`Could not inspect ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1980
|
+
"warn"
|
|
1981
|
+
);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
const rendered = await runSynchroniser(["litestream-config"], {
|
|
1985
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
1986
|
+
env
|
|
1987
|
+
});
|
|
1988
|
+
logSynchroniserDiagnostics(rendered, options);
|
|
1989
|
+
if (rendered.timedOut || rendered.code !== 0) {
|
|
1990
|
+
options.log(
|
|
1991
|
+
`Could not generate ${configPath}: runner-synchroniser litestream-config failed (${commandError(rendered)})`,
|
|
1992
|
+
"error"
|
|
1993
|
+
);
|
|
1994
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
1995
|
+
reportRecord(
|
|
1996
|
+
"restore",
|
|
1997
|
+
"restore_misconfigured",
|
|
1998
|
+
"litestream_config_unavailable",
|
|
1999
|
+
null,
|
|
2000
|
+
options
|
|
2001
|
+
);
|
|
2002
|
+
return null;
|
|
2003
|
+
}
|
|
2004
|
+
try {
|
|
2005
|
+
mkdirSync(dirname2(configPath), { recursive: true });
|
|
2006
|
+
writeFileSync(configPath, rendered.stdout);
|
|
2007
|
+
} catch (error2) {
|
|
2008
|
+
options.log(
|
|
2009
|
+
`Could not write ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2010
|
+
"error"
|
|
2011
|
+
);
|
|
2012
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2013
|
+
reportRecord(
|
|
2014
|
+
"restore",
|
|
2015
|
+
"restore_misconfigured",
|
|
2016
|
+
"litestream_config_unavailable",
|
|
2017
|
+
null,
|
|
2018
|
+
options
|
|
2019
|
+
);
|
|
2020
|
+
return null;
|
|
2021
|
+
}
|
|
2022
|
+
const version2 = await runCommand("litestream", ["version"], {
|
|
2023
|
+
env,
|
|
2024
|
+
timeoutMs: 1e4
|
|
2025
|
+
});
|
|
2026
|
+
const litestreamVersion = version2.code === 0 ? version2.stdout.trim() || "unknown" : "unknown";
|
|
2027
|
+
const regionEmpty = !/^\s*region:\s*\S+/m.test(rendered.stdout);
|
|
2028
|
+
options.log(
|
|
2029
|
+
`litestream ${litestreamVersion}; AWS_REGION=${env.AWS_REGION ?? "<unset>"} AWS_DEFAULT_REGION=${env.AWS_DEFAULT_REGION ?? "<unset>"}; rendered litestream.yml region empty: ${regionEmpty ? "yes" : "no"}`
|
|
2030
|
+
);
|
|
2031
|
+
return configPath;
|
|
2032
|
+
}
|
|
2033
|
+
function restoreGiveUp(options, reason, message, litestreamExitCode, outcome = "fresh_session_db") {
|
|
2034
|
+
discardSessionDbDebris(options);
|
|
2035
|
+
markNoReplicate(options, message);
|
|
2036
|
+
reportRecord("restore", outcome, reason, litestreamExitCode, options);
|
|
2037
|
+
}
|
|
2038
|
+
async function restoreSessionDb(options, configPath, env) {
|
|
2039
|
+
const restored = await runCommand(
|
|
2040
|
+
"litestream",
|
|
2041
|
+
["restore", "-config", configPath, "-if-db-not-exists", "-if-replica-exists", options.dbPath],
|
|
2042
|
+
{ env, timeoutMs: SESSION_DB_RESTORE_TIMEOUT_MS }
|
|
2043
|
+
);
|
|
2044
|
+
for (const line of splitDiagnostics(restored.stderr)) options.log(line, "warn");
|
|
2045
|
+
if (restored.timedOut || restored.code === 124 || restored.code === 137) {
|
|
2046
|
+
restoreGiveUp(
|
|
2047
|
+
options,
|
|
2048
|
+
"restore_deadline_exceeded",
|
|
2049
|
+
`SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within ${SESSION_DB_RESTORE_TIMEOUT_MS}ms; opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2050
|
+
restored.code ?? 124
|
|
2051
|
+
);
|
|
2052
|
+
return;
|
|
2053
|
+
}
|
|
2054
|
+
if (restored.code === null || restored.code === 125 || restored.code === 126 || restored.code === 127) {
|
|
2055
|
+
options.log(`litestream restore could not run (${commandError(restored)})`, "error");
|
|
2056
|
+
restoreGiveUp(
|
|
2057
|
+
options,
|
|
2058
|
+
"restore_tool_unusable",
|
|
2059
|
+
`restore tool is broken (${commandError(restored)}); opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2060
|
+
restored.code
|
|
2061
|
+
);
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
const classified = await runSynchroniser(
|
|
2065
|
+
[
|
|
2066
|
+
"session-db-classify",
|
|
2067
|
+
String(restored.code ?? 1),
|
|
2068
|
+
"1",
|
|
2069
|
+
"--on-unusable-replica=leave",
|
|
2070
|
+
"--fresh-db-fallback"
|
|
2071
|
+
],
|
|
2072
|
+
{ timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS, env }
|
|
2073
|
+
);
|
|
2074
|
+
logSynchroniserDiagnostics(classified, options);
|
|
2075
|
+
const classifyCode = classified.code;
|
|
2076
|
+
switch (classifyCode) {
|
|
2077
|
+
case 0:
|
|
2078
|
+
return;
|
|
2079
|
+
case 31:
|
|
2080
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2081
|
+
options.log(
|
|
2082
|
+
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
2083
|
+
"warn"
|
|
2084
|
+
);
|
|
2085
|
+
return;
|
|
2086
|
+
case 32:
|
|
2087
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2088
|
+
discardSessionDbDebris(options);
|
|
2089
|
+
markNoReplicate(
|
|
2090
|
+
options,
|
|
2091
|
+
"session-db-classify asked for another restore attempt (32), but this boot has budget for only one; treating it as a give-up rather than retrying"
|
|
2092
|
+
);
|
|
2093
|
+
return;
|
|
2094
|
+
case 30:
|
|
2095
|
+
restoreGiveUp(
|
|
2096
|
+
options,
|
|
2097
|
+
"classification_fatal",
|
|
2098
|
+
"session-db-classify returned fatal (30); see the FATAL message above",
|
|
2099
|
+
restored.code,
|
|
2100
|
+
"restore_misconfigured"
|
|
2101
|
+
);
|
|
2102
|
+
return;
|
|
2103
|
+
default:
|
|
2104
|
+
restoreGiveUp(
|
|
2105
|
+
options,
|
|
2106
|
+
"classification_unrecognised",
|
|
2107
|
+
`session-db-classify exited ${classifyCode ?? "null"}, which is none of its documented answers`,
|
|
2108
|
+
restored.code
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
async function verifySessionDb(options, configPath, env) {
|
|
2113
|
+
if (!options.noReplicateMarker || !fileExists(options.noReplicateMarker)) {
|
|
2114
|
+
const result = await runSynchroniser(["session-db-verify", configPath], {
|
|
2115
|
+
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
2116
|
+
env: {
|
|
2117
|
+
...env,
|
|
2118
|
+
// The synchroniser reads this value in SECONDS. Keep this at 120, not
|
|
2119
|
+
// 120_000, so the walkback gives up before the outer process bound.
|
|
2120
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
2121
|
+
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
2122
|
+
)
|
|
2123
|
+
}
|
|
2124
|
+
});
|
|
2125
|
+
logSynchroniserDiagnostics(result, options);
|
|
2126
|
+
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
2127
|
+
options.log(
|
|
2128
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
2129
|
+
"warn"
|
|
2130
|
+
);
|
|
2131
|
+
return false;
|
|
2132
|
+
}
|
|
2133
|
+
if (result.code === 34) {
|
|
2134
|
+
reportRecord(
|
|
2135
|
+
"verify",
|
|
2136
|
+
"session_db_boot_refused",
|
|
2137
|
+
result.stderr.includes("SESSION-DB-LOCAL-DISCARD-FAILED") ? "local_discard_failed" : "replica_separation_unproven",
|
|
2138
|
+
null,
|
|
2139
|
+
options
|
|
2140
|
+
);
|
|
2141
|
+
return true;
|
|
2142
|
+
}
|
|
2143
|
+
if (result.code === 33) {
|
|
2144
|
+
options.log(
|
|
2145
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
2146
|
+
"warn"
|
|
2147
|
+
);
|
|
2148
|
+
return false;
|
|
2149
|
+
}
|
|
2150
|
+
if (result.code !== 0) {
|
|
2151
|
+
options.log(
|
|
2152
|
+
`SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${result.code ?? "null"}, which is none of its documented answers; continuing with the restored opencode.db as-is`,
|
|
2153
|
+
"warn"
|
|
2154
|
+
);
|
|
2155
|
+
}
|
|
2156
|
+
return false;
|
|
2157
|
+
}
|
|
2158
|
+
options.log(
|
|
2159
|
+
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
2160
|
+
"debug"
|
|
2161
|
+
);
|
|
2162
|
+
return false;
|
|
2163
|
+
}
|
|
2164
|
+
function fileExists(path) {
|
|
2165
|
+
try {
|
|
2166
|
+
statSync2(path);
|
|
2167
|
+
return true;
|
|
2168
|
+
} catch (error2) {
|
|
2169
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
2170
|
+
return true;
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
async function restoreAndVerifySessionDb(options) {
|
|
2174
|
+
const env = options.env ?? process.env;
|
|
2175
|
+
clearMarker(options);
|
|
2176
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2177
|
+
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
2178
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2179
|
+
env
|
|
2180
|
+
});
|
|
2181
|
+
logSynchroniserDiagnostics(synchroniserEnv, options);
|
|
2182
|
+
if (synchroniserEnv.timedOut || synchroniserEnv.code !== 0) {
|
|
2183
|
+
options.log(
|
|
2184
|
+
`runner-synchroniser env could not resolve the session-DB configuration (${commandError(synchroniserEnv)})`,
|
|
2185
|
+
"error"
|
|
2186
|
+
);
|
|
2187
|
+
markNoReplicate(
|
|
2188
|
+
options,
|
|
2189
|
+
"could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
2190
|
+
);
|
|
2191
|
+
reportRecord(
|
|
2192
|
+
"restore",
|
|
2193
|
+
"restore_misconfigured",
|
|
2194
|
+
"synchroniser_config_unresolved",
|
|
2195
|
+
null,
|
|
2196
|
+
options
|
|
2197
|
+
);
|
|
2198
|
+
return { verifyFatal: false };
|
|
2199
|
+
}
|
|
2200
|
+
const values = parseSynchroniserEnv(synchroniserEnv.stdout);
|
|
2201
|
+
if (!values) {
|
|
2202
|
+
markNoReplicate(
|
|
2203
|
+
options,
|
|
2204
|
+
"the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
2205
|
+
);
|
|
2206
|
+
reportRecord(
|
|
2207
|
+
"restore",
|
|
2208
|
+
"restore_misconfigured",
|
|
2209
|
+
"synchroniser_config_unevaluable",
|
|
2210
|
+
null,
|
|
2211
|
+
options
|
|
2212
|
+
);
|
|
2213
|
+
return { verifyFatal: false };
|
|
2214
|
+
}
|
|
2215
|
+
const synchroniserDbPath = values.OPENCODE_DB_PATH;
|
|
2216
|
+
if (!synchroniserDbPath) {
|
|
2217
|
+
markNoReplicate(
|
|
2218
|
+
options,
|
|
2219
|
+
"run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
2220
|
+
);
|
|
2221
|
+
reportRecord(
|
|
2222
|
+
"restore",
|
|
2223
|
+
"restore_misconfigured",
|
|
2224
|
+
"synchroniser_config_incomplete",
|
|
2225
|
+
null,
|
|
2226
|
+
options
|
|
2227
|
+
);
|
|
2228
|
+
return { verifyFatal: false };
|
|
2229
|
+
}
|
|
2230
|
+
if (resolvePath(synchroniserDbPath) !== resolvePath(options.dbPath)) {
|
|
2231
|
+
options.log(
|
|
2232
|
+
`runner-synchroniser reported OPENCODE_DB_PATH=${synchroniserDbPath}, but OpenCode uses ${options.dbPath}; continuing with OpenCode's session-DB path`,
|
|
2233
|
+
"warn"
|
|
2234
|
+
);
|
|
2235
|
+
}
|
|
2236
|
+
if (!values.PERSISTENCE_BUCKET) {
|
|
2237
|
+
options.log(
|
|
2238
|
+
"SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated.",
|
|
2239
|
+
"warn"
|
|
2240
|
+
);
|
|
2241
|
+
return { verifyFatal: false };
|
|
2242
|
+
}
|
|
2243
|
+
const configPath = await ensureLitestreamConfig(options, env);
|
|
2244
|
+
if (!configPath) return { verifyFatal: false };
|
|
2245
|
+
await restoreSessionDb(options, configPath, env);
|
|
2246
|
+
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
2247
|
+
return { verifyFatal: false };
|
|
2248
|
+
}
|
|
2249
|
+
return { verifyFatal: await verifySessionDb(options, configPath, env) };
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// src/lib/opencode/session-db-provenance.ts
|
|
2253
|
+
import { createRequire } from "module";
|
|
2254
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
2255
|
+
import { dirname as dirname3, join as join3 } from "path";
|
|
2256
|
+
var require2 = createRequire(import.meta.url);
|
|
2257
|
+
function readSessionDbMigrationIds(dbPath) {
|
|
2258
|
+
let db;
|
|
2259
|
+
try {
|
|
2260
|
+
const { DatabaseSync } = require2("node:sqlite");
|
|
2261
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
2262
|
+
const columns = db.prepare("PRAGMA table_info(migration)").all();
|
|
2263
|
+
const hasExpectedShape = columns.length === 2 && columns.some(
|
|
2264
|
+
(column) => column.name === "id" && typeof column.type === "string" && column.type.toUpperCase() === "TEXT" && column.pk === 1
|
|
2265
|
+
) && columns.some(
|
|
2266
|
+
(column) => column.name === "time_completed" && typeof column.type === "string" && column.type.toUpperCase() === "INTEGER" && column.notnull === 1 && column.pk === 0
|
|
2267
|
+
);
|
|
2268
|
+
if (!hasExpectedShape) {
|
|
2269
|
+
console.warn(`[readSessionDbMigrationIds] unsupported migration table shape in ${dbPath}`);
|
|
2270
|
+
return null;
|
|
2271
|
+
}
|
|
2272
|
+
const rows = db.prepare("SELECT id FROM migration ORDER BY id").all();
|
|
2273
|
+
if (rows.some((row) => typeof row.id !== "string")) return null;
|
|
2274
|
+
return rows.map((row) => row.id);
|
|
2275
|
+
} catch (error2) {
|
|
2276
|
+
console.warn(
|
|
2277
|
+
`[readSessionDbMigrationIds] could not read migration history from ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2278
|
+
);
|
|
2279
|
+
return null;
|
|
2280
|
+
} finally {
|
|
2281
|
+
try {
|
|
2282
|
+
db?.close();
|
|
2283
|
+
} catch (error2) {
|
|
2284
|
+
console.warn(
|
|
2285
|
+
`[readSessionDbMigrationIds] could not close ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2286
|
+
);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
2291
|
+
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
2292
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
2293
|
+
}
|
|
2294
|
+
function loadSessionDbProvenanceState(path) {
|
|
2295
|
+
let value;
|
|
2296
|
+
try {
|
|
2297
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
2298
|
+
} catch (error2) {
|
|
2299
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return {};
|
|
2300
|
+
console.error(
|
|
2301
|
+
`[session-db-provenance] could not read ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2302
|
+
);
|
|
2303
|
+
return {};
|
|
2304
|
+
}
|
|
2305
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2306
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2307
|
+
return {};
|
|
2308
|
+
}
|
|
2309
|
+
const state = {};
|
|
2310
|
+
for (const [dbPath, record] of Object.entries(value)) {
|
|
2311
|
+
if (!isSessionDbProvenanceRecord(record)) {
|
|
2312
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2313
|
+
return {};
|
|
2314
|
+
}
|
|
2315
|
+
state[dbPath] = record;
|
|
2316
|
+
}
|
|
2317
|
+
return state;
|
|
2318
|
+
}
|
|
2319
|
+
function saveSessionDbProvenanceState(path, state) {
|
|
2320
|
+
try {
|
|
2321
|
+
mkdirSync2(dirname3(path), { recursive: true });
|
|
2322
|
+
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
2323
|
+
`, "utf8");
|
|
2324
|
+
} catch (error2) {
|
|
2325
|
+
console.error(
|
|
2326
|
+
`[session-db-provenance] could not write ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2327
|
+
);
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
function evaluateSessionDbProvenance(input) {
|
|
2331
|
+
const { currentVersion, currentIds, previous } = input;
|
|
2332
|
+
if (!previous) return { anomaly: false, reason: null };
|
|
2333
|
+
const current = new Set(currentIds);
|
|
2334
|
+
const prior = new Set(previous.migrationIds);
|
|
2335
|
+
for (const id of prior) {
|
|
2336
|
+
if (!current.has(id)) return { anomaly: true, reason: "migration-history-regressed" };
|
|
2337
|
+
}
|
|
2338
|
+
if (current.size > prior.size && previous.opencodeVersion === currentVersion) {
|
|
2339
|
+
return { anomaly: true, reason: "foreign-version-migrations" };
|
|
2340
|
+
}
|
|
2341
|
+
return { anomaly: false, reason: null };
|
|
2342
|
+
}
|
|
2343
|
+
function checkSessionDbProvenance(input) {
|
|
2344
|
+
const { dbPath, currentVersion, homeDir, env } = input;
|
|
2345
|
+
const path = sessionDbProvenanceStatePath(homeDir, env);
|
|
2346
|
+
const state = loadSessionDbProvenanceState(path);
|
|
2347
|
+
const previous = state[dbPath];
|
|
2348
|
+
const currentIds = readSessionDbMigrationIds(dbPath);
|
|
2349
|
+
if (currentIds === null) {
|
|
2350
|
+
return {
|
|
2351
|
+
anomaly: false,
|
|
2352
|
+
reason: null,
|
|
2353
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2354
|
+
migrationDelta: null
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
const decision = evaluateSessionDbProvenance({ currentVersion, currentIds, previous });
|
|
2358
|
+
const migrationDelta = previous ? new Set(currentIds).size - new Set(previous.migrationIds).size : null;
|
|
2359
|
+
state[dbPath] = {
|
|
2360
|
+
opencodeVersion: currentVersion,
|
|
2361
|
+
migrationIds: currentIds,
|
|
2362
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2363
|
+
};
|
|
2364
|
+
saveSessionDbProvenanceState(path, state);
|
|
2365
|
+
return {
|
|
2366
|
+
...decision,
|
|
2367
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2368
|
+
migrationDelta
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
function isSessionDbProvenanceRecord(value) {
|
|
2372
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2373
|
+
const record = value;
|
|
2374
|
+
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
2375
|
+
}
|
|
2376
|
+
|
|
1696
2377
|
// src/lib/opencode/opencode-version-gate.ts
|
|
1697
2378
|
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
1698
2379
|
function isQueueValidatedVersion(version2) {
|
|
@@ -1707,7 +2388,7 @@ function buildOpenCodeVersionWarning(version2) {
|
|
|
1707
2388
|
}
|
|
1708
2389
|
|
|
1709
2390
|
// src/lib/opencode/process.ts
|
|
1710
|
-
import { execSync, spawn } from "child_process";
|
|
2391
|
+
import { execSync, spawn as spawn3 } from "child_process";
|
|
1711
2392
|
|
|
1712
2393
|
// src/lib/process-stop.ts
|
|
1713
2394
|
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
@@ -1717,7 +2398,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1717
2398
|
if (child.exitCode !== null || child.signalCode !== null) {
|
|
1718
2399
|
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
1719
2400
|
}
|
|
1720
|
-
return new Promise((
|
|
2401
|
+
return new Promise((resolve4, reject) => {
|
|
1721
2402
|
let forced = false;
|
|
1722
2403
|
let settled = false;
|
|
1723
2404
|
const timer = setTimeout(() => {
|
|
@@ -1737,7 +2418,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
1737
2418
|
settled = true;
|
|
1738
2419
|
clearTimeout(timer);
|
|
1739
2420
|
child.removeListener("exit", onExit);
|
|
1740
|
-
|
|
2421
|
+
resolve4(result);
|
|
1741
2422
|
};
|
|
1742
2423
|
const fail = (error2) => {
|
|
1743
2424
|
if (settled) return;
|
|
@@ -1919,18 +2600,27 @@ async function findHealthyOpenCodeInstances() {
|
|
|
1919
2600
|
}
|
|
1920
2601
|
return healthy;
|
|
1921
2602
|
}
|
|
1922
|
-
async function startOpenCode(port) {
|
|
2603
|
+
async function startOpenCode(port, options = {}) {
|
|
1923
2604
|
let command = "opencode";
|
|
1924
|
-
|
|
2605
|
+
const printLogs = options.inheritStdio ? ["--print-logs"] : [];
|
|
2606
|
+
let args = ["serve", "--port", port.toString(), "--hostname", "127.0.0.1", ...printLogs];
|
|
1925
2607
|
try {
|
|
1926
2608
|
execSync("which opencode", { stdio: "ignore" });
|
|
1927
2609
|
} catch {
|
|
1928
2610
|
command = "npx";
|
|
1929
|
-
args = [
|
|
1930
|
-
|
|
1931
|
-
|
|
2611
|
+
args = [
|
|
2612
|
+
"opencode",
|
|
2613
|
+
"serve",
|
|
2614
|
+
"--port",
|
|
2615
|
+
port.toString(),
|
|
2616
|
+
"--hostname",
|
|
2617
|
+
"127.0.0.1",
|
|
2618
|
+
...printLogs
|
|
2619
|
+
];
|
|
2620
|
+
}
|
|
2621
|
+
const child = spawn3(command, args, {
|
|
1932
2622
|
detached: true,
|
|
1933
|
-
stdio: "ignore",
|
|
2623
|
+
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
1934
2624
|
cwd: process.cwd()
|
|
1935
2625
|
});
|
|
1936
2626
|
return child;
|
|
@@ -2416,7 +3106,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2416
3106
|
}
|
|
2417
3107
|
}
|
|
2418
3108
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2419
|
-
await new Promise((
|
|
3109
|
+
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
2420
3110
|
}
|
|
2421
3111
|
}
|
|
2422
3112
|
return null;
|
|
@@ -2462,6 +3152,44 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2462
3152
|
}
|
|
2463
3153
|
return lastOk ?? last;
|
|
2464
3154
|
}
|
|
3155
|
+
function collectSubagentSessions(messages, userMessageId) {
|
|
3156
|
+
if (!messages || messages.length === 0) return [];
|
|
3157
|
+
const byParent = messages.filter(
|
|
3158
|
+
(message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
|
|
3159
|
+
);
|
|
3160
|
+
const assistants = byParent.length > 0 ? byParent : [];
|
|
3161
|
+
if (assistants.length === 0) {
|
|
3162
|
+
const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
|
|
3163
|
+
if (userIndex === -1) return [];
|
|
3164
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
3165
|
+
const message = messages[i];
|
|
3166
|
+
if (roleOf(message) === "user") break;
|
|
3167
|
+
if (roleOf(message) === "assistant") assistants.push(message);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
const refs = [];
|
|
3171
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3172
|
+
for (const message of assistants) {
|
|
3173
|
+
const parts = Array.isArray(message.parts) ? message.parts : [];
|
|
3174
|
+
for (const part of parts) {
|
|
3175
|
+
if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
|
|
3176
|
+
continue;
|
|
3177
|
+
const state = part.state;
|
|
3178
|
+
if (!state || typeof state !== "object") continue;
|
|
3179
|
+
const metadata = state.metadata;
|
|
3180
|
+
if (!metadata || typeof metadata !== "object") continue;
|
|
3181
|
+
const sessionId = metadata.sessionId;
|
|
3182
|
+
if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
|
|
3183
|
+
seen.add(sessionId);
|
|
3184
|
+
const start = state.time?.start;
|
|
3185
|
+
refs.push({
|
|
3186
|
+
sessionId,
|
|
3187
|
+
startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
return refs;
|
|
3192
|
+
}
|
|
2465
3193
|
function messageUsage(messages, userMessageId) {
|
|
2466
3194
|
if (!messages || messages.length === 0) return null;
|
|
2467
3195
|
const byParentAll = messages.filter(
|
|
@@ -2590,8 +3318,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2590
3318
|
}
|
|
2591
3319
|
return false;
|
|
2592
3320
|
}
|
|
2593
|
-
function
|
|
2594
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3321
|
+
function classifyReplyAuthError(reply) {
|
|
2595
3322
|
const error2 = errorOf(reply);
|
|
2596
3323
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2597
3324
|
const e = error2;
|
|
@@ -2616,6 +3343,32 @@ function messageFailure(messages, userMessageId) {
|
|
|
2616
3343
|
}
|
|
2617
3344
|
return null;
|
|
2618
3345
|
}
|
|
3346
|
+
function messageFailure(messages, userMessageId) {
|
|
3347
|
+
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
3348
|
+
}
|
|
3349
|
+
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
3350
|
+
if (!messages || messages.length === 0) return null;
|
|
3351
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3352
|
+
const message = messages[i];
|
|
3353
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3354
|
+
const created = createdOf(message);
|
|
3355
|
+
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
3356
|
+
const failure = classifyReplyAuthError(message);
|
|
3357
|
+
if (failure) {
|
|
3358
|
+
if (!failure.providerId) return null;
|
|
3359
|
+
return { providerId: failure.providerId, outcome: "failed", failure };
|
|
3360
|
+
}
|
|
3361
|
+
const providerId = message.info?.providerID;
|
|
3362
|
+
if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
|
|
3363
|
+
return { providerId, outcome: "succeeded" };
|
|
3364
|
+
}
|
|
3365
|
+
return null;
|
|
3366
|
+
}
|
|
3367
|
+
return null;
|
|
3368
|
+
}
|
|
3369
|
+
function findSubagentAuthOutcome(messages, sinceMs) {
|
|
3370
|
+
return findLatestSubagentAuthOutcome(messages, sinceMs);
|
|
3371
|
+
}
|
|
2619
3372
|
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2620
3373
|
if (classified != null) return classified;
|
|
2621
3374
|
if (hasConfiguredProvider !== false) return null;
|
|
@@ -2769,13 +3522,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2769
3522
|
}
|
|
2770
3523
|
|
|
2771
3524
|
// src/lib/opencode/session-db-size.ts
|
|
2772
|
-
import { statSync as
|
|
2773
|
-
import { join as
|
|
3525
|
+
import { statSync as statSync3 } from "fs";
|
|
3526
|
+
import { join as join4 } from "path";
|
|
2774
3527
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2775
3528
|
function statSessionDbBytes(homeDir) {
|
|
2776
|
-
const dbPath =
|
|
3529
|
+
const dbPath = join4(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2777
3530
|
try {
|
|
2778
|
-
return
|
|
3531
|
+
return statSync3(dbPath).size;
|
|
2779
3532
|
} catch (err) {
|
|
2780
3533
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2781
3534
|
if (!isMissingFile) {
|
|
@@ -2801,11 +3554,11 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
2801
3554
|
}
|
|
2802
3555
|
|
|
2803
3556
|
// src/lib/opencode/session-db-reclaim.ts
|
|
2804
|
-
import { statSync as
|
|
2805
|
-
import { dirname as
|
|
3557
|
+
import { statSync as statSync4, statfsSync } from "fs";
|
|
3558
|
+
import { dirname as dirname4 } from "path";
|
|
2806
3559
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2807
3560
|
try {
|
|
2808
|
-
const fsStats = statfsSync(
|
|
3561
|
+
const fsStats = statfsSync(dirname4(dbPath));
|
|
2809
3562
|
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2810
3563
|
if (availableBytes < requiredBytes) {
|
|
2811
3564
|
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
@@ -2874,7 +3627,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2874
3627
|
);
|
|
2875
3628
|
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2876
3629
|
}
|
|
2877
|
-
const fileBytesForGuard =
|
|
3630
|
+
const fileBytesForGuard = statSync4(dbPath).size;
|
|
2878
3631
|
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2879
3632
|
if (skipReason !== null) {
|
|
2880
3633
|
console.warn(
|
|
@@ -2946,7 +3699,6 @@ var StreamForwarder = class {
|
|
|
2946
3699
|
handleFrame(frame) {
|
|
2947
3700
|
switch (frame.type) {
|
|
2948
3701
|
case "open":
|
|
2949
|
-
this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
|
|
2950
3702
|
void this.handleOpen(frame);
|
|
2951
3703
|
break;
|
|
2952
3704
|
case "req_data":
|
|
@@ -2982,12 +3734,21 @@ var StreamForwarder = class {
|
|
|
2982
3734
|
const { sid, method, path, headers, has_body } = frame;
|
|
2983
3735
|
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
2984
3736
|
const startedAt = Date.now();
|
|
3737
|
+
if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3738
|
+
this.callbacks.onOpen?.(sid, method, path);
|
|
3739
|
+
}
|
|
2985
3740
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
2986
3741
|
this.callbacks.onDrainPing?.();
|
|
2987
3742
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
2988
3743
|
this.send({ type: "res_end", sid });
|
|
2989
3744
|
return;
|
|
2990
3745
|
}
|
|
3746
|
+
if (path === TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3747
|
+
this.callbacks.onUsageRearmPing?.();
|
|
3748
|
+
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
3749
|
+
this.send({ type: "res_end", sid });
|
|
3750
|
+
return;
|
|
3751
|
+
}
|
|
2991
3752
|
if (process.env.DEBUG) {
|
|
2992
3753
|
log("debug", "agent_request", {
|
|
2993
3754
|
correlation_id: correlationId,
|
|
@@ -3002,12 +3763,12 @@ var StreamForwarder = class {
|
|
|
3002
3763
|
let endBody;
|
|
3003
3764
|
if (has_body) {
|
|
3004
3765
|
const chunks = [];
|
|
3005
|
-
bodyPromise = new Promise((
|
|
3766
|
+
bodyPromise = new Promise((resolve4) => {
|
|
3006
3767
|
pushBody = (buf) => {
|
|
3007
3768
|
chunks.push(buf);
|
|
3008
3769
|
};
|
|
3009
3770
|
endBody = () => {
|
|
3010
|
-
|
|
3771
|
+
resolve4(Buffer.concat(chunks));
|
|
3011
3772
|
};
|
|
3012
3773
|
});
|
|
3013
3774
|
}
|
|
@@ -3132,11 +3893,12 @@ function connectTunnel(options) {
|
|
|
3132
3893
|
onResponse,
|
|
3133
3894
|
onInfo,
|
|
3134
3895
|
onWarning,
|
|
3135
|
-
onDrainPing
|
|
3896
|
+
onDrainPing,
|
|
3897
|
+
onUsageRearmPing
|
|
3136
3898
|
} = options;
|
|
3137
3899
|
const tunnelUrl = getTunnelUrlConfig();
|
|
3138
3900
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
3139
|
-
return new Promise((
|
|
3901
|
+
return new Promise((resolve4, reject) => {
|
|
3140
3902
|
const ws = new WebSocket2(url, {
|
|
3141
3903
|
headers: {
|
|
3142
3904
|
Authorization: authHeader
|
|
@@ -3144,7 +3906,8 @@ function connectTunnel(options) {
|
|
|
3144
3906
|
});
|
|
3145
3907
|
const forwarder = new StreamForwarder(ws, port, {
|
|
3146
3908
|
onHead: () => onResponse?.(),
|
|
3147
|
-
onDrainPing: () => onDrainPing?.()
|
|
3909
|
+
onDrainPing: () => onDrainPing?.(),
|
|
3910
|
+
onUsageRearmPing: () => onUsageRearmPing?.()
|
|
3148
3911
|
});
|
|
3149
3912
|
const connectionTimeout = setTimeout(() => {
|
|
3150
3913
|
ws.close();
|
|
@@ -3187,8 +3950,8 @@ function connectTunnel(options) {
|
|
|
3187
3950
|
try {
|
|
3188
3951
|
message = JSON.parse(data.toString());
|
|
3189
3952
|
} catch (error2) {
|
|
3190
|
-
const
|
|
3191
|
-
onError?.(`Failed to handle message: ${
|
|
3953
|
+
const errorMessage2 = error2 instanceof Error ? error2.message : "Unknown error";
|
|
3954
|
+
onError?.(`Failed to handle message: ${errorMessage2}`);
|
|
3192
3955
|
return;
|
|
3193
3956
|
}
|
|
3194
3957
|
if (isStreamFrame(message)) {
|
|
@@ -3200,7 +3963,7 @@ function connectTunnel(options) {
|
|
|
3200
3963
|
clearTimeout(connectionTimeout);
|
|
3201
3964
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
3202
3965
|
onConnected?.(connectedAgentId);
|
|
3203
|
-
|
|
3966
|
+
resolve4({
|
|
3204
3967
|
ws,
|
|
3205
3968
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
3206
3969
|
});
|
|
@@ -3305,6 +4068,7 @@ var RunnerConnection = class {
|
|
|
3305
4068
|
onError: (error2) => events.onError?.(error2),
|
|
3306
4069
|
onResponse: () => events.onResponse?.(),
|
|
3307
4070
|
onDrainPing: () => events.onDrainPing?.(),
|
|
4071
|
+
onUsageRearmPing: () => events.onUsageRearmPing?.(),
|
|
3308
4072
|
onInfo: (message) => events.onInfo?.(message),
|
|
3309
4073
|
onWarning: (message) => events.onWarning?.(message)
|
|
3310
4074
|
});
|
|
@@ -3331,10 +4095,10 @@ var RunnerConnection = class {
|
|
|
3331
4095
|
};
|
|
3332
4096
|
|
|
3333
4097
|
// src/lib/tunnel/ready-marker.ts
|
|
3334
|
-
import { writeFileSync } from "fs";
|
|
4098
|
+
import { writeFileSync as writeFileSync3 } from "fs";
|
|
3335
4099
|
function writeTunnelReadyMarker(path, agentId) {
|
|
3336
4100
|
try {
|
|
3337
|
-
|
|
4101
|
+
writeFileSync3(path, `${agentId}
|
|
3338
4102
|
`);
|
|
3339
4103
|
return { ok: true };
|
|
3340
4104
|
} catch (error2) {
|
|
@@ -3343,9 +4107,9 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
3343
4107
|
}
|
|
3344
4108
|
|
|
3345
4109
|
// src/lib/replication.ts
|
|
3346
|
-
import { spawn as
|
|
4110
|
+
import { spawn as spawn4 } from "child_process";
|
|
3347
4111
|
function startSessionDbReplication(configPath) {
|
|
3348
|
-
return
|
|
4112
|
+
return spawn4("litestream", ["replicate", "-config", configPath], {
|
|
3349
4113
|
stdio: "inherit"
|
|
3350
4114
|
});
|
|
3351
4115
|
}
|
|
@@ -3358,10 +4122,36 @@ async function stopSessionDbReplication(child, timeoutMs) {
|
|
|
3358
4122
|
);
|
|
3359
4123
|
}
|
|
3360
4124
|
|
|
4125
|
+
// src/lib/process-liveness.ts
|
|
4126
|
+
import { readFileSync as readFileSync4 } from "fs";
|
|
4127
|
+
function isProcessAlive(pid) {
|
|
4128
|
+
try {
|
|
4129
|
+
process.kill(pid, 0);
|
|
4130
|
+
} catch (error2) {
|
|
4131
|
+
const code = error2.code;
|
|
4132
|
+
if (code === "ESRCH") return false;
|
|
4133
|
+
if (code === "EPERM") return true;
|
|
4134
|
+
console.error(
|
|
4135
|
+
`[process-liveness] could not probe process ${pid}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4136
|
+
);
|
|
4137
|
+
return false;
|
|
4138
|
+
}
|
|
4139
|
+
if (process.platform !== "linux") return true;
|
|
4140
|
+
try {
|
|
4141
|
+
const status2 = readFileSync4(`/proc/${pid}/status`, "utf8");
|
|
4142
|
+
return !/^State:\s+Z(?:\s|$)/m.test(status2);
|
|
4143
|
+
} catch (error2) {
|
|
4144
|
+
console.error(
|
|
4145
|
+
`[process-liveness] could not inspect /proc/${pid}/status: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4146
|
+
);
|
|
4147
|
+
return true;
|
|
4148
|
+
}
|
|
4149
|
+
}
|
|
4150
|
+
|
|
3361
4151
|
// src/lib/openai-usage.ts
|
|
3362
|
-
import { readFileSync as
|
|
3363
|
-
import { homedir as
|
|
3364
|
-
import { join as
|
|
4152
|
+
import { readFileSync as readFileSync5 } from "fs";
|
|
4153
|
+
import { homedir as homedir3 } from "os";
|
|
4154
|
+
import { join as join5 } from "path";
|
|
3365
4155
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3366
4156
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3367
4157
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3375,7 +4165,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3375
4165
|
}
|
|
3376
4166
|
function readOpenCodeChatGptCredentials() {
|
|
3377
4167
|
try {
|
|
3378
|
-
const raw =
|
|
4168
|
+
const raw = readFileSync5(join5(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3379
4169
|
let parsed;
|
|
3380
4170
|
try {
|
|
3381
4171
|
parsed = JSON.parse(raw);
|
|
@@ -3397,6 +4187,23 @@ function readOpenCodeChatGptCredentials() {
|
|
|
3397
4187
|
return null;
|
|
3398
4188
|
}
|
|
3399
4189
|
}
|
|
4190
|
+
function parseChatGptIdentity(accessToken) {
|
|
4191
|
+
const segments = accessToken.split(".");
|
|
4192
|
+
if (segments.length !== 3) return null;
|
|
4193
|
+
let payload;
|
|
4194
|
+
try {
|
|
4195
|
+
const parsed = JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8"));
|
|
4196
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
4197
|
+
payload = parsed;
|
|
4198
|
+
} catch {
|
|
4199
|
+
return null;
|
|
4200
|
+
}
|
|
4201
|
+
const profile = payload["https://api.openai.com/profile"];
|
|
4202
|
+
const auth = payload["https://api.openai.com/auth"];
|
|
4203
|
+
const ownerEmail = typeof profile?.email === "string" ? profile.email.trim() || null : null;
|
|
4204
|
+
const planType = typeof auth?.chatgpt_plan_type === "string" ? auth.chatgpt_plan_type.trim() || null : null;
|
|
4205
|
+
return ownerEmail === null && planType === null ? null : { ownerEmail, planType };
|
|
4206
|
+
}
|
|
3400
4207
|
function toWindow2(headers, name) {
|
|
3401
4208
|
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
3402
4209
|
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
@@ -3472,6 +4279,7 @@ async function getOpenAiUsage(port) {
|
|
|
3472
4279
|
"credentials_expired"
|
|
3473
4280
|
);
|
|
3474
4281
|
}
|
|
4282
|
+
const subscription = parseChatGptIdentity(credentials2.accessToken);
|
|
3475
4283
|
const models = await resolveProbeModels(port);
|
|
3476
4284
|
if (models.length === 0) {
|
|
3477
4285
|
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
@@ -3504,7 +4312,7 @@ async function getOpenAiUsage(port) {
|
|
|
3504
4312
|
"no_usable_window"
|
|
3505
4313
|
);
|
|
3506
4314
|
}
|
|
3507
|
-
return usage;
|
|
4315
|
+
return { ...usage, subscription };
|
|
3508
4316
|
}
|
|
3509
4317
|
if (res.status === 401) {
|
|
3510
4318
|
throw new OpenAiUsageError(
|
|
@@ -3705,58 +4513,97 @@ function readDisk(homeDir) {
|
|
|
3705
4513
|
};
|
|
3706
4514
|
}
|
|
3707
4515
|
}
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
4516
|
+
var CPU_PEAK_WINDOW_MS = 6e4;
|
|
4517
|
+
var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
|
|
4518
|
+
var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4519
|
+
function createCpuPeakSampler() {
|
|
4520
|
+
const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4521
|
+
sampleHistory[0] = readCpuSample();
|
|
4522
|
+
let nextSampleIndex = 1;
|
|
4523
|
+
let sampleCount = 1;
|
|
4524
|
+
let peak = null;
|
|
4525
|
+
const timer = setInterval(() => {
|
|
3711
4526
|
const current = readCpuSample();
|
|
3712
|
-
const
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
const warnings = [];
|
|
3719
|
-
if (disk.warning) warnings.push(disk.warning);
|
|
3720
|
-
if (ecsWarning) warnings.push(ecsWarning);
|
|
3721
|
-
let cpuPercent = hostCpuPercent;
|
|
3722
|
-
let cpuCount = hostCpuCount;
|
|
3723
|
-
let memoryTotalBytes = totalmem();
|
|
3724
|
-
let memoryAvailableBytes = freemem();
|
|
3725
|
-
if (limits !== null) {
|
|
3726
|
-
cpuCount = limits.cpuCount;
|
|
3727
|
-
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3728
|
-
memoryAvailableBytes = clamp(
|
|
3729
|
-
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3730
|
-
0,
|
|
3731
|
-
limits.memoryTotalBytes
|
|
3732
|
-
);
|
|
3733
|
-
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4527
|
+
const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
|
|
4528
|
+
if (sampleFromWindowAgo !== void 0) {
|
|
4529
|
+
const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
|
|
4530
|
+
if (percentage !== null) {
|
|
4531
|
+
peak = peak === null ? percentage : Math.max(peak, percentage);
|
|
4532
|
+
}
|
|
3734
4533
|
}
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
4534
|
+
sampleHistory[nextSampleIndex] = current;
|
|
4535
|
+
nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4536
|
+
sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4537
|
+
}, CPU_PEAK_SAMPLE_INTERVAL_MS);
|
|
4538
|
+
return {
|
|
4539
|
+
takeAndReset: () => {
|
|
4540
|
+
const currentPeak = peak;
|
|
4541
|
+
peak = null;
|
|
4542
|
+
return currentPeak;
|
|
4543
|
+
},
|
|
4544
|
+
stop: () => clearInterval(timer)
|
|
4545
|
+
};
|
|
4546
|
+
}
|
|
4547
|
+
function createResourceUsageCollector(homeDir) {
|
|
4548
|
+
let previous = readCpuSample();
|
|
4549
|
+
const cpuPeakSampler = createCpuPeakSampler();
|
|
4550
|
+
return {
|
|
4551
|
+
collect: async () => {
|
|
4552
|
+
const current = readCpuSample();
|
|
4553
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
4554
|
+
const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
|
|
4555
|
+
const hostCpuCount = cpus().length;
|
|
4556
|
+
previous = current;
|
|
4557
|
+
const disk = readDisk(homeDir);
|
|
4558
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
4559
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
4560
|
+
const warnings = [];
|
|
4561
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
4562
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
4563
|
+
let cpuPercent = hostCpuPercent;
|
|
4564
|
+
let cpuPeakPercent = hostCpuPeakPercent;
|
|
4565
|
+
let cpuCount = hostCpuCount;
|
|
4566
|
+
let memoryTotalBytes = totalmem();
|
|
4567
|
+
let memoryAvailableBytes = freemem();
|
|
4568
|
+
if (limits !== null) {
|
|
4569
|
+
cpuCount = limits.cpuCount;
|
|
4570
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4571
|
+
memoryAvailableBytes = clamp(
|
|
4572
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4573
|
+
0,
|
|
4574
|
+
limits.memoryTotalBytes
|
|
4575
|
+
);
|
|
4576
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4577
|
+
cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4578
|
+
}
|
|
4579
|
+
return {
|
|
4580
|
+
usage: {
|
|
4581
|
+
cpuPercent,
|
|
4582
|
+
cpuPeakPercent,
|
|
4583
|
+
cpuCount,
|
|
4584
|
+
memoryTotalBytes,
|
|
4585
|
+
memoryAvailableBytes,
|
|
4586
|
+
diskTotalBytes: disk.totalBytes,
|
|
4587
|
+
diskFreeBytes: disk.freeBytes,
|
|
4588
|
+
opencodeDbBytes
|
|
4589
|
+
},
|
|
4590
|
+
warnings
|
|
4591
|
+
};
|
|
4592
|
+
},
|
|
4593
|
+
stop: cpuPeakSampler.stop
|
|
3747
4594
|
};
|
|
3748
4595
|
}
|
|
3749
4596
|
|
|
3750
4597
|
// src/lib/channels/driver.ts
|
|
3751
|
-
import { homedir as
|
|
4598
|
+
import { homedir as homedir4 } from "os";
|
|
3752
4599
|
|
|
3753
4600
|
// src/lib/runner-file-sync.ts
|
|
3754
|
-
import { join as
|
|
4601
|
+
import { join as join7 } from "path";
|
|
3755
4602
|
|
|
3756
4603
|
// src/lib/file-push.ts
|
|
3757
4604
|
import { randomUUID } from "crypto";
|
|
3758
4605
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
3759
|
-
import { basename, dirname as
|
|
4606
|
+
import { basename, dirname as dirname5, isAbsolute, join as join6, relative, resolve as resolve2, sep } from "path";
|
|
3760
4607
|
var FILE_MODE = 384;
|
|
3761
4608
|
var DIRECTORY_MODE = 448;
|
|
3762
4609
|
async function writePushedFile(request) {
|
|
@@ -3787,9 +4634,9 @@ async function writePushedFile(request) {
|
|
|
3787
4634
|
}
|
|
3788
4635
|
try {
|
|
3789
4636
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3790
|
-
|
|
4637
|
+
dirname5(candidate)
|
|
3791
4638
|
);
|
|
3792
|
-
const realTarget =
|
|
4639
|
+
const realTarget = join6(existingAncestor, ...missingSegments, basename(candidate));
|
|
3793
4640
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3794
4641
|
if (allowedDirectory === null) {
|
|
3795
4642
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3799,8 +4646,8 @@ async function writePushedFile(request) {
|
|
|
3799
4646
|
}
|
|
3800
4647
|
if (missingSegments.length > 0) {
|
|
3801
4648
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
3802
|
-
const realParent = await realpath(
|
|
3803
|
-
if (realParent !==
|
|
4649
|
+
const realParent = await realpath(dirname5(realTarget));
|
|
4650
|
+
if (realParent !== dirname5(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
3804
4651
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
3805
4652
|
path: realTarget,
|
|
3806
4653
|
bytes,
|
|
@@ -3825,7 +4672,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3825
4672
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3826
4673
|
return null;
|
|
3827
4674
|
}
|
|
3828
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4675
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3829
4676
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3830
4677
|
return null;
|
|
3831
4678
|
}
|
|
@@ -3843,7 +4690,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
3843
4690
|
try {
|
|
3844
4691
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
3845
4692
|
} catch (err) {
|
|
3846
|
-
const parent =
|
|
4693
|
+
const parent = dirname5(current);
|
|
3847
4694
|
if (err.code !== "ENOENT" || parent === current) {
|
|
3848
4695
|
throw err;
|
|
3849
4696
|
}
|
|
@@ -3898,13 +4745,13 @@ function contains(realDirectory, realTarget) {
|
|
|
3898
4745
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3899
4746
|
let current = existingAncestor;
|
|
3900
4747
|
for (const segment of missingSegments) {
|
|
3901
|
-
current =
|
|
4748
|
+
current = join6(current, segment);
|
|
3902
4749
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3903
4750
|
await chmod(current, DIRECTORY_MODE);
|
|
3904
4751
|
}
|
|
3905
4752
|
}
|
|
3906
4753
|
async function writeAtomically(realTarget, content) {
|
|
3907
|
-
const temporaryPath =
|
|
4754
|
+
const temporaryPath = join6(dirname5(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3908
4755
|
let handle;
|
|
3909
4756
|
try {
|
|
3910
4757
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -4034,12 +4881,12 @@ var NOT_APPLIED = {
|
|
|
4034
4881
|
opencodeAuthApplied: false
|
|
4035
4882
|
};
|
|
4036
4883
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
4037
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4038
|
-
return expanded ===
|
|
4884
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4885
|
+
return expanded === join7(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
4039
4886
|
}
|
|
4040
4887
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
4041
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4042
|
-
return expanded ===
|
|
4888
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4889
|
+
return expanded === join7(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
4043
4890
|
}
|
|
4044
4891
|
async function applyOne(options, file) {
|
|
4045
4892
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -4566,6 +5413,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4566
5413
|
* and stops opencode.
|
|
4567
5414
|
*/
|
|
4568
5415
|
stopped = false;
|
|
5416
|
+
recycleRequestedFlag = false;
|
|
4569
5417
|
constructor(config) {
|
|
4570
5418
|
this.agentId = config.agentId;
|
|
4571
5419
|
this.port = config.port;
|
|
@@ -4585,7 +5433,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4585
5433
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
4586
5434
|
this.now = config.now ?? (() => Date.now());
|
|
4587
5435
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
4588
|
-
this.homeDir = config.homeDir ??
|
|
5436
|
+
this.homeDir = config.homeDir ?? homedir4();
|
|
4589
5437
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4590
5438
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4591
5439
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
@@ -4671,6 +5519,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4671
5519
|
let dispatched = 0;
|
|
4672
5520
|
try {
|
|
4673
5521
|
const conversations = await this.getPendingConversations();
|
|
5522
|
+
if (this.recycleRequestedFlag) {
|
|
5523
|
+
this.stop();
|
|
5524
|
+
}
|
|
4674
5525
|
if (conversations.length > 0) {
|
|
4675
5526
|
const total = conversations.reduce((sum, c) => sum + (c.pending_message_count ?? 0), 0);
|
|
4676
5527
|
this.log({
|
|
@@ -4799,6 +5650,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4799
5650
|
stop() {
|
|
4800
5651
|
this.stopped = true;
|
|
4801
5652
|
}
|
|
5653
|
+
/**
|
|
5654
|
+
* The server clears this request when a new MicroVM identity is recorded, so a
|
|
5655
|
+
* same-VM tunnel reconnect does not consume it. This is a plain read rather
|
|
5656
|
+
* than a consume; `run.ts` guards the action once-only.
|
|
5657
|
+
*/
|
|
5658
|
+
get recycleRequested() {
|
|
5659
|
+
return this.recycleRequestedFlag;
|
|
5660
|
+
}
|
|
4802
5661
|
/**
|
|
4803
5662
|
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
4804
5663
|
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
@@ -4936,7 +5795,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4936
5795
|
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
4937
5796
|
break;
|
|
4938
5797
|
}
|
|
4939
|
-
const
|
|
5798
|
+
const errorMessage2 = err instanceof Error ? err.message : String(err);
|
|
4940
5799
|
this.sessions.delete(conv.id);
|
|
4941
5800
|
this.supersede(conv.id, sessionId);
|
|
4942
5801
|
this.log({
|
|
@@ -4945,7 +5804,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4945
5804
|
conversation_id: conv.id,
|
|
4946
5805
|
message_id: message.id
|
|
4947
5806
|
});
|
|
4948
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5807
|
+
await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
|
|
4949
5808
|
this.log({
|
|
4950
5809
|
level: "warn",
|
|
4951
5810
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4956,7 +5815,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4956
5815
|
});
|
|
4957
5816
|
this.log({
|
|
4958
5817
|
level: "error",
|
|
4959
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
5818
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage2}`,
|
|
4960
5819
|
conversation_id: conv.id,
|
|
4961
5820
|
message_id: message.id
|
|
4962
5821
|
});
|
|
@@ -4977,14 +5836,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4977
5836
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4978
5837
|
this.sessions.delete(conv.id);
|
|
4979
5838
|
this.supersede(conv.id, sessionId);
|
|
4980
|
-
const
|
|
5839
|
+
const errorMessage2 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
4981
5840
|
this.log({
|
|
4982
5841
|
level: "error",
|
|
4983
|
-
message:
|
|
5842
|
+
message: errorMessage2,
|
|
4984
5843
|
conversation_id: conv.id,
|
|
4985
5844
|
message_id: message.id
|
|
4986
5845
|
});
|
|
4987
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5846
|
+
await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
|
|
4988
5847
|
this.log({
|
|
4989
5848
|
level: "warn",
|
|
4990
5849
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -5318,6 +6177,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5318
6177
|
});
|
|
5319
6178
|
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
5320
6179
|
}
|
|
6180
|
+
if (ocId !== null) {
|
|
6181
|
+
await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
|
|
6182
|
+
}
|
|
5321
6183
|
} catch (err) {
|
|
5322
6184
|
if (err instanceof ChannelAuthError) throw err;
|
|
5323
6185
|
this.log({
|
|
@@ -6282,6 +7144,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6282
7144
|
return;
|
|
6283
7145
|
}
|
|
6284
7146
|
inFlight.done = true;
|
|
7147
|
+
await this.reportSubagentAuthFailures(
|
|
7148
|
+
watcher.conv.id,
|
|
7149
|
+
inFlight.opencodeMessageId,
|
|
7150
|
+
inFlight.evidentMessageId,
|
|
7151
|
+
messages
|
|
7152
|
+
);
|
|
6285
7153
|
}
|
|
6286
7154
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6287
7155
|
return;
|
|
@@ -6524,6 +7392,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6524
7392
|
return;
|
|
6525
7393
|
}
|
|
6526
7394
|
inFlight.done = true;
|
|
7395
|
+
await this.reportSubagentAuthFailures(
|
|
7396
|
+
watcher.conv.id,
|
|
7397
|
+
inFlight.opencodeMessageId,
|
|
7398
|
+
inFlight.evidentMessageId,
|
|
7399
|
+
messages
|
|
7400
|
+
);
|
|
6527
7401
|
}
|
|
6528
7402
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6529
7403
|
}
|
|
@@ -6694,6 +7568,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6694
7568
|
});
|
|
6695
7569
|
return;
|
|
6696
7570
|
}
|
|
7571
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
6697
7572
|
this.dontRedispatch.delete(row.id);
|
|
6698
7573
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
6699
7574
|
return;
|
|
@@ -6855,6 +7730,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6855
7730
|
});
|
|
6856
7731
|
return;
|
|
6857
7732
|
}
|
|
7733
|
+
if (ocId !== null) {
|
|
7734
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
|
|
7735
|
+
}
|
|
6858
7736
|
this.dontRedispatch.delete(row.id);
|
|
6859
7737
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
6860
7738
|
}
|
|
@@ -6948,14 +7826,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6948
7826
|
this.unconfirmedDispatchFailures.delete(row.id);
|
|
6949
7827
|
this.sessions.delete(readoptConv.id);
|
|
6950
7828
|
this.supersede(readoptConv.id, sessionId);
|
|
6951
|
-
const
|
|
7829
|
+
const errorMessage2 = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
6952
7830
|
this.log({
|
|
6953
7831
|
level: "error",
|
|
6954
|
-
message:
|
|
7832
|
+
message: errorMessage2,
|
|
6955
7833
|
conversation_id: row.conversation_id,
|
|
6956
7834
|
message_id: row.id
|
|
6957
7835
|
});
|
|
6958
|
-
await this.markFailed(row.conversation_id, row.id, null,
|
|
7836
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage2).catch((markErr) => {
|
|
6959
7837
|
this.log({
|
|
6960
7838
|
level: "warn",
|
|
6961
7839
|
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -7570,6 +8448,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7570
8448
|
throw new Error(`Failed to get pending conversations: HTTP ${res.status}`);
|
|
7571
8449
|
}
|
|
7572
8450
|
const data = await res.json();
|
|
8451
|
+
this.recycleRequestedFlag = data.recycle_requested === true;
|
|
7573
8452
|
let conversations = data.conversations;
|
|
7574
8453
|
if (this.conversationFilter) {
|
|
7575
8454
|
conversations = conversations.filter((c) => c.id === this.conversationFilter);
|
|
@@ -7805,6 +8684,111 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7805
8684
|
reply?.info?.modelID ?? null
|
|
7806
8685
|
);
|
|
7807
8686
|
}
|
|
8687
|
+
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
8688
|
+
const providerId = failure.providerId ?? "(unknown)";
|
|
8689
|
+
try {
|
|
8690
|
+
const res = await this.fetchImpl(
|
|
8691
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
|
|
8692
|
+
{
|
|
8693
|
+
method: "POST",
|
|
8694
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
8695
|
+
body: JSON.stringify({
|
|
8696
|
+
provider_id: failure.providerId,
|
|
8697
|
+
model_id: failure.modelId,
|
|
8698
|
+
reason: failure.reason
|
|
8699
|
+
})
|
|
8700
|
+
}
|
|
8701
|
+
);
|
|
8702
|
+
if (!res.ok) {
|
|
8703
|
+
this.log({
|
|
8704
|
+
level: "warn",
|
|
8705
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8706
|
+
conversation_id: conversationId,
|
|
8707
|
+
message_id: messageId
|
|
8708
|
+
});
|
|
8709
|
+
}
|
|
8710
|
+
} catch (err) {
|
|
8711
|
+
this.log({
|
|
8712
|
+
level: "warn",
|
|
8713
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8714
|
+
conversation_id: conversationId,
|
|
8715
|
+
message_id: messageId
|
|
8716
|
+
});
|
|
8717
|
+
}
|
|
8718
|
+
}
|
|
8719
|
+
async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
|
|
8720
|
+
try {
|
|
8721
|
+
const res = await this.fetchImpl(
|
|
8722
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
|
|
8723
|
+
{
|
|
8724
|
+
method: "DELETE",
|
|
8725
|
+
headers: { Authorization: this.getAuthHeader() }
|
|
8726
|
+
}
|
|
8727
|
+
);
|
|
8728
|
+
if (!res.ok) {
|
|
8729
|
+
this.log({
|
|
8730
|
+
level: "warn",
|
|
8731
|
+
message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8732
|
+
conversation_id: conversationId,
|
|
8733
|
+
message_id: messageId
|
|
8734
|
+
});
|
|
8735
|
+
}
|
|
8736
|
+
} catch (err) {
|
|
8737
|
+
this.log({
|
|
8738
|
+
level: "warn",
|
|
8739
|
+
message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8740
|
+
conversation_id: conversationId,
|
|
8741
|
+
message_id: messageId
|
|
8742
|
+
});
|
|
8743
|
+
}
|
|
8744
|
+
}
|
|
8745
|
+
async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
|
|
8746
|
+
const refs = collectSubagentSessions(messages, opencodeMessageId);
|
|
8747
|
+
if (refs.length === 0) return;
|
|
8748
|
+
const failedProviders = /* @__PURE__ */ new Map();
|
|
8749
|
+
const succeededProviders = /* @__PURE__ */ new Set();
|
|
8750
|
+
for (const ref of refs) {
|
|
8751
|
+
try {
|
|
8752
|
+
const childMessages = await getSessionMessages(this.port, ref.sessionId);
|
|
8753
|
+
if (childMessages === null) {
|
|
8754
|
+
this.log({
|
|
8755
|
+
level: "debug",
|
|
8756
|
+
message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
|
|
8757
|
+
conversation_id: conversationId,
|
|
8758
|
+
message_id: evidentMessageId
|
|
8759
|
+
});
|
|
8760
|
+
continue;
|
|
8761
|
+
}
|
|
8762
|
+
const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
|
|
8763
|
+
if (!outcome) continue;
|
|
8764
|
+
if (outcome.outcome === "failed") {
|
|
8765
|
+
failedProviders.set(outcome.providerId, outcome.failure);
|
|
8766
|
+
} else {
|
|
8767
|
+
succeededProviders.add(outcome.providerId);
|
|
8768
|
+
}
|
|
8769
|
+
} catch (err) {
|
|
8770
|
+
this.log({
|
|
8771
|
+
level: "warn",
|
|
8772
|
+
message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8773
|
+
conversation_id: conversationId,
|
|
8774
|
+
message_id: evidentMessageId
|
|
8775
|
+
});
|
|
8776
|
+
}
|
|
8777
|
+
}
|
|
8778
|
+
for (const [providerId, failure] of failedProviders) {
|
|
8779
|
+
this.log({
|
|
8780
|
+
level: "warn",
|
|
8781
|
+
message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
|
|
8782
|
+
conversation_id: conversationId,
|
|
8783
|
+
message_id: evidentMessageId
|
|
8784
|
+
});
|
|
8785
|
+
await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
|
|
8786
|
+
}
|
|
8787
|
+
for (const providerId of succeededProviders) {
|
|
8788
|
+
if (failedProviders.has(providerId)) continue;
|
|
8789
|
+
await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
|
|
8790
|
+
}
|
|
8791
|
+
}
|
|
7808
8792
|
/**
|
|
7809
8793
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
7810
8794
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -8006,7 +8990,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
8006
8990
|
}
|
|
8007
8991
|
if (!ctx.interactive) {
|
|
8008
8992
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
8009
|
-
const proc = await startOpenCode(ctx.port);
|
|
8993
|
+
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
8010
8994
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
8011
8995
|
if (!health.healthy) {
|
|
8012
8996
|
return {
|
|
@@ -8074,7 +9058,7 @@ Port ${port} is already in use.`));
|
|
|
8074
9058
|
}
|
|
8075
9059
|
if (action === "start") {
|
|
8076
9060
|
const spinner = ora2("Starting OpenCode...").start();
|
|
8077
|
-
const proc = await startOpenCode(port);
|
|
9061
|
+
const proc = await startOpenCode(port, { inheritStdio: ctx.inheritStdio });
|
|
8078
9062
|
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
8079
9063
|
if (!health.healthy) {
|
|
8080
9064
|
spinner.fail("Failed to start OpenCode");
|
|
@@ -8086,6 +9070,551 @@ Port ${port} is already in use.`));
|
|
|
8086
9070
|
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
8087
9071
|
}
|
|
8088
9072
|
|
|
9073
|
+
// src/lib/runner-credentials.ts
|
|
9074
|
+
import { chmodSync as chmodSync2, writeFileSync as writeFileSync4 } from "fs";
|
|
9075
|
+
import { spawn as spawn5 } from "child_process";
|
|
9076
|
+
var RUNNER_SECRET_FETCH_TIMEOUT_MS = 6e4;
|
|
9077
|
+
var CREDENTIAL_RESTORE_TIMEOUT_MS = 6e4;
|
|
9078
|
+
var GITHUB_PROBE_TIMEOUT_MS = 1e4;
|
|
9079
|
+
var GIT_CONFIG_GLOBAL = "/tmp/gitconfig";
|
|
9080
|
+
var GIT_CREDENTIAL_HELPER = "/tmp/git-credential-helper.sh";
|
|
9081
|
+
function commandError2(result) {
|
|
9082
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
9083
|
+
}
|
|
9084
|
+
var runCommand2 = (command, args, opts) => {
|
|
9085
|
+
return new Promise((resolve4) => {
|
|
9086
|
+
let child;
|
|
9087
|
+
let stdout = "";
|
|
9088
|
+
let stderr = "";
|
|
9089
|
+
let settled = false;
|
|
9090
|
+
const timer = {};
|
|
9091
|
+
const finish = (result) => {
|
|
9092
|
+
if (settled) return;
|
|
9093
|
+
settled = true;
|
|
9094
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
9095
|
+
resolve4(result);
|
|
9096
|
+
};
|
|
9097
|
+
try {
|
|
9098
|
+
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
9099
|
+
} catch (error2) {
|
|
9100
|
+
finish({
|
|
9101
|
+
code: null,
|
|
9102
|
+
stdout,
|
|
9103
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
9104
|
+
timedOut: false
|
|
9105
|
+
});
|
|
9106
|
+
return;
|
|
9107
|
+
}
|
|
9108
|
+
child.stdout?.setEncoding("utf8");
|
|
9109
|
+
child.stdout?.on("data", (chunk) => {
|
|
9110
|
+
stdout += chunk;
|
|
9111
|
+
});
|
|
9112
|
+
child.stderr?.setEncoding("utf8");
|
|
9113
|
+
child.stderr?.on("data", (chunk) => {
|
|
9114
|
+
stderr += chunk;
|
|
9115
|
+
});
|
|
9116
|
+
child.once("error", (error2) => {
|
|
9117
|
+
finish({
|
|
9118
|
+
code: null,
|
|
9119
|
+
stdout,
|
|
9120
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
9121
|
+
${error2.message}`,
|
|
9122
|
+
timedOut: false
|
|
9123
|
+
});
|
|
9124
|
+
});
|
|
9125
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
9126
|
+
timer.handle = setTimeout(
|
|
9127
|
+
() => {
|
|
9128
|
+
child.kill("SIGKILL");
|
|
9129
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
9130
|
+
},
|
|
9131
|
+
Math.max(0, opts.timeoutMs)
|
|
9132
|
+
);
|
|
9133
|
+
});
|
|
9134
|
+
};
|
|
9135
|
+
function isEnvironmentObject(value) {
|
|
9136
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9137
|
+
}
|
|
9138
|
+
function secretFailure(marker, detail, log3) {
|
|
9139
|
+
const message = `${marker}: ${detail}`;
|
|
9140
|
+
log3(message, "error");
|
|
9141
|
+
return new Error(message);
|
|
9142
|
+
}
|
|
9143
|
+
async function installRunnerSecret({
|
|
9144
|
+
env,
|
|
9145
|
+
log: log3,
|
|
9146
|
+
commandRunner
|
|
9147
|
+
}) {
|
|
9148
|
+
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
9149
|
+
if (!arn) {
|
|
9150
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
9151
|
+
return false;
|
|
9152
|
+
}
|
|
9153
|
+
const result = await (commandRunner ?? runCommand2)(
|
|
9154
|
+
"aws",
|
|
9155
|
+
[
|
|
9156
|
+
"secretsmanager",
|
|
9157
|
+
"get-secret-value",
|
|
9158
|
+
"--secret-id",
|
|
9159
|
+
arn,
|
|
9160
|
+
"--query",
|
|
9161
|
+
"SecretString",
|
|
9162
|
+
"--output",
|
|
9163
|
+
"text"
|
|
9164
|
+
],
|
|
9165
|
+
{ env, timeoutMs: RUNNER_SECRET_FETCH_TIMEOUT_MS }
|
|
9166
|
+
);
|
|
9167
|
+
if (result.timedOut) {
|
|
9168
|
+
throw secretFailure(
|
|
9169
|
+
"CREDENTIAL-RESTORE-TIMEOUT",
|
|
9170
|
+
`runner-secret did not finish within ${RUNNER_SECRET_FETCH_TIMEOUT_MS}ms`,
|
|
9171
|
+
log3
|
|
9172
|
+
);
|
|
9173
|
+
}
|
|
9174
|
+
if (result.code !== 0) {
|
|
9175
|
+
throw secretFailure("RUNNER-SECRET-UNREADABLE", commandError2(result), log3);
|
|
9176
|
+
}
|
|
9177
|
+
let payload;
|
|
9178
|
+
try {
|
|
9179
|
+
payload = JSON.parse(result.stdout);
|
|
9180
|
+
} catch (error2) {
|
|
9181
|
+
log3(
|
|
9182
|
+
`RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9183
|
+
"warn"
|
|
9184
|
+
);
|
|
9185
|
+
return false;
|
|
9186
|
+
}
|
|
9187
|
+
if (!isEnvironmentObject(payload)) {
|
|
9188
|
+
log3("RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object", "warn");
|
|
9189
|
+
return false;
|
|
9190
|
+
}
|
|
9191
|
+
let populated = 0;
|
|
9192
|
+
let skipped = 0;
|
|
9193
|
+
let githubTokenPopulated = false;
|
|
9194
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
9195
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
9196
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
9197
|
+
log3(
|
|
9198
|
+
`RUNNER-SECRET-KEY-SKIPPED: ${JSON.stringify(key)} is not a valid environment variable name`,
|
|
9199
|
+
"warn"
|
|
9200
|
+
);
|
|
9201
|
+
skipped += 1;
|
|
9202
|
+
continue;
|
|
9203
|
+
}
|
|
9204
|
+
env[key] = value;
|
|
9205
|
+
populated += 1;
|
|
9206
|
+
if (key === "GH_TOKEN") githubTokenPopulated = true;
|
|
9207
|
+
}
|
|
9208
|
+
if (populated === 0) {
|
|
9209
|
+
log3(
|
|
9210
|
+
"RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md",
|
|
9211
|
+
"warn"
|
|
9212
|
+
);
|
|
9213
|
+
} else {
|
|
9214
|
+
log3(
|
|
9215
|
+
`RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names`
|
|
9216
|
+
);
|
|
9217
|
+
}
|
|
9218
|
+
return githubTokenPopulated;
|
|
9219
|
+
}
|
|
9220
|
+
function restoreFailure(operation, result, log3) {
|
|
9221
|
+
const message = `CREDENTIAL-RESTORE-FAILED: ${operation} failed: ${commandError2(result)}`;
|
|
9222
|
+
log3(message, "error");
|
|
9223
|
+
return new Error(message);
|
|
9224
|
+
}
|
|
9225
|
+
async function runCredentialRestore(args, operation, log3, synchroniserRunner) {
|
|
9226
|
+
const result = await synchroniserRunner(args, { timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS });
|
|
9227
|
+
if (result.timedOut) {
|
|
9228
|
+
log3(
|
|
9229
|
+
`CREDENTIAL-RESTORE-TIMEOUT: ${operation} did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9230
|
+
"warn"
|
|
9231
|
+
);
|
|
9232
|
+
return result;
|
|
9233
|
+
}
|
|
9234
|
+
if (result.code !== 0) throw restoreFailure(operation, result, log3);
|
|
9235
|
+
return result;
|
|
9236
|
+
}
|
|
9237
|
+
async function restoreCredentialStores({
|
|
9238
|
+
env,
|
|
9239
|
+
log: log3,
|
|
9240
|
+
synchroniserRunner = runSynchroniser
|
|
9241
|
+
}) {
|
|
9242
|
+
await runCredentialRestore(["restore", "claude"], "restore-claude", log3, synchroniserRunner);
|
|
9243
|
+
await runCredentialRestore(["restore", "opencode"], "restore-opencode", log3, synchroniserRunner);
|
|
9244
|
+
const result = await synchroniserRunner(["model-auth-ready"], {
|
|
9245
|
+
timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS
|
|
9246
|
+
});
|
|
9247
|
+
if (result.timedOut) {
|
|
9248
|
+
log3(
|
|
9249
|
+
`CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9250
|
+
"warn"
|
|
9251
|
+
);
|
|
9252
|
+
return;
|
|
9253
|
+
}
|
|
9254
|
+
switch (result.code) {
|
|
9255
|
+
case 0:
|
|
9256
|
+
return;
|
|
9257
|
+
case 10:
|
|
9258
|
+
log3(
|
|
9259
|
+
`no model credentials under s3://${env.LITESTREAM_BUCKET ?? ""}/${env.LITESTREAM_PREFIX ?? ""}/ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs a model provider fails until one is connected. See 'Seeding a credential store' in infrastructure/evident-runner/MICROVM.md.`,
|
|
9260
|
+
"warn"
|
|
9261
|
+
);
|
|
9262
|
+
return;
|
|
9263
|
+
default:
|
|
9264
|
+
log3("could not determine whether this VM has model credentials", "warn");
|
|
9265
|
+
}
|
|
9266
|
+
}
|
|
9267
|
+
var GIT_CREDENTIAL_HELPER_CONTENT = [
|
|
9268
|
+
"#!/usr/bin/env bash",
|
|
9269
|
+
'[ "$1" = get ] || exit 0',
|
|
9270
|
+
"echo username=x-access-token",
|
|
9271
|
+
'echo "password=${GH_TOKEN}"',
|
|
9272
|
+
""
|
|
9273
|
+
].join("\n");
|
|
9274
|
+
async function probeGitHubAccess({ env, log: log3 }) {
|
|
9275
|
+
const auth = await runCommand2("gh", ["api", "user", "--jq", ".login"], {
|
|
9276
|
+
env,
|
|
9277
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9278
|
+
});
|
|
9279
|
+
if (auth.timedOut) {
|
|
9280
|
+
log3(`GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`, "warn");
|
|
9281
|
+
return;
|
|
9282
|
+
}
|
|
9283
|
+
if (auth.code !== 0) {
|
|
9284
|
+
log3(`GITHUB-AUTH-REJECTED: ${commandError2(auth)}`, "warn");
|
|
9285
|
+
return;
|
|
9286
|
+
}
|
|
9287
|
+
log3(`GITHUB-AUTH-OK: ${auth.stdout.trim()}`, "debug");
|
|
9288
|
+
const remote = await runCommand2("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
|
9289
|
+
env,
|
|
9290
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9291
|
+
});
|
|
9292
|
+
if (remote.code !== 0 || remote.timedOut) return;
|
|
9293
|
+
const repo = remote.stdout.trim().replace(/^(?:https:\/\/github\.com\/|git@github\.com:)/, "").replace(/\.git$/, "");
|
|
9294
|
+
if (!repo) return;
|
|
9295
|
+
const repository = await runCommand2("gh", ["api", `repos/${repo}`, "--jq", ".full_name"], {
|
|
9296
|
+
env,
|
|
9297
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9298
|
+
});
|
|
9299
|
+
if (repository.timedOut) {
|
|
9300
|
+
log3(
|
|
9301
|
+
`GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`,
|
|
9302
|
+
"warn"
|
|
9303
|
+
);
|
|
9304
|
+
} else if (repository.code !== 0) {
|
|
9305
|
+
log3(`GITHUB-REPO-INACCESSIBLE: ${repo}: ${commandError2(repository)}`, "warn");
|
|
9306
|
+
}
|
|
9307
|
+
}
|
|
9308
|
+
async function configureGitHubAccess({ env, log: log3 }) {
|
|
9309
|
+
if (!env.GH_TOKEN) {
|
|
9310
|
+
log3("GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above", "warn");
|
|
9311
|
+
return;
|
|
9312
|
+
}
|
|
9313
|
+
try {
|
|
9314
|
+
env.GIT_CONFIG_GLOBAL = GIT_CONFIG_GLOBAL;
|
|
9315
|
+
writeFileSync4(GIT_CONFIG_GLOBAL, "");
|
|
9316
|
+
writeFileSync4(GIT_CREDENTIAL_HELPER, GIT_CREDENTIAL_HELPER_CONTENT, { mode: 448 });
|
|
9317
|
+
chmodSync2(GIT_CREDENTIAL_HELPER, 448);
|
|
9318
|
+
const config = [
|
|
9319
|
+
["user.name", env.GIT_USER_NAME ?? "evident-bot"],
|
|
9320
|
+
["user.email", env.GIT_USER_EMAIL ?? "evident-bot@users.noreply.github.com"],
|
|
9321
|
+
["init.defaultBranch", "main"],
|
|
9322
|
+
["credential.https://github.com.helper", GIT_CREDENTIAL_HELPER]
|
|
9323
|
+
];
|
|
9324
|
+
for (const [key, value] of config) {
|
|
9325
|
+
const result = await runCommand2("git", ["config", "--global", key, value], {
|
|
9326
|
+
env,
|
|
9327
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9328
|
+
});
|
|
9329
|
+
if (result.timedOut || result.code !== 0) throw new Error(commandError2(result));
|
|
9330
|
+
}
|
|
9331
|
+
} catch (error2) {
|
|
9332
|
+
log3(
|
|
9333
|
+
`GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9334
|
+
"warn"
|
|
9335
|
+
);
|
|
9336
|
+
return;
|
|
9337
|
+
}
|
|
9338
|
+
void probeGitHubAccess({ env, log: log3 }).catch((error2) => {
|
|
9339
|
+
log3(
|
|
9340
|
+
`GITHUB-SETUP-FAILED: GitHub access probe failed: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9341
|
+
"warn"
|
|
9342
|
+
);
|
|
9343
|
+
});
|
|
9344
|
+
}
|
|
9345
|
+
|
|
9346
|
+
// src/lib/opencode/config-overlay.ts
|
|
9347
|
+
import { execFileSync as execFileSync2 } from "child_process";
|
|
9348
|
+
import { copyFileSync, existsSync as existsSync2, statSync as statSync5 } from "fs";
|
|
9349
|
+
import { isAbsolute as isAbsolute2, join as join8, resolve as resolve3 } from "path";
|
|
9350
|
+
function isFile(filePath) {
|
|
9351
|
+
return existsSync2(filePath) && statSync5(filePath).isFile();
|
|
9352
|
+
}
|
|
9353
|
+
function applyRunnerOpenCodeConfig({
|
|
9354
|
+
overlayPath,
|
|
9355
|
+
cwd = process.cwd(),
|
|
9356
|
+
log: log3
|
|
9357
|
+
}) {
|
|
9358
|
+
if (!overlayPath) {
|
|
9359
|
+
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
9360
|
+
return;
|
|
9361
|
+
}
|
|
9362
|
+
const source = isAbsolute2(overlayPath) ? overlayPath : resolve3(cwd, overlayPath);
|
|
9363
|
+
const target = isFile(join8(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
9364
|
+
if (!isFile(source)) {
|
|
9365
|
+
log3(
|
|
9366
|
+
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
9367
|
+
"error"
|
|
9368
|
+
);
|
|
9369
|
+
return;
|
|
9370
|
+
}
|
|
9371
|
+
copyFileSync(source, join8(cwd, target));
|
|
9372
|
+
try {
|
|
9373
|
+
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
9374
|
+
stdio: "ignore"
|
|
9375
|
+
});
|
|
9376
|
+
} catch (error2) {
|
|
9377
|
+
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
9378
|
+
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
9379
|
+
}
|
|
9380
|
+
log3(`Applied runner OpenCode config ${source} to ${join8(cwd, target)}`);
|
|
9381
|
+
}
|
|
9382
|
+
|
|
9383
|
+
// src/lib/credential-sync.ts
|
|
9384
|
+
import { renameSync, writeFileSync as writeFileSync5 } from "fs";
|
|
9385
|
+
var CREDENTIAL_SYNC_TICK_TIMEOUT_MS = 3e4;
|
|
9386
|
+
var CREDENTIAL_FLUSH_DEADLINE_MS = 8e3;
|
|
9387
|
+
var CREDENTIAL_FLUSH_ABORT_GRACE_MS = 500;
|
|
9388
|
+
var DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS = 60;
|
|
9389
|
+
var STORES = ["claude", "opencode"];
|
|
9390
|
+
var MAX_FLUSH_PASSES = 2;
|
|
9391
|
+
function outcomesWith(outcome) {
|
|
9392
|
+
return { claude: outcome, opencode: outcome };
|
|
9393
|
+
}
|
|
9394
|
+
function errorMessage(error2) {
|
|
9395
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
9396
|
+
}
|
|
9397
|
+
function waitForSettlement(promise, timeoutMs) {
|
|
9398
|
+
return new Promise((resolve4) => {
|
|
9399
|
+
let settled = false;
|
|
9400
|
+
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
9401
|
+
const finish = (value) => {
|
|
9402
|
+
if (settled) return;
|
|
9403
|
+
settled = true;
|
|
9404
|
+
clearTimeout(timer);
|
|
9405
|
+
resolve4(value);
|
|
9406
|
+
};
|
|
9407
|
+
promise.then(
|
|
9408
|
+
() => finish(true),
|
|
9409
|
+
() => finish(true)
|
|
9410
|
+
);
|
|
9411
|
+
});
|
|
9412
|
+
}
|
|
9413
|
+
function writeMarker(markerPath, outcomes, log3) {
|
|
9414
|
+
const body = `${STORES.map((store) => `${store}=${outcomes[store]}`).join("\n")}
|
|
9415
|
+
`;
|
|
9416
|
+
const temporaryPath = `${markerPath}.tmp`;
|
|
9417
|
+
try {
|
|
9418
|
+
writeFileSync5(temporaryPath, body, { mode: 384 });
|
|
9419
|
+
renameSync(temporaryPath, markerPath);
|
|
9420
|
+
} catch (error2) {
|
|
9421
|
+
log3(`CREDENTIAL-FLUSH-MARKER-UNWRITABLE: ${markerPath}: ${errorMessage(error2)}`, "warn");
|
|
9422
|
+
}
|
|
9423
|
+
}
|
|
9424
|
+
function intervalSeconds(env, log3) {
|
|
9425
|
+
const raw = env.CREDS_SYNC_INTERVAL;
|
|
9426
|
+
if (raw === void 0 || /^[1-9][0-9]*$/.test(raw) && Number.isSafeInteger(Number(raw))) {
|
|
9427
|
+
return raw === void 0 ? DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS : Number(raw);
|
|
9428
|
+
}
|
|
9429
|
+
log3(
|
|
9430
|
+
`CREDENTIAL-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${raw}' is not a positive integer; using ${DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS}s`,
|
|
9431
|
+
"warn"
|
|
9432
|
+
);
|
|
9433
|
+
return DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS;
|
|
9434
|
+
}
|
|
9435
|
+
async function runFlushStore(store, env, synchroniserRunner, log3, deadlineAt) {
|
|
9436
|
+
const remainingMs = deadlineAt - Date.now();
|
|
9437
|
+
if (remainingMs <= 0) return { outcome: "timeout", orphaned: false };
|
|
9438
|
+
const controller = new AbortController();
|
|
9439
|
+
let result;
|
|
9440
|
+
let failed = false;
|
|
9441
|
+
const completion = Promise.resolve().then(
|
|
9442
|
+
() => synchroniserRunner(["sync-once", store], {
|
|
9443
|
+
timeoutMs: remainingMs,
|
|
9444
|
+
env,
|
|
9445
|
+
signal: controller.signal
|
|
9446
|
+
})
|
|
9447
|
+
).then(
|
|
9448
|
+
(value) => {
|
|
9449
|
+
result = value;
|
|
9450
|
+
},
|
|
9451
|
+
(error2) => {
|
|
9452
|
+
failed = true;
|
|
9453
|
+
log3(`CREDENTIAL-FLUSH-ERROR: ${store}: ${errorMessage(error2)}`, "warn");
|
|
9454
|
+
}
|
|
9455
|
+
);
|
|
9456
|
+
const abortTimer = setTimeout(() => controller.abort(), remainingMs);
|
|
9457
|
+
const settledBeforeDeadline = await waitForSettlement(completion, remainingMs);
|
|
9458
|
+
clearTimeout(abortTimer);
|
|
9459
|
+
if (!settledBeforeDeadline) {
|
|
9460
|
+
controller.abort();
|
|
9461
|
+
const settledAfterAbort = await waitForSettlement(completion, CREDENTIAL_FLUSH_ABORT_GRACE_MS);
|
|
9462
|
+
if (!settledAfterAbort) return { outcome: "timeout", orphaned: true };
|
|
9463
|
+
return { outcome: "timeout", orphaned: false };
|
|
9464
|
+
}
|
|
9465
|
+
if (failed || !result) return { outcome: "failed", orphaned: false };
|
|
9466
|
+
if (result.timedOut || Date.now() >= deadlineAt) {
|
|
9467
|
+
return { outcome: "timeout", orphaned: false };
|
|
9468
|
+
}
|
|
9469
|
+
return { outcome: result.code === 0 ? "ok" : "failed", orphaned: false };
|
|
9470
|
+
}
|
|
9471
|
+
function createCredentialSync({
|
|
9472
|
+
markerPath,
|
|
9473
|
+
env,
|
|
9474
|
+
log: log3,
|
|
9475
|
+
synchroniserRunner = runSynchroniser
|
|
9476
|
+
}) {
|
|
9477
|
+
const persistenceDisabled = !env.PERSISTENCE_BUCKET;
|
|
9478
|
+
let disabled = persistenceDisabled;
|
|
9479
|
+
let armed = false;
|
|
9480
|
+
let stopped = false;
|
|
9481
|
+
let timer;
|
|
9482
|
+
let inFlight;
|
|
9483
|
+
let activeTickAbort;
|
|
9484
|
+
let lastTickFailed;
|
|
9485
|
+
let flushPromise;
|
|
9486
|
+
const scheduleTick = (intervalMs, startTick2) => {
|
|
9487
|
+
if (stopped) return;
|
|
9488
|
+
timer = setTimeout(() => {
|
|
9489
|
+
timer = void 0;
|
|
9490
|
+
startTick2();
|
|
9491
|
+
}, intervalMs);
|
|
9492
|
+
};
|
|
9493
|
+
const startTick = (intervalMs) => {
|
|
9494
|
+
if (stopped) return;
|
|
9495
|
+
const controller = new AbortController();
|
|
9496
|
+
activeTickAbort = controller;
|
|
9497
|
+
const tick = (async () => {
|
|
9498
|
+
const outcomes = {
|
|
9499
|
+
claude: "failed",
|
|
9500
|
+
opencode: "failed"
|
|
9501
|
+
};
|
|
9502
|
+
for (const store of STORES) {
|
|
9503
|
+
if (controller.signal.aborted) break;
|
|
9504
|
+
try {
|
|
9505
|
+
const result = await synchroniserRunner(["sync-once", store], {
|
|
9506
|
+
timeoutMs: CREDENTIAL_SYNC_TICK_TIMEOUT_MS,
|
|
9507
|
+
env,
|
|
9508
|
+
signal: controller.signal
|
|
9509
|
+
});
|
|
9510
|
+
outcomes[store] = !result.timedOut && result.code === 0 ? "ok" : "failed";
|
|
9511
|
+
} catch (error2) {
|
|
9512
|
+
outcomes[store] = "failed";
|
|
9513
|
+
log3(`CREDENTIAL-SYNC-TICK-ERROR: ${store}: ${errorMessage(error2)}`, "debug");
|
|
9514
|
+
}
|
|
9515
|
+
}
|
|
9516
|
+
const failed = STORES.some((store) => outcomes[store] === "failed");
|
|
9517
|
+
log3(
|
|
9518
|
+
`CREDENTIAL-SYNC-TICK: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
9519
|
+
"debug"
|
|
9520
|
+
);
|
|
9521
|
+
if (failed && lastTickFailed !== true) {
|
|
9522
|
+
log3(
|
|
9523
|
+
"CREDENTIAL-SYNC-FAILED: an interval credential sync failed; retrying on the next tick",
|
|
9524
|
+
"warn"
|
|
9525
|
+
);
|
|
9526
|
+
} else if (!failed && lastTickFailed === true) {
|
|
9527
|
+
log3("CREDENTIAL-SYNC-RECOVERED: interval credential sync succeeded again", "warn");
|
|
9528
|
+
}
|
|
9529
|
+
lastTickFailed = failed;
|
|
9530
|
+
})().finally(() => {
|
|
9531
|
+
if (activeTickAbort === controller) activeTickAbort = void 0;
|
|
9532
|
+
if (inFlight === tick) inFlight = void 0;
|
|
9533
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9534
|
+
});
|
|
9535
|
+
inFlight = tick;
|
|
9536
|
+
};
|
|
9537
|
+
const performFlush = async () => {
|
|
9538
|
+
stopped = true;
|
|
9539
|
+
if (timer) {
|
|
9540
|
+
clearTimeout(timer);
|
|
9541
|
+
timer = void 0;
|
|
9542
|
+
}
|
|
9543
|
+
const deadlineAt = Date.now() + CREDENTIAL_FLUSH_DEADLINE_MS;
|
|
9544
|
+
if (inFlight) {
|
|
9545
|
+
const settled = await waitForSettlement(inFlight, CREDENTIAL_FLUSH_DEADLINE_MS);
|
|
9546
|
+
if (!settled) {
|
|
9547
|
+
activeTickAbort?.abort();
|
|
9548
|
+
const settledAfterAbort = await waitForSettlement(
|
|
9549
|
+
inFlight,
|
|
9550
|
+
CREDENTIAL_FLUSH_ABORT_GRACE_MS
|
|
9551
|
+
);
|
|
9552
|
+
if (!settledAfterAbort) {
|
|
9553
|
+
log3(
|
|
9554
|
+
"CREDENTIAL-FLUSH-ORPHANED-TICK: a sync-once child could not be proven settled; skipping the boundary flush rather than becoming a second writer",
|
|
9555
|
+
"warn"
|
|
9556
|
+
);
|
|
9557
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9558
|
+
}
|
|
9559
|
+
}
|
|
9560
|
+
}
|
|
9561
|
+
if (disabled) return { outcomes: outcomesWith("skipped"), orphaned: false };
|
|
9562
|
+
const outcomes = outcomesWith("timeout");
|
|
9563
|
+
for (const store of STORES) {
|
|
9564
|
+
const result = await runFlushStore(store, env, synchroniserRunner, log3, deadlineAt);
|
|
9565
|
+
if (result.orphaned) {
|
|
9566
|
+
log3(
|
|
9567
|
+
"CREDENTIAL-FLUSH-ORPHANED-CHILD: a flush sync-once child could not be proven settled; refusing another flush pass rather than becoming a second writer",
|
|
9568
|
+
"warn"
|
|
9569
|
+
);
|
|
9570
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9571
|
+
}
|
|
9572
|
+
outcomes[store] = result.outcome;
|
|
9573
|
+
}
|
|
9574
|
+
return { outcomes, orphaned: false };
|
|
9575
|
+
};
|
|
9576
|
+
let flushPasses = 0;
|
|
9577
|
+
let lastFlush;
|
|
9578
|
+
return {
|
|
9579
|
+
arm() {
|
|
9580
|
+
if (stopped || armed) return;
|
|
9581
|
+
armed = true;
|
|
9582
|
+
if (persistenceDisabled) {
|
|
9583
|
+
disabled = true;
|
|
9584
|
+
log3(
|
|
9585
|
+
"CREDENTIAL-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot",
|
|
9586
|
+
"warn"
|
|
9587
|
+
);
|
|
9588
|
+
return;
|
|
9589
|
+
}
|
|
9590
|
+
disabled = false;
|
|
9591
|
+
const intervalMs = intervalSeconds(env, log3) * 1e3;
|
|
9592
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9593
|
+
},
|
|
9594
|
+
async stopAndFlush(publish) {
|
|
9595
|
+
let result;
|
|
9596
|
+
const runningFlush = flushPromise;
|
|
9597
|
+
if (runningFlush) {
|
|
9598
|
+
result = await runningFlush;
|
|
9599
|
+
} else if (flushPasses >= MAX_FLUSH_PASSES || lastFlush?.orphaned) {
|
|
9600
|
+
result = lastFlush ?? { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9601
|
+
} else {
|
|
9602
|
+
flushPasses++;
|
|
9603
|
+
const currentFlush = performFlush();
|
|
9604
|
+
flushPromise = currentFlush;
|
|
9605
|
+
try {
|
|
9606
|
+
result = await currentFlush;
|
|
9607
|
+
lastFlush = result;
|
|
9608
|
+
} finally {
|
|
9609
|
+
if (flushPromise === currentFlush) flushPromise = void 0;
|
|
9610
|
+
}
|
|
9611
|
+
}
|
|
9612
|
+
if (publish) writeMarker(markerPath, result.outcomes, log3);
|
|
9613
|
+
return result.outcomes;
|
|
9614
|
+
}
|
|
9615
|
+
};
|
|
9616
|
+
}
|
|
9617
|
+
|
|
8089
9618
|
// src/commands/run.ts
|
|
8090
9619
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
8091
9620
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
@@ -8123,11 +9652,11 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
8123
9652
|
if (trimmed === "") {
|
|
8124
9653
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
8125
9654
|
}
|
|
8126
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
8127
|
-
if (!
|
|
9655
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join9(homeDir, trimmed.slice(2)) : trimmed;
|
|
9656
|
+
if (!isAbsolute3(expanded)) {
|
|
8128
9657
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
8129
9658
|
}
|
|
8130
|
-
const normalized =
|
|
9659
|
+
const normalized = resolvePath2(expanded);
|
|
8131
9660
|
if (parse(normalized).root === normalized) {
|
|
8132
9661
|
throw new Error(
|
|
8133
9662
|
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
@@ -8243,7 +9772,7 @@ function logActivity(state, entry) {
|
|
|
8243
9772
|
}
|
|
8244
9773
|
function reportSessionDbRecovery(state) {
|
|
8245
9774
|
try {
|
|
8246
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
9775
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir5(), env: process.env });
|
|
8247
9776
|
for (const record of report.records) {
|
|
8248
9777
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
8249
9778
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -8261,6 +9790,16 @@ function reportSessionDbRecovery(state) {
|
|
|
8261
9790
|
);
|
|
8262
9791
|
}
|
|
8263
9792
|
}
|
|
9793
|
+
function reportSessionDbRecoveryRecord(state, record) {
|
|
9794
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9795
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9796
|
+
logActivity(state, {
|
|
9797
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9798
|
+
level: activity.level,
|
|
9799
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9800
|
+
metadata: activity.metadata
|
|
9801
|
+
});
|
|
9802
|
+
}
|
|
8264
9803
|
function displayStatus(state) {
|
|
8265
9804
|
if (!state.interactive) return;
|
|
8266
9805
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
@@ -8373,6 +9912,10 @@ async function driveChannels(state, driver) {
|
|
|
8373
9912
|
consecutiveDrainFailures = 0;
|
|
8374
9913
|
unreachableMs = 0;
|
|
8375
9914
|
state.messageCount += processed;
|
|
9915
|
+
if (driver.recycleRequested) {
|
|
9916
|
+
await beginGracefulShutdown(state, "recycle");
|
|
9917
|
+
return;
|
|
9918
|
+
}
|
|
8376
9919
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
8377
9920
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
8378
9921
|
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
@@ -8415,8 +9958,8 @@ async function driveChannels(state, driver) {
|
|
|
8415
9958
|
state.running = false;
|
|
8416
9959
|
break;
|
|
8417
9960
|
}
|
|
8418
|
-
const
|
|
8419
|
-
logActivity(state, { type: "error", error: `Channel processing error: ${
|
|
9961
|
+
const errorMessage2 = error2 instanceof Error ? error2.message : String(error2);
|
|
9962
|
+
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage2}` });
|
|
8420
9963
|
if (state.interactive) displayStatus(state);
|
|
8421
9964
|
if (driver.hasInFlightWatchers()) {
|
|
8422
9965
|
consecutiveDrainFailures = 0;
|
|
@@ -8433,7 +9976,7 @@ async function driveChannels(state, driver) {
|
|
|
8433
9976
|
}
|
|
8434
9977
|
}
|
|
8435
9978
|
}
|
|
8436
|
-
await new Promise((
|
|
9979
|
+
await new Promise((resolve4) => setTimeout(resolve4, CHANNEL_POLL_INTERVAL_MS));
|
|
8437
9980
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
8438
9981
|
if (idleThisCycle) idleMs += cycleMs;
|
|
8439
9982
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -8456,7 +9999,43 @@ async function driveChannels(state, driver) {
|
|
|
8456
9999
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
8457
10000
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
8458
10001
|
function sessionDbPath() {
|
|
8459
|
-
return
|
|
10002
|
+
return join9(homedir5(), ".local", "share", "opencode", "opencode.db");
|
|
10003
|
+
}
|
|
10004
|
+
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
10005
|
+
const record = {
|
|
10006
|
+
v: 1,
|
|
10007
|
+
event: "session_db_recovery",
|
|
10008
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10009
|
+
stage: "verify",
|
|
10010
|
+
outcome: "schema_provenance_mismatch",
|
|
10011
|
+
severity: "error",
|
|
10012
|
+
reason: provenance.reason ?? "schema-provenance-mismatch",
|
|
10013
|
+
litestream_exit_code: null,
|
|
10014
|
+
attempt: null,
|
|
10015
|
+
replica_objects: null,
|
|
10016
|
+
replica_bytes: null,
|
|
10017
|
+
quarantine_destination: null,
|
|
10018
|
+
quarantined_objects: null,
|
|
10019
|
+
quarantine_failed_objects: null,
|
|
10020
|
+
quarantined_bytes: null,
|
|
10021
|
+
verified_restore_point: null,
|
|
10022
|
+
restore_points_tried: null,
|
|
10023
|
+
provenance_reason: provenance.reason,
|
|
10024
|
+
provenance_migration_delta: provenance.migrationDelta,
|
|
10025
|
+
replication_suspended: false,
|
|
10026
|
+
dbPath: sessionDbPath(),
|
|
10027
|
+
recorded_version: provenance.recordedVersion,
|
|
10028
|
+
current_version: currentVersion,
|
|
10029
|
+
provenance_pre_boot_migration_count: preBootMigrationCount
|
|
10030
|
+
};
|
|
10031
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10032
|
+
if (!activity) throw new Error("could not map session-DB provenance activity");
|
|
10033
|
+
logActivity(state, {
|
|
10034
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10035
|
+
level: activity.level,
|
|
10036
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10037
|
+
metadata: activity.metadata
|
|
10038
|
+
});
|
|
8460
10039
|
}
|
|
8461
10040
|
async function runSweep(state, driver, config) {
|
|
8462
10041
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -8503,7 +10082,7 @@ async function runSweep(state, driver, config) {
|
|
|
8503
10082
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
8504
10083
|
dbPath: sessionDbPath(),
|
|
8505
10084
|
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
8506
|
-
allowFullVacuum: protectedNow.size === 0
|
|
10085
|
+
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
8507
10086
|
});
|
|
8508
10087
|
if (reclaimResult.ok) {
|
|
8509
10088
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
@@ -8539,7 +10118,7 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
8539
10118
|
for (const warning2 of config.warnings) {
|
|
8540
10119
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
8541
10120
|
}
|
|
8542
|
-
const dbBytes = statSessionDbBytes(
|
|
10121
|
+
const dbBytes = statSessionDbBytes(homedir5());
|
|
8543
10122
|
void (async () => {
|
|
8544
10123
|
const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
8545
10124
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -8740,7 +10319,8 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
8740
10319
|
});
|
|
8741
10320
|
return;
|
|
8742
10321
|
}
|
|
8743
|
-
const collect = createResourceUsageCollector(
|
|
10322
|
+
const { collect, stop } = createResourceUsageCollector(homedir5());
|
|
10323
|
+
state.stopResourceUsageSampling = stop;
|
|
8744
10324
|
let consecutiveFailures = 0;
|
|
8745
10325
|
const tick = async () => {
|
|
8746
10326
|
try {
|
|
@@ -8850,21 +10430,41 @@ async function cleanup(state, opts = {}) {
|
|
|
8850
10430
|
clearTimeout(state.resourceUsageTimer);
|
|
8851
10431
|
state.resourceUsageTimer = null;
|
|
8852
10432
|
}
|
|
10433
|
+
state.stopResourceUsageSampling?.();
|
|
10434
|
+
state.stopResourceUsageSampling = null;
|
|
10435
|
+
const credentialSync = state.credentialSync;
|
|
10436
|
+
const flushCredentials = credentialSync ? async (phase, publish) => {
|
|
10437
|
+
await timeShutdownPhase(state, durations, phase, async () => {
|
|
10438
|
+
const outcomes = await credentialSync.stopAndFlush(publish);
|
|
10439
|
+
const level = Object.values(outcomes).every((outcome) => outcome === "ok") ? "info" : "warn";
|
|
10440
|
+
log2(
|
|
10441
|
+
state,
|
|
10442
|
+
`Credential flush: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
10443
|
+
level
|
|
10444
|
+
);
|
|
10445
|
+
});
|
|
10446
|
+
} : void 0;
|
|
10447
|
+
let drainSettled = true;
|
|
8853
10448
|
if (opts.graceful && state.channelDriver) {
|
|
8854
10449
|
state.channelDriver.stop();
|
|
10450
|
+
}
|
|
10451
|
+
if (flushCredentials) {
|
|
10452
|
+
await flushCredentials("credential_flush", !(opts.graceful && state.channelDriver));
|
|
10453
|
+
}
|
|
10454
|
+
if (opts.graceful && state.channelDriver) {
|
|
8855
10455
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
8856
10456
|
if (state.interactive) {
|
|
8857
10457
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
8858
10458
|
displayStatus(state);
|
|
8859
10459
|
}
|
|
8860
10460
|
const driver = state.channelDriver;
|
|
8861
|
-
|
|
10461
|
+
drainSettled = await timeShutdownPhase(
|
|
8862
10462
|
state,
|
|
8863
10463
|
durations,
|
|
8864
10464
|
"drain",
|
|
8865
10465
|
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
8866
10466
|
);
|
|
8867
|
-
if (!
|
|
10467
|
+
if (!drainSettled) {
|
|
8868
10468
|
logActivity(state, {
|
|
8869
10469
|
type: "info",
|
|
8870
10470
|
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
@@ -8872,6 +10472,9 @@ async function cleanup(state, opts = {}) {
|
|
|
8872
10472
|
if (state.interactive) displayStatus(state);
|
|
8873
10473
|
}
|
|
8874
10474
|
}
|
|
10475
|
+
if (opts.graceful && state.channelDriver && flushCredentials && drainSettled) {
|
|
10476
|
+
await flushCredentials("credential_flush_final", true);
|
|
10477
|
+
}
|
|
8875
10478
|
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
8876
10479
|
if (state.connection) {
|
|
8877
10480
|
const connection = state.connection;
|
|
@@ -8907,13 +10510,56 @@ async function cleanup(state, opts = {}) {
|
|
|
8907
10510
|
}
|
|
8908
10511
|
return durations;
|
|
8909
10512
|
}
|
|
10513
|
+
async function beginGracefulShutdown(state, trigger) {
|
|
10514
|
+
if (state.shuttingDown) return;
|
|
10515
|
+
state.shuttingDown = true;
|
|
10516
|
+
const shutdownStartedAt = Date.now();
|
|
10517
|
+
const shutdownMessage = trigger === "recycle" ? "Recycle requested \u2014 finishing in-flight work, then exiting" : "Shutting down...";
|
|
10518
|
+
if (state.interactive) {
|
|
10519
|
+
logActivity(state, { type: "info", message: shutdownMessage });
|
|
10520
|
+
displayStatus(state);
|
|
10521
|
+
} else {
|
|
10522
|
+
log2(state, shutdownMessage);
|
|
10523
|
+
}
|
|
10524
|
+
const durations = await cleanup(state, { graceful: true });
|
|
10525
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
10526
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
10527
|
+
let timer;
|
|
10528
|
+
const flushed = shutdownTelemetry().then(
|
|
10529
|
+
() => true,
|
|
10530
|
+
(error2) => {
|
|
10531
|
+
log2(
|
|
10532
|
+
state,
|
|
10533
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10534
|
+
"warn"
|
|
10535
|
+
);
|
|
10536
|
+
return true;
|
|
10537
|
+
}
|
|
10538
|
+
);
|
|
10539
|
+
const timedOut = new Promise((resolve4) => {
|
|
10540
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
10541
|
+
});
|
|
10542
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
10543
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
10544
|
+
}
|
|
10545
|
+
clearTimeout(timer);
|
|
10546
|
+
});
|
|
10547
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
10548
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
10549
|
+
process.exit(0);
|
|
10550
|
+
}
|
|
8910
10551
|
async function run(options) {
|
|
8911
10552
|
const interactive = isInteractive(options.json);
|
|
8912
10553
|
let logLevel;
|
|
8913
10554
|
let fileSyncDirectories;
|
|
8914
10555
|
try {
|
|
8915
10556
|
logLevel = resolveLogLevel(options);
|
|
8916
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
10557
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir5());
|
|
10558
|
+
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
10559
|
+
throw new Error(
|
|
10560
|
+
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
10561
|
+
);
|
|
10562
|
+
}
|
|
8917
10563
|
} catch (error2) {
|
|
8918
10564
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8919
10565
|
if (options.json) {
|
|
@@ -8937,6 +10583,7 @@ async function run(options) {
|
|
|
8937
10583
|
connected: false,
|
|
8938
10584
|
opencodeConnected: false,
|
|
8939
10585
|
opencodeVersion: null,
|
|
10586
|
+
sessionDbProvenanceAnomaly: false,
|
|
8940
10587
|
opencodeProcess: null,
|
|
8941
10588
|
litestreamProcess: null,
|
|
8942
10589
|
connection: null,
|
|
@@ -8952,9 +10599,24 @@ async function run(options) {
|
|
|
8952
10599
|
openaiUsageTimer: null,
|
|
8953
10600
|
openaiUsageRearm: null,
|
|
8954
10601
|
resourceUsageTimer: null,
|
|
10602
|
+
stopResourceUsageSampling: null,
|
|
10603
|
+
credentialSync: null,
|
|
8955
10604
|
authHeader: ""
|
|
8956
10605
|
};
|
|
8957
10606
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
10607
|
+
if (options.credentialSyncMarker) {
|
|
10608
|
+
state.credentialSync = createCredentialSync({
|
|
10609
|
+
markerPath: options.credentialSyncMarker,
|
|
10610
|
+
env: process.env,
|
|
10611
|
+
log: (message, level = "info") => {
|
|
10612
|
+
if (level === "error") {
|
|
10613
|
+
logActivity(state, { type: "error", error: message });
|
|
10614
|
+
} else {
|
|
10615
|
+
logActivity(state, { type: "info", level, message });
|
|
10616
|
+
}
|
|
10617
|
+
}
|
|
10618
|
+
});
|
|
10619
|
+
}
|
|
8958
10620
|
if (fileSyncDirectories.length > 0) {
|
|
8959
10621
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
8960
10622
|
} else {
|
|
@@ -8980,43 +10642,7 @@ async function run(options) {
|
|
|
8980
10642
|
"warn"
|
|
8981
10643
|
);
|
|
8982
10644
|
}
|
|
8983
|
-
const handleSignal =
|
|
8984
|
-
if (state.shuttingDown) return;
|
|
8985
|
-
state.shuttingDown = true;
|
|
8986
|
-
const shutdownStartedAt = Date.now();
|
|
8987
|
-
if (state.interactive) {
|
|
8988
|
-
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
8989
|
-
displayStatus(state);
|
|
8990
|
-
} else {
|
|
8991
|
-
log2(state, "Shutting down...");
|
|
8992
|
-
}
|
|
8993
|
-
const durations = await cleanup(state, { graceful: true });
|
|
8994
|
-
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
8995
|
-
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
8996
|
-
let timer;
|
|
8997
|
-
const flushed = shutdownTelemetry().then(
|
|
8998
|
-
() => true,
|
|
8999
|
-
(error2) => {
|
|
9000
|
-
log2(
|
|
9001
|
-
state,
|
|
9002
|
-
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9003
|
-
"warn"
|
|
9004
|
-
);
|
|
9005
|
-
return true;
|
|
9006
|
-
}
|
|
9007
|
-
);
|
|
9008
|
-
const timedOut = new Promise((resolve3) => {
|
|
9009
|
-
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
9010
|
-
});
|
|
9011
|
-
if (!await Promise.race([flushed, timedOut])) {
|
|
9012
|
-
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
9013
|
-
}
|
|
9014
|
-
clearTimeout(timer);
|
|
9015
|
-
});
|
|
9016
|
-
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
9017
|
-
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
9018
|
-
process.exit(0);
|
|
9019
|
-
};
|
|
10645
|
+
const handleSignal = () => beginGracefulShutdown(state, "signal");
|
|
9020
10646
|
process.on("SIGINT", handleSignal);
|
|
9021
10647
|
process.on("SIGTERM", handleSignal);
|
|
9022
10648
|
try {
|
|
@@ -9146,7 +10772,68 @@ async function run(options) {
|
|
|
9146
10772
|
} else {
|
|
9147
10773
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
9148
10774
|
}
|
|
10775
|
+
if (options.restoreRunnerCredentials) {
|
|
10776
|
+
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
10777
|
+
const credentialContext = {
|
|
10778
|
+
env: process.env,
|
|
10779
|
+
log: (message, level = "info") => {
|
|
10780
|
+
if (level === "error") {
|
|
10781
|
+
logActivity(state, { type: "error", error: message });
|
|
10782
|
+
} else {
|
|
10783
|
+
logActivity(state, { type: "info", level, message });
|
|
10784
|
+
}
|
|
10785
|
+
}
|
|
10786
|
+
};
|
|
10787
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
10788
|
+
await restoreCredentialStores(credentialContext);
|
|
10789
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
10790
|
+
}
|
|
10791
|
+
state.credentialSync?.arm();
|
|
10792
|
+
let sessionDbVerifyFatal = false;
|
|
10793
|
+
if (!options.restoreSessionDb) {
|
|
10794
|
+
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
10795
|
+
} else {
|
|
10796
|
+
const health = await checkOpenCodeHealth(state.port);
|
|
10797
|
+
if (health.healthy) {
|
|
10798
|
+
log2(
|
|
10799
|
+
state,
|
|
10800
|
+
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
10801
|
+
"debug"
|
|
10802
|
+
);
|
|
10803
|
+
} else {
|
|
10804
|
+
const result = await restoreAndVerifySessionDb({
|
|
10805
|
+
dbPath: sessionDbPath(),
|
|
10806
|
+
litestreamConfig: options.litestreamConfig,
|
|
10807
|
+
noReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
10808
|
+
env: process.env,
|
|
10809
|
+
log: (message, level = "info") => {
|
|
10810
|
+
if (level === "error") {
|
|
10811
|
+
logActivity(state, { type: "error", error: message });
|
|
10812
|
+
} else {
|
|
10813
|
+
logActivity(state, { type: "info", level, message });
|
|
10814
|
+
}
|
|
10815
|
+
},
|
|
10816
|
+
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
10817
|
+
});
|
|
10818
|
+
sessionDbVerifyFatal = result.verifyFatal;
|
|
10819
|
+
}
|
|
10820
|
+
}
|
|
9149
10821
|
reportSessionDbRecovery(state);
|
|
10822
|
+
if (sessionDbVerifyFatal) {
|
|
10823
|
+
throw new Error(
|
|
10824
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
10825
|
+
);
|
|
10826
|
+
}
|
|
10827
|
+
applyRunnerOpenCodeConfig({
|
|
10828
|
+
overlayPath: options.opencodeConfigOverlay,
|
|
10829
|
+
log: (message, level = "info") => {
|
|
10830
|
+
if (level === "error") {
|
|
10831
|
+
logActivity(state, { type: "error", error: message });
|
|
10832
|
+
} else {
|
|
10833
|
+
logActivity(state, { type: "info", level, message });
|
|
10834
|
+
}
|
|
10835
|
+
}
|
|
10836
|
+
});
|
|
9150
10837
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
9151
10838
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
9152
10839
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
@@ -9155,6 +10842,7 @@ async function run(options) {
|
|
|
9155
10842
|
for (const warning2 of maxActiveSessionsWarnings) {
|
|
9156
10843
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
9157
10844
|
}
|
|
10845
|
+
const preBootMigrationIds = readSessionDbMigrationIds(sessionDbPath());
|
|
9158
10846
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
9159
10847
|
try {
|
|
9160
10848
|
const oc = await ensureOpenCodeRunning({
|
|
@@ -9162,11 +10850,41 @@ async function run(options) {
|
|
|
9162
10850
|
interactive: state.interactive,
|
|
9163
10851
|
agentId: state.agentId,
|
|
9164
10852
|
log: (message) => log2(state, message),
|
|
9165
|
-
startTimeoutMs: opencodeStartTimeoutMs
|
|
10853
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
10854
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
9166
10855
|
});
|
|
9167
10856
|
state.port = oc.port;
|
|
9168
|
-
state.opencodeProcess = oc.process;
|
|
10857
|
+
state.opencodeProcess = options.opencodePidFile ? null : oc.process;
|
|
9169
10858
|
state.opencodeVersion = oc.version;
|
|
10859
|
+
if (options.opencodePidFile && oc.process?.pid !== void 0) {
|
|
10860
|
+
try {
|
|
10861
|
+
writeFileSync6(options.opencodePidFile, `${oc.process.pid}
|
|
10862
|
+
`, { mode: 384 });
|
|
10863
|
+
chmodSync3(options.opencodePidFile, 384);
|
|
10864
|
+
} catch (error2) {
|
|
10865
|
+
logActivity(state, {
|
|
10866
|
+
type: "error",
|
|
10867
|
+
error: `Failed to write OpenCode pid file ${options.opencodePidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10868
|
+
});
|
|
10869
|
+
}
|
|
10870
|
+
}
|
|
10871
|
+
if (state.opencodeVersion !== null) {
|
|
10872
|
+
const provenance = checkSessionDbProvenance({
|
|
10873
|
+
dbPath: sessionDbPath(),
|
|
10874
|
+
currentVersion: state.opencodeVersion,
|
|
10875
|
+
homeDir: homedir5(),
|
|
10876
|
+
env: process.env
|
|
10877
|
+
});
|
|
10878
|
+
if (provenance.anomaly) {
|
|
10879
|
+
state.sessionDbProvenanceAnomaly = true;
|
|
10880
|
+
logSessionDbProvenanceMismatch(
|
|
10881
|
+
state,
|
|
10882
|
+
provenance,
|
|
10883
|
+
state.opencodeVersion,
|
|
10884
|
+
preBootMigrationIds?.length ?? null
|
|
10885
|
+
);
|
|
10886
|
+
}
|
|
10887
|
+
}
|
|
9170
10888
|
state.opencodeConnected = oc.notReadyReason === null;
|
|
9171
10889
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
9172
10890
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
@@ -9203,7 +10921,75 @@ async function run(options) {
|
|
|
9203
10921
|
ocSpinner?.fail(error2.message);
|
|
9204
10922
|
throw error2;
|
|
9205
10923
|
}
|
|
9206
|
-
if (options.
|
|
10924
|
+
if (options.litestreamPidFile) {
|
|
10925
|
+
if (options.sessionDbNoReplicateMarker && existsSync3(options.sessionDbNoReplicateMarker)) {
|
|
10926
|
+
log2(
|
|
10927
|
+
state,
|
|
10928
|
+
`Skipping Litestream replication because ${options.sessionDbNoReplicateMarker} marks this session database unsafe to replicate`
|
|
10929
|
+
);
|
|
10930
|
+
} else if (!options.litestreamConfig) {
|
|
10931
|
+
logActivity(state, {
|
|
10932
|
+
type: "info",
|
|
10933
|
+
level: "warn",
|
|
10934
|
+
message: "Skipping Litestream replication because no configuration file was provided"
|
|
10935
|
+
});
|
|
10936
|
+
} else {
|
|
10937
|
+
let existingPid;
|
|
10938
|
+
if (existsSync3(options.litestreamPidFile)) {
|
|
10939
|
+
try {
|
|
10940
|
+
const rawPid = readFileSync6(options.litestreamPidFile, "utf8").trim();
|
|
10941
|
+
const parsedPid = Number(rawPid);
|
|
10942
|
+
if (/^\d+$/.test(rawPid) && Number.isSafeInteger(parsedPid) && parsedPid > 0) {
|
|
10943
|
+
existingPid = parsedPid;
|
|
10944
|
+
}
|
|
10945
|
+
} catch (error2) {
|
|
10946
|
+
logActivity(state, {
|
|
10947
|
+
type: "info",
|
|
10948
|
+
level: "warn",
|
|
10949
|
+
message: `Could not read Litestream pid file ${options.litestreamPidFile}; checking for a new process: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10950
|
+
});
|
|
10951
|
+
}
|
|
10952
|
+
}
|
|
10953
|
+
if (existingPid !== void 0 && isProcessAlive(existingPid)) {
|
|
10954
|
+
log2(state, `Litestream replication is already running with pid ${existingPid}`);
|
|
10955
|
+
} else {
|
|
10956
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
10957
|
+
state.litestreamProcess = null;
|
|
10958
|
+
let failureHandled = false;
|
|
10959
|
+
const reportImageOwnedReplicationFailure = (message) => {
|
|
10960
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
10961
|
+
failureHandled = true;
|
|
10962
|
+
logActivity(state, { type: "error", error: message });
|
|
10963
|
+
if (state.interactive) displayStatus(state);
|
|
10964
|
+
};
|
|
10965
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
10966
|
+
reportImageOwnedReplicationFailure(
|
|
10967
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
10968
|
+
);
|
|
10969
|
+
});
|
|
10970
|
+
litestreamProcess.on("error", (error2) => {
|
|
10971
|
+
reportImageOwnedReplicationFailure(
|
|
10972
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10973
|
+
);
|
|
10974
|
+
});
|
|
10975
|
+
try {
|
|
10976
|
+
if (litestreamProcess.pid !== void 0) {
|
|
10977
|
+
writeFileSync6(options.litestreamPidFile, `${litestreamProcess.pid}
|
|
10978
|
+
`, {
|
|
10979
|
+
mode: 384
|
|
10980
|
+
});
|
|
10981
|
+
chmodSync3(options.litestreamPidFile, 384);
|
|
10982
|
+
}
|
|
10983
|
+
} catch (error2) {
|
|
10984
|
+
logActivity(state, {
|
|
10985
|
+
type: "error",
|
|
10986
|
+
error: `Failed to write Litestream pid file ${options.litestreamPidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10987
|
+
});
|
|
10988
|
+
}
|
|
10989
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
10990
|
+
}
|
|
10991
|
+
}
|
|
10992
|
+
} else if (options.litestreamConfig) {
|
|
9207
10993
|
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
9208
10994
|
state.litestreamProcess = litestreamProcess;
|
|
9209
10995
|
let failureHandled = false;
|
|
@@ -9248,7 +11034,7 @@ async function run(options) {
|
|
|
9248
11034
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
9249
11035
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
9250
11036
|
fileSyncDirectories,
|
|
9251
|
-
homeDir:
|
|
11037
|
+
homeDir: homedir5(),
|
|
9252
11038
|
maxActiveSessions,
|
|
9253
11039
|
log: (entry) => (
|
|
9254
11040
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -9374,6 +11160,18 @@ async function run(options) {
|
|
|
9374
11160
|
if (state.interactive) displayStatus(state);
|
|
9375
11161
|
});
|
|
9376
11162
|
},
|
|
11163
|
+
// Both loops are rearmed because `rearm()` is idempotent for the
|
|
11164
|
+
// provider that did not just connect, and is a no-op when reporting is off.
|
|
11165
|
+
onUsageRearmPing: () => {
|
|
11166
|
+
if (!state.running) return;
|
|
11167
|
+
logActivity(state, {
|
|
11168
|
+
type: "info",
|
|
11169
|
+
level: "debug",
|
|
11170
|
+
message: "Usage rearm ping received"
|
|
11171
|
+
});
|
|
11172
|
+
state.claudeUsageRearm?.();
|
|
11173
|
+
state.openaiUsageRearm?.();
|
|
11174
|
+
},
|
|
9377
11175
|
onInfo: (message) => logActivity(state, { type: "info", message })
|
|
9378
11176
|
}
|
|
9379
11177
|
});
|
|
@@ -9394,7 +11192,17 @@ async function run(options) {
|
|
|
9394
11192
|
setTimer: (timer) => {
|
|
9395
11193
|
state.openaiUsageTimer = timer;
|
|
9396
11194
|
},
|
|
9397
|
-
fetchUsage: () =>
|
|
11195
|
+
fetchUsage: async () => {
|
|
11196
|
+
const usage = await getOpenAiUsage(state.port);
|
|
11197
|
+
if (usage.subscription === null) {
|
|
11198
|
+
logActivity(state, {
|
|
11199
|
+
type: "info",
|
|
11200
|
+
level: "debug",
|
|
11201
|
+
message: "OpenAI usage subscription could not be identified from the local credential"
|
|
11202
|
+
});
|
|
11203
|
+
}
|
|
11204
|
+
return usage;
|
|
11205
|
+
},
|
|
9398
11206
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
9399
11207
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
9400
11208
|
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
@@ -9440,7 +11248,7 @@ async function run(options) {
|
|
|
9440
11248
|
}
|
|
9441
11249
|
|
|
9442
11250
|
// src/index.ts
|
|
9443
|
-
var { version } =
|
|
11251
|
+
var { version } = createRequire2(import.meta.url)("../package.json");
|
|
9444
11252
|
var program = new Command();
|
|
9445
11253
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
9446
11254
|
"--endpoint <url>",
|
|
@@ -9500,6 +11308,27 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9500
11308
|
).option(
|
|
9501
11309
|
"--litestream-config <path>",
|
|
9502
11310
|
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
11311
|
+
).option(
|
|
11312
|
+
"--opencode-pid-file <path>",
|
|
11313
|
+
"Record the OpenCode pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11314
|
+
).option(
|
|
11315
|
+
"--litestream-pid-file <path>",
|
|
11316
|
+
"Record the Litestream pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11317
|
+
).option(
|
|
11318
|
+
"--session-db-no-replicate-marker <path>",
|
|
11319
|
+
"Read this marker as a gate before starting Litestream; the session-DB restore writes it when this boot's database is not safe to replicate."
|
|
11320
|
+
).option(
|
|
11321
|
+
"--restore-session-db",
|
|
11322
|
+
"Restore and verify the OpenCode session database before starting OpenCode; requires --litestream-config for the replica and --session-db-no-replicate-marker for give-up records."
|
|
11323
|
+
).option(
|
|
11324
|
+
"--restore-runner-credentials",
|
|
11325
|
+
"Restore the hosted runner secret and persisted credential stores before starting OpenCode."
|
|
11326
|
+
).option(
|
|
11327
|
+
"--opencode-config-overlay <path>",
|
|
11328
|
+
"Apply this runner-provided OpenCode config before starting OpenCode."
|
|
11329
|
+
).option(
|
|
11330
|
+
"--credential-sync-marker <path>",
|
|
11331
|
+
"Own the interval credential sync and write this marker once the shutdown flush has finished, so the runner image's lifecycle hooks can wait on it."
|
|
9503
11332
|
).action(
|
|
9504
11333
|
(options) => {
|
|
9505
11334
|
run({
|
|
@@ -9532,7 +11361,14 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9532
11361
|
// resolveFileSyncDirectories.
|
|
9533
11362
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
9534
11363
|
tunnelReadyFile: options.tunnelReadyFile,
|
|
9535
|
-
litestreamConfig: options.litestreamConfig
|
|
11364
|
+
litestreamConfig: options.litestreamConfig,
|
|
11365
|
+
opencodePidFile: options.opencodePidFile,
|
|
11366
|
+
litestreamPidFile: options.litestreamPidFile,
|
|
11367
|
+
sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
11368
|
+
restoreSessionDb: options.restoreSessionDb,
|
|
11369
|
+
restoreRunnerCredentials: options.restoreRunnerCredentials,
|
|
11370
|
+
opencodeConfigOverlay: options.opencodeConfigOverlay,
|
|
11371
|
+
credentialSyncMarker: options.credentialSyncMarker
|
|
9536
11372
|
});
|
|
9537
11373
|
}
|
|
9538
11374
|
);
|