@evident-ai/cli 3.4.1-dev.c6e7cf0 → 3.4.1-dev.d2c12e9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/index.js +2234 -209
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire } from "module";
|
|
4
|
+
import { createRequire as createRequire2 } from "module";
|
|
5
5
|
import { Command } from "commander";
|
|
6
6
|
|
|
7
7
|
// src/commands/login.ts
|
|
@@ -371,14 +371,14 @@ function blank() {
|
|
|
371
371
|
console.log();
|
|
372
372
|
}
|
|
373
373
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
374
|
-
return new Promise((
|
|
374
|
+
return new Promise((resolve4) => {
|
|
375
375
|
process.stdout.write(chalk.dim(prompt));
|
|
376
376
|
const handler = () => {
|
|
377
377
|
process.stdin.removeListener("data", handler);
|
|
378
378
|
process.stdin.setRawMode?.(false);
|
|
379
379
|
process.stdin.pause();
|
|
380
380
|
console.log();
|
|
381
|
-
|
|
381
|
+
resolve4();
|
|
382
382
|
};
|
|
383
383
|
if (process.stdin.isTTY) {
|
|
384
384
|
process.stdin.setRawMode?.(true);
|
|
@@ -388,7 +388,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
388
388
|
});
|
|
389
389
|
}
|
|
390
390
|
function sleep(ms) {
|
|
391
|
-
return new Promise((
|
|
391
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
392
392
|
}
|
|
393
393
|
|
|
394
394
|
// src/commands/login.ts
|
|
@@ -466,19 +466,19 @@ async function tokenLogin() {
|
|
|
466
466
|
);
|
|
467
467
|
blank();
|
|
468
468
|
process.stdout.write("Paste token: ");
|
|
469
|
-
const token = await new Promise((
|
|
469
|
+
const token = await new Promise((resolve4) => {
|
|
470
470
|
let data = "";
|
|
471
471
|
process.stdin.setEncoding("utf8");
|
|
472
472
|
process.stdin.on("data", (chunk) => {
|
|
473
473
|
data += chunk;
|
|
474
474
|
});
|
|
475
475
|
process.stdin.on("end", () => {
|
|
476
|
-
|
|
476
|
+
resolve4(data.trim());
|
|
477
477
|
});
|
|
478
478
|
if (process.stdin.isTTY) {
|
|
479
479
|
process.stdin.once("data", (chunk) => {
|
|
480
480
|
process.stdin.pause();
|
|
481
|
-
|
|
481
|
+
resolve4(chunk.toString().trim());
|
|
482
482
|
});
|
|
483
483
|
process.stdin.resume();
|
|
484
484
|
}
|
|
@@ -763,6 +763,13 @@ function toReportedOpenAiWindow(window) {
|
|
|
763
763
|
resets_at: window.resetsAt
|
|
764
764
|
};
|
|
765
765
|
}
|
|
766
|
+
function toReportedOpenAiSubscription(snapshot) {
|
|
767
|
+
if (!snapshot.subscription) return null;
|
|
768
|
+
return {
|
|
769
|
+
owner_email: snapshot.subscription.ownerEmail,
|
|
770
|
+
plan_type: snapshot.subscription.planType
|
|
771
|
+
};
|
|
772
|
+
}
|
|
766
773
|
async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
767
774
|
try {
|
|
768
775
|
const apiUrl = getApiUrlConfig();
|
|
@@ -773,7 +780,8 @@ async function reportOpenAiUsage(agentId, authHeader, snapshot) {
|
|
|
773
780
|
primary: toReportedOpenAiWindow(snapshot.primary),
|
|
774
781
|
secondary: toReportedOpenAiWindow(snapshot.secondary),
|
|
775
782
|
has_credits: snapshot.hasCredits,
|
|
776
|
-
credits_unlimited: snapshot.creditsUnlimited
|
|
783
|
+
credits_unlimited: snapshot.creditsUnlimited,
|
|
784
|
+
subscription: toReportedOpenAiSubscription(snapshot)
|
|
777
785
|
}),
|
|
778
786
|
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
779
787
|
});
|
|
@@ -797,6 +805,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
797
805
|
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
798
806
|
body: JSON.stringify({
|
|
799
807
|
cpu_percent: usage.cpuPercent,
|
|
808
|
+
cpu_peak_percent: usage.cpuPeakPercent,
|
|
800
809
|
cpu_count: usage.cpuCount,
|
|
801
810
|
memory_total_bytes: usage.memoryTotalBytes,
|
|
802
811
|
memory_available_bytes: usage.memoryAvailableBytes,
|
|
@@ -1183,8 +1192,9 @@ async function claudeUsage() {
|
|
|
1183
1192
|
}
|
|
1184
1193
|
|
|
1185
1194
|
// src/commands/run.ts
|
|
1186
|
-
import {
|
|
1187
|
-
import {
|
|
1195
|
+
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "fs";
|
|
1196
|
+
import { homedir as homedir5 } from "os";
|
|
1197
|
+
import { isAbsolute as isAbsolute3, join as join9, parse, resolve as resolvePath2 } from "path";
|
|
1188
1198
|
import chalk6 from "chalk";
|
|
1189
1199
|
|
|
1190
1200
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1206,6 +1216,7 @@ var TelemetryEventTypes = {
|
|
|
1206
1216
|
// ../../packages/types/src/tunnel/index.ts
|
|
1207
1217
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1208
1218
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1219
|
+
var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
|
|
1209
1220
|
|
|
1210
1221
|
// ../../packages/types/src/runner-files.ts
|
|
1211
1222
|
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
@@ -1524,7 +1535,14 @@ function drainSessionDbRecoveryReport({
|
|
|
1524
1535
|
skippedLines++;
|
|
1525
1536
|
return [];
|
|
1526
1537
|
}
|
|
1527
|
-
return [
|
|
1538
|
+
return [
|
|
1539
|
+
{
|
|
1540
|
+
...value,
|
|
1541
|
+
provenance_reason: value.provenance_reason ?? null,
|
|
1542
|
+
provenance_migration_delta: value.provenance_migration_delta ?? null,
|
|
1543
|
+
replication_suspended: value.replication_suspended ?? false
|
|
1544
|
+
}
|
|
1545
|
+
];
|
|
1528
1546
|
} catch (error2) {
|
|
1529
1547
|
skippedLines++;
|
|
1530
1548
|
console.error(
|
|
@@ -1549,12 +1567,39 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1549
1567
|
const level = record.severity === "warning" ? "warn" : record.severity === "error" ? "error" : null;
|
|
1550
1568
|
if (!level) return null;
|
|
1551
1569
|
const noVerifiedPoint = record.verified_restore_point ? ` The verified restore point is ${record.verified_restore_point}.` : " No verified restore point is known.";
|
|
1570
|
+
const replication = record.replication_suspended ? " This start is not backing up its new session history; restart after fixing the cause." : "";
|
|
1571
|
+
const giveupMessage = (() => {
|
|
1572
|
+
switch (record.reason) {
|
|
1573
|
+
case "restore_deadline_exceeded":
|
|
1574
|
+
return "The restore did not finish before this startup deadline. Earlier sessions are unavailable for this start; collect the SESSION-DB-RESTORE-TRUNCATED boot log before retrying.";
|
|
1575
|
+
case "restore_tool_unusable":
|
|
1576
|
+
case "classification_unrecognised":
|
|
1577
|
+
return "The restore or classification tool produced an unsupported result. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1578
|
+
case "synchroniser_config_unevaluable":
|
|
1579
|
+
case "synchroniser_config_incomplete":
|
|
1580
|
+
return "The installed hook and synchroniser bundle could not agree on configuration. Inspect the named boot marker or log and repair the runner image or bundle before restarting.";
|
|
1581
|
+
case "synchroniser_config_unresolved":
|
|
1582
|
+
return "The synchroniser configuration could not be resolved. Inspect the preceding boot error and repair the identified runner image, bundle, or local environment before restarting.";
|
|
1583
|
+
case "litestream_config_unavailable":
|
|
1584
|
+
return "The local Litestream configuration could not be generated or written. Inspect the preceding boot error and repair the runner image, bundle, local environment, or write permissions before restarting.";
|
|
1585
|
+
case "classification_fatal":
|
|
1586
|
+
return "Session history could not be restored because backup setup could not be established. Inspect the named boot marker or log and repair the identified backup configuration or permissions.";
|
|
1587
|
+
default:
|
|
1588
|
+
return null;
|
|
1589
|
+
}
|
|
1590
|
+
})();
|
|
1591
|
+
if (giveupMessage)
|
|
1592
|
+
return {
|
|
1593
|
+
level,
|
|
1594
|
+
metadata: withoutContractFields(record),
|
|
1595
|
+
message: `${giveupMessage}${replication}`
|
|
1596
|
+
};
|
|
1552
1597
|
switch (record.outcome) {
|
|
1553
1598
|
case "fresh_session_db":
|
|
1554
1599
|
return {
|
|
1555
1600
|
level,
|
|
1556
1601
|
metadata: withoutContractFields(record),
|
|
1557
|
-
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history
|
|
1602
|
+
message: `This runner started with a fresh session database. Earlier sessions are unavailable, and queued conversation or session references from before this start may no longer resolve.${noVerifiedPoint} Review the runner's backup configuration before relying on restored session history.${replication}`
|
|
1558
1603
|
};
|
|
1559
1604
|
case "restore_retried":
|
|
1560
1605
|
return {
|
|
@@ -1592,7 +1637,7 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1592
1637
|
return {
|
|
1593
1638
|
level,
|
|
1594
1639
|
metadata: withoutContractFields(record),
|
|
1595
|
-
message:
|
|
1640
|
+
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1596
1641
|
};
|
|
1597
1642
|
case "session_db_boot_refused":
|
|
1598
1643
|
return {
|
|
@@ -1600,6 +1645,12 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1600
1645
|
metadata: withoutContractFields(record),
|
|
1601
1646
|
message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
|
|
1602
1647
|
};
|
|
1648
|
+
case "schema_provenance_mismatch":
|
|
1649
|
+
return {
|
|
1650
|
+
level,
|
|
1651
|
+
metadata: withoutContractFields(record),
|
|
1652
|
+
message: `Session database schema provenance mismatch for ${record.dbPath ?? record.db_path ?? "unknown"}: recorded version=${record.recorded_version ?? "unknown"}, current version=${record.current_version ?? "unknown"}, reason=${record.provenance_reason ?? "unknown"}, migration delta=${record.provenance_migration_delta ?? "unknown"}. Inspect the session database and runner backup before continuing.`
|
|
1653
|
+
};
|
|
1603
1654
|
default:
|
|
1604
1655
|
return null;
|
|
1605
1656
|
}
|
|
@@ -1614,7 +1665,8 @@ var OUTCOMES = /* @__PURE__ */ new Set([
|
|
|
1614
1665
|
"fresh_session_db",
|
|
1615
1666
|
"history_rolled_back",
|
|
1616
1667
|
"restore_misconfigured",
|
|
1617
|
-
"session_db_boot_refused"
|
|
1668
|
+
"session_db_boot_refused",
|
|
1669
|
+
"schema_provenance_mismatch"
|
|
1618
1670
|
]);
|
|
1619
1671
|
var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
|
|
1620
1672
|
var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
|
|
@@ -1632,7 +1684,7 @@ var STRING_OR_NULL_FIELDS = ["quarantine_destination", "verified_restore_point"]
|
|
|
1632
1684
|
function isSessionDbRecoveryRecord(value) {
|
|
1633
1685
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1634
1686
|
const record = value;
|
|
1635
|
-
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1687
|
+
return record.v === 1 && record.event === "session_db_recovery" && typeof record.at === "string" && STAGES.has(record.stage) && OUTCOMES.has(record.outcome) && SEVERITIES.has(record.severity) && typeof record.reason === "string" && (record.replication_suspended === void 0 || typeof record.replication_suspended === "boolean") && (record.provenance_reason === void 0 || record.provenance_reason === null || typeof record.provenance_reason === "string") && (record.provenance_migration_delta === void 0 || record.provenance_migration_delta === null || Number.isInteger(record.provenance_migration_delta)) && NUMBER_FIELDS.every((field) => record[field] === null || Number.isInteger(record[field])) && STRING_OR_NULL_FIELDS.every(
|
|
1636
1688
|
(field) => record[field] === null || typeof record[field] === "string"
|
|
1637
1689
|
);
|
|
1638
1690
|
}
|
|
@@ -1661,11 +1713,667 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1661
1713
|
if (health.healthy) {
|
|
1662
1714
|
return health;
|
|
1663
1715
|
}
|
|
1664
|
-
await new Promise((
|
|
1716
|
+
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
1665
1717
|
}
|
|
1666
1718
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1667
1719
|
}
|
|
1668
1720
|
|
|
1721
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1722
|
+
import { spawn as spawn2 } from "child_process";
|
|
1723
|
+
import { mkdirSync, statSync as statSync2, unlinkSync as unlinkSync2, writeFileSync } from "fs";
|
|
1724
|
+
import { homedir as homedir2 } from "os";
|
|
1725
|
+
import { dirname as dirname2, resolve as resolvePath } from "path";
|
|
1726
|
+
|
|
1727
|
+
// src/lib/runner-synchroniser.ts
|
|
1728
|
+
import { spawn } from "child_process";
|
|
1729
|
+
function appendError(stderr, error2) {
|
|
1730
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1731
|
+
return stderr === "" ? message : `${stderr}
|
|
1732
|
+
${message}`;
|
|
1733
|
+
}
|
|
1734
|
+
function runSynchroniser(args, opts) {
|
|
1735
|
+
return new Promise((resolve4) => {
|
|
1736
|
+
let child;
|
|
1737
|
+
let stdout = "";
|
|
1738
|
+
let stderr = "";
|
|
1739
|
+
let settled = false;
|
|
1740
|
+
const timer = {};
|
|
1741
|
+
let abortListener;
|
|
1742
|
+
let spawnListener;
|
|
1743
|
+
const finish = (result) => {
|
|
1744
|
+
if (settled) return;
|
|
1745
|
+
settled = true;
|
|
1746
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
1747
|
+
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
1748
|
+
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
1749
|
+
resolve4(result);
|
|
1750
|
+
};
|
|
1751
|
+
try {
|
|
1752
|
+
child = spawn("runner-synchroniser", args, {
|
|
1753
|
+
env: opts.env ?? process.env,
|
|
1754
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1755
|
+
});
|
|
1756
|
+
} catch (error2) {
|
|
1757
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
child.stdout?.setEncoding("utf8");
|
|
1761
|
+
child.stdout?.on("data", (chunk) => {
|
|
1762
|
+
stdout += chunk;
|
|
1763
|
+
});
|
|
1764
|
+
child.stderr?.setEncoding("utf8");
|
|
1765
|
+
child.stderr?.on("data", (chunk) => {
|
|
1766
|
+
stderr += chunk;
|
|
1767
|
+
});
|
|
1768
|
+
child.once("error", (error2) => {
|
|
1769
|
+
finish({ code: null, stdout, stderr: appendError(stderr, error2), timedOut: false });
|
|
1770
|
+
});
|
|
1771
|
+
child.once("close", (code) => {
|
|
1772
|
+
finish({ code, stdout, stderr, timedOut: false });
|
|
1773
|
+
});
|
|
1774
|
+
if (opts.signal) {
|
|
1775
|
+
const killChild = () => {
|
|
1776
|
+
if (child.pid === void 0) {
|
|
1777
|
+
if (!spawnListener) {
|
|
1778
|
+
spawnListener = killChild;
|
|
1779
|
+
child.once("spawn", spawnListener);
|
|
1780
|
+
}
|
|
1781
|
+
return;
|
|
1782
|
+
}
|
|
1783
|
+
child.kill("SIGKILL");
|
|
1784
|
+
};
|
|
1785
|
+
abortListener = killChild;
|
|
1786
|
+
if (opts.signal.aborted) {
|
|
1787
|
+
abortListener();
|
|
1788
|
+
} else {
|
|
1789
|
+
opts.signal.addEventListener("abort", abortListener, { once: true });
|
|
1790
|
+
if (opts.signal.aborted) abortListener();
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
timer.handle = setTimeout(
|
|
1794
|
+
() => {
|
|
1795
|
+
child.kill("SIGKILL");
|
|
1796
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1797
|
+
},
|
|
1798
|
+
Math.max(0, opts.timeoutMs)
|
|
1799
|
+
);
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
// src/lib/opencode/session-db-boot.ts
|
|
1804
|
+
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
1805
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
|
|
1806
|
+
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
1807
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
|
|
1808
|
+
function commandError(result) {
|
|
1809
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
1810
|
+
}
|
|
1811
|
+
function reportRecord(stage, outcome, reason, litestreamExitCode, options) {
|
|
1812
|
+
options.reportRecovery({
|
|
1813
|
+
v: 1,
|
|
1814
|
+
event: "session_db_recovery",
|
|
1815
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1816
|
+
stage,
|
|
1817
|
+
outcome,
|
|
1818
|
+
severity: "error",
|
|
1819
|
+
reason,
|
|
1820
|
+
litestream_exit_code: litestreamExitCode,
|
|
1821
|
+
attempt: null,
|
|
1822
|
+
replica_objects: null,
|
|
1823
|
+
replica_bytes: null,
|
|
1824
|
+
quarantine_destination: null,
|
|
1825
|
+
quarantined_objects: null,
|
|
1826
|
+
quarantine_failed_objects: null,
|
|
1827
|
+
quarantined_bytes: null,
|
|
1828
|
+
verified_restore_point: null,
|
|
1829
|
+
restore_points_tried: null,
|
|
1830
|
+
provenance_reason: null,
|
|
1831
|
+
provenance_migration_delta: null,
|
|
1832
|
+
replication_suspended: stage === "restore"
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
function clearMarker(options) {
|
|
1836
|
+
if (!options.noReplicateMarker) return;
|
|
1837
|
+
try {
|
|
1838
|
+
unlinkSync2(options.noReplicateMarker);
|
|
1839
|
+
} catch (error2) {
|
|
1840
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return;
|
|
1841
|
+
options.log(
|
|
1842
|
+
`Could not clear the previous session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1843
|
+
"warn"
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
function markNoReplicate(options, message) {
|
|
1848
|
+
if (options.noReplicateMarker) {
|
|
1849
|
+
try {
|
|
1850
|
+
mkdirSync(dirname2(options.noReplicateMarker), { recursive: true });
|
|
1851
|
+
writeFileSync(options.noReplicateMarker, "");
|
|
1852
|
+
} catch (error2) {
|
|
1853
|
+
options.log(
|
|
1854
|
+
`Could not write session-DB no-replicate marker ${options.noReplicateMarker}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1855
|
+
"error"
|
|
1856
|
+
);
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
options.log(`SESSION-DB-NO-REPLICATE: ${message}`, "warn");
|
|
1860
|
+
}
|
|
1861
|
+
function discardSessionDbDebris(options) {
|
|
1862
|
+
for (const path of [options.dbPath, `${options.dbPath}-wal`, `${options.dbPath}-shm`]) {
|
|
1863
|
+
try {
|
|
1864
|
+
unlinkSync2(path);
|
|
1865
|
+
} catch (error2) {
|
|
1866
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") continue;
|
|
1867
|
+
options.log(
|
|
1868
|
+
`Could not remove session-DB debris ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1869
|
+
"warn"
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
function splitDiagnostics(text) {
|
|
1875
|
+
return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
1876
|
+
}
|
|
1877
|
+
function logSynchroniserDiagnostics(result, options) {
|
|
1878
|
+
for (const line of splitDiagnostics(result.stderr)) options.log(line, "warn");
|
|
1879
|
+
}
|
|
1880
|
+
function parseSingleQuotedAssignment(line) {
|
|
1881
|
+
const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
1882
|
+
if (!match || !match[2].startsWith("'")) return null;
|
|
1883
|
+
const valueSource = match[2];
|
|
1884
|
+
let value = "";
|
|
1885
|
+
for (let index = 1; index < valueSource.length; index++) {
|
|
1886
|
+
const character = valueSource[index];
|
|
1887
|
+
if (character !== "'") {
|
|
1888
|
+
value += character;
|
|
1889
|
+
continue;
|
|
1890
|
+
}
|
|
1891
|
+
if (index === valueSource.length - 1) return [match[1], value];
|
|
1892
|
+
if (valueSource.slice(index + 1, index + 4) !== "\\''") return null;
|
|
1893
|
+
value += "'";
|
|
1894
|
+
index += 3;
|
|
1895
|
+
}
|
|
1896
|
+
return null;
|
|
1897
|
+
}
|
|
1898
|
+
function parseSynchroniserEnv(stdout) {
|
|
1899
|
+
const values = {};
|
|
1900
|
+
for (const line of stdout.split("\n")) {
|
|
1901
|
+
if (line.trim() === "") continue;
|
|
1902
|
+
const assignment = parseSingleQuotedAssignment(line);
|
|
1903
|
+
if (!assignment) return null;
|
|
1904
|
+
values[assignment[0]] = assignment[1];
|
|
1905
|
+
}
|
|
1906
|
+
return values;
|
|
1907
|
+
}
|
|
1908
|
+
function runCommand(command, args, options) {
|
|
1909
|
+
return new Promise((resolve4) => {
|
|
1910
|
+
let child;
|
|
1911
|
+
let stdout = "";
|
|
1912
|
+
let stderr = "";
|
|
1913
|
+
let settled = false;
|
|
1914
|
+
const finish = (result) => {
|
|
1915
|
+
if (settled) return;
|
|
1916
|
+
settled = true;
|
|
1917
|
+
if (timer) clearTimeout(timer);
|
|
1918
|
+
resolve4(result);
|
|
1919
|
+
};
|
|
1920
|
+
try {
|
|
1921
|
+
child = spawn2(command, args, {
|
|
1922
|
+
env: options.env,
|
|
1923
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
1924
|
+
});
|
|
1925
|
+
} catch (error2) {
|
|
1926
|
+
resolve4({
|
|
1927
|
+
code: null,
|
|
1928
|
+
stdout,
|
|
1929
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
1930
|
+
timedOut: false
|
|
1931
|
+
});
|
|
1932
|
+
return;
|
|
1933
|
+
}
|
|
1934
|
+
child.stdout?.setEncoding("utf8");
|
|
1935
|
+
child.stdout?.on("data", (chunk) => {
|
|
1936
|
+
stdout += chunk;
|
|
1937
|
+
});
|
|
1938
|
+
child.stderr?.setEncoding("utf8");
|
|
1939
|
+
child.stderr?.on("data", (chunk) => {
|
|
1940
|
+
stderr += chunk;
|
|
1941
|
+
});
|
|
1942
|
+
child.once("error", (error2) => {
|
|
1943
|
+
finish({
|
|
1944
|
+
code: null,
|
|
1945
|
+
stdout,
|
|
1946
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
1947
|
+
${error2.message}`,
|
|
1948
|
+
timedOut: false
|
|
1949
|
+
});
|
|
1950
|
+
});
|
|
1951
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
1952
|
+
const timer = setTimeout(
|
|
1953
|
+
() => {
|
|
1954
|
+
child.kill("SIGKILL");
|
|
1955
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
1956
|
+
},
|
|
1957
|
+
Math.max(0, options.timeoutMs)
|
|
1958
|
+
);
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
async function ensureLitestreamConfig(options, env) {
|
|
1962
|
+
const configPath = options.litestreamConfig;
|
|
1963
|
+
if (!configPath) {
|
|
1964
|
+
markNoReplicate(options, "no Litestream configuration path was provided");
|
|
1965
|
+
reportRecord(
|
|
1966
|
+
"restore",
|
|
1967
|
+
"restore_misconfigured",
|
|
1968
|
+
"litestream_config_unavailable",
|
|
1969
|
+
null,
|
|
1970
|
+
options
|
|
1971
|
+
);
|
|
1972
|
+
return null;
|
|
1973
|
+
}
|
|
1974
|
+
try {
|
|
1975
|
+
if (statSync2(configPath).size > 0) return configPath;
|
|
1976
|
+
} catch (error2) {
|
|
1977
|
+
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
1978
|
+
options.log(
|
|
1979
|
+
`Could not inspect ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
1980
|
+
"warn"
|
|
1981
|
+
);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
const rendered = await runSynchroniser(["litestream-config"], {
|
|
1985
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
1986
|
+
env
|
|
1987
|
+
});
|
|
1988
|
+
logSynchroniserDiagnostics(rendered, options);
|
|
1989
|
+
if (rendered.timedOut || rendered.code !== 0) {
|
|
1990
|
+
options.log(
|
|
1991
|
+
`Could not generate ${configPath}: runner-synchroniser litestream-config failed (${commandError(rendered)})`,
|
|
1992
|
+
"error"
|
|
1993
|
+
);
|
|
1994
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
1995
|
+
reportRecord(
|
|
1996
|
+
"restore",
|
|
1997
|
+
"restore_misconfigured",
|
|
1998
|
+
"litestream_config_unavailable",
|
|
1999
|
+
null,
|
|
2000
|
+
options
|
|
2001
|
+
);
|
|
2002
|
+
return null;
|
|
2003
|
+
}
|
|
2004
|
+
try {
|
|
2005
|
+
mkdirSync(dirname2(configPath), { recursive: true });
|
|
2006
|
+
writeFileSync(configPath, rendered.stdout);
|
|
2007
|
+
} catch (error2) {
|
|
2008
|
+
options.log(
|
|
2009
|
+
`Could not write ${configPath}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
2010
|
+
"error"
|
|
2011
|
+
);
|
|
2012
|
+
markNoReplicate(options, `could not generate ${configPath}`);
|
|
2013
|
+
reportRecord(
|
|
2014
|
+
"restore",
|
|
2015
|
+
"restore_misconfigured",
|
|
2016
|
+
"litestream_config_unavailable",
|
|
2017
|
+
null,
|
|
2018
|
+
options
|
|
2019
|
+
);
|
|
2020
|
+
return null;
|
|
2021
|
+
}
|
|
2022
|
+
const version2 = await runCommand("litestream", ["version"], {
|
|
2023
|
+
env,
|
|
2024
|
+
timeoutMs: 1e4
|
|
2025
|
+
});
|
|
2026
|
+
const litestreamVersion = version2.code === 0 ? version2.stdout.trim() || "unknown" : "unknown";
|
|
2027
|
+
const regionEmpty = !/^\s*region:\s*\S+/m.test(rendered.stdout);
|
|
2028
|
+
options.log(
|
|
2029
|
+
`litestream ${litestreamVersion}; AWS_REGION=${env.AWS_REGION ?? "<unset>"} AWS_DEFAULT_REGION=${env.AWS_DEFAULT_REGION ?? "<unset>"}; rendered litestream.yml region empty: ${regionEmpty ? "yes" : "no"}`
|
|
2030
|
+
);
|
|
2031
|
+
return configPath;
|
|
2032
|
+
}
|
|
2033
|
+
function restoreGiveUp(options, reason, message, litestreamExitCode, outcome = "fresh_session_db") {
|
|
2034
|
+
discardSessionDbDebris(options);
|
|
2035
|
+
markNoReplicate(options, message);
|
|
2036
|
+
reportRecord("restore", outcome, reason, litestreamExitCode, options);
|
|
2037
|
+
}
|
|
2038
|
+
async function restoreSessionDb(options, configPath, env) {
|
|
2039
|
+
const restored = await runCommand(
|
|
2040
|
+
"litestream",
|
|
2041
|
+
["restore", "-config", configPath, "-if-db-not-exists", "-if-replica-exists", options.dbPath],
|
|
2042
|
+
{ env, timeoutMs: SESSION_DB_RESTORE_TIMEOUT_MS }
|
|
2043
|
+
);
|
|
2044
|
+
for (const line of splitDiagnostics(restored.stderr)) options.log(line, "warn");
|
|
2045
|
+
if (restored.timedOut || restored.code === 124 || restored.code === 137) {
|
|
2046
|
+
restoreGiveUp(
|
|
2047
|
+
options,
|
|
2048
|
+
"restore_deadline_exceeded",
|
|
2049
|
+
`SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within ${SESSION_DB_RESTORE_TIMEOUT_MS}ms; opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2050
|
+
restored.code ?? 124
|
|
2051
|
+
);
|
|
2052
|
+
return;
|
|
2053
|
+
}
|
|
2054
|
+
if (restored.code === null || restored.code === 125 || restored.code === 126 || restored.code === 127) {
|
|
2055
|
+
options.log(`litestream restore could not run (${commandError(restored)})`, "error");
|
|
2056
|
+
restoreGiveUp(
|
|
2057
|
+
options,
|
|
2058
|
+
"restore_tool_unusable",
|
|
2059
|
+
`restore tool is broken (${commandError(restored)}); opencode starts with a fresh session DB and nothing is replicated this boot`,
|
|
2060
|
+
restored.code
|
|
2061
|
+
);
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
const classified = await runSynchroniser(
|
|
2065
|
+
[
|
|
2066
|
+
"session-db-classify",
|
|
2067
|
+
String(restored.code ?? 1),
|
|
2068
|
+
"1",
|
|
2069
|
+
"--on-unusable-replica=leave",
|
|
2070
|
+
"--fresh-db-fallback"
|
|
2071
|
+
],
|
|
2072
|
+
{ timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS, env }
|
|
2073
|
+
);
|
|
2074
|
+
logSynchroniserDiagnostics(classified, options);
|
|
2075
|
+
const classifyCode = classified.code;
|
|
2076
|
+
switch (classifyCode) {
|
|
2077
|
+
case 0:
|
|
2078
|
+
return;
|
|
2079
|
+
case 31:
|
|
2080
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2081
|
+
options.log(
|
|
2082
|
+
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
2083
|
+
"warn"
|
|
2084
|
+
);
|
|
2085
|
+
return;
|
|
2086
|
+
case 32:
|
|
2087
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2088
|
+
discardSessionDbDebris(options);
|
|
2089
|
+
markNoReplicate(
|
|
2090
|
+
options,
|
|
2091
|
+
"session-db-classify asked for another restore attempt (32), but this boot has budget for only one; treating it as a give-up rather than retrying"
|
|
2092
|
+
);
|
|
2093
|
+
return;
|
|
2094
|
+
case 30:
|
|
2095
|
+
restoreGiveUp(
|
|
2096
|
+
options,
|
|
2097
|
+
"classification_fatal",
|
|
2098
|
+
"session-db-classify returned fatal (30); see the FATAL message above",
|
|
2099
|
+
restored.code,
|
|
2100
|
+
"restore_misconfigured"
|
|
2101
|
+
);
|
|
2102
|
+
return;
|
|
2103
|
+
default:
|
|
2104
|
+
restoreGiveUp(
|
|
2105
|
+
options,
|
|
2106
|
+
"classification_unrecognised",
|
|
2107
|
+
`session-db-classify exited ${classifyCode ?? "null"}, which is none of its documented answers`,
|
|
2108
|
+
restored.code
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
async function verifySessionDb(options, configPath, env) {
|
|
2113
|
+
if (!options.noReplicateMarker || !fileExists(options.noReplicateMarker)) {
|
|
2114
|
+
const result = await runSynchroniser(["session-db-verify", configPath], {
|
|
2115
|
+
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
2116
|
+
env: {
|
|
2117
|
+
...env,
|
|
2118
|
+
// The synchroniser reads this value in SECONDS. Keep this at 120, not
|
|
2119
|
+
// 120_000, so the walkback gives up before the outer process bound.
|
|
2120
|
+
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
2121
|
+
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
2122
|
+
)
|
|
2123
|
+
}
|
|
2124
|
+
});
|
|
2125
|
+
logSynchroniserDiagnostics(result, options);
|
|
2126
|
+
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
2127
|
+
options.log(
|
|
2128
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
2129
|
+
"warn"
|
|
2130
|
+
);
|
|
2131
|
+
return false;
|
|
2132
|
+
}
|
|
2133
|
+
if (result.code === 34) {
|
|
2134
|
+
reportRecord(
|
|
2135
|
+
"verify",
|
|
2136
|
+
"session_db_boot_refused",
|
|
2137
|
+
result.stderr.includes("SESSION-DB-LOCAL-DISCARD-FAILED") ? "local_discard_failed" : "replica_separation_unproven",
|
|
2138
|
+
null,
|
|
2139
|
+
options
|
|
2140
|
+
);
|
|
2141
|
+
return true;
|
|
2142
|
+
}
|
|
2143
|
+
if (result.code === 33) {
|
|
2144
|
+
options.log(
|
|
2145
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
2146
|
+
"warn"
|
|
2147
|
+
);
|
|
2148
|
+
return false;
|
|
2149
|
+
}
|
|
2150
|
+
if (result.code !== 0) {
|
|
2151
|
+
options.log(
|
|
2152
|
+
`SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${result.code ?? "null"}, which is none of its documented answers; continuing with the restored opencode.db as-is`,
|
|
2153
|
+
"warn"
|
|
2154
|
+
);
|
|
2155
|
+
}
|
|
2156
|
+
return false;
|
|
2157
|
+
}
|
|
2158
|
+
options.log(
|
|
2159
|
+
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
2160
|
+
"debug"
|
|
2161
|
+
);
|
|
2162
|
+
return false;
|
|
2163
|
+
}
|
|
2164
|
+
function fileExists(path) {
|
|
2165
|
+
try {
|
|
2166
|
+
statSync2(path);
|
|
2167
|
+
return true;
|
|
2168
|
+
} catch (error2) {
|
|
2169
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
2170
|
+
return true;
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
async function restoreAndVerifySessionDb(options) {
|
|
2174
|
+
const env = options.env ?? process.env;
|
|
2175
|
+
clearMarker(options);
|
|
2176
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir2(), env));
|
|
2177
|
+
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
2178
|
+
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
2179
|
+
env
|
|
2180
|
+
});
|
|
2181
|
+
logSynchroniserDiagnostics(synchroniserEnv, options);
|
|
2182
|
+
if (synchroniserEnv.timedOut || synchroniserEnv.code !== 0) {
|
|
2183
|
+
options.log(
|
|
2184
|
+
`runner-synchroniser env could not resolve the session-DB configuration (${commandError(synchroniserEnv)})`,
|
|
2185
|
+
"error"
|
|
2186
|
+
);
|
|
2187
|
+
markNoReplicate(
|
|
2188
|
+
options,
|
|
2189
|
+
"could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
2190
|
+
);
|
|
2191
|
+
reportRecord(
|
|
2192
|
+
"restore",
|
|
2193
|
+
"restore_misconfigured",
|
|
2194
|
+
"synchroniser_config_unresolved",
|
|
2195
|
+
null,
|
|
2196
|
+
options
|
|
2197
|
+
);
|
|
2198
|
+
return { verifyFatal: false };
|
|
2199
|
+
}
|
|
2200
|
+
const values = parseSynchroniserEnv(synchroniserEnv.stdout);
|
|
2201
|
+
if (!values) {
|
|
2202
|
+
markNoReplicate(
|
|
2203
|
+
options,
|
|
2204
|
+
"the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
2205
|
+
);
|
|
2206
|
+
reportRecord(
|
|
2207
|
+
"restore",
|
|
2208
|
+
"restore_misconfigured",
|
|
2209
|
+
"synchroniser_config_unevaluable",
|
|
2210
|
+
null,
|
|
2211
|
+
options
|
|
2212
|
+
);
|
|
2213
|
+
return { verifyFatal: false };
|
|
2214
|
+
}
|
|
2215
|
+
const synchroniserDbPath = values.OPENCODE_DB_PATH;
|
|
2216
|
+
if (!synchroniserDbPath) {
|
|
2217
|
+
markNoReplicate(
|
|
2218
|
+
options,
|
|
2219
|
+
"run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
2220
|
+
);
|
|
2221
|
+
reportRecord(
|
|
2222
|
+
"restore",
|
|
2223
|
+
"restore_misconfigured",
|
|
2224
|
+
"synchroniser_config_incomplete",
|
|
2225
|
+
null,
|
|
2226
|
+
options
|
|
2227
|
+
);
|
|
2228
|
+
return { verifyFatal: false };
|
|
2229
|
+
}
|
|
2230
|
+
if (resolvePath(synchroniserDbPath) !== resolvePath(options.dbPath)) {
|
|
2231
|
+
options.log(
|
|
2232
|
+
`runner-synchroniser reported OPENCODE_DB_PATH=${synchroniserDbPath}, but OpenCode uses ${options.dbPath}; continuing with OpenCode's session-DB path`,
|
|
2233
|
+
"warn"
|
|
2234
|
+
);
|
|
2235
|
+
}
|
|
2236
|
+
if (!values.PERSISTENCE_BUCKET) {
|
|
2237
|
+
options.log(
|
|
2238
|
+
"SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated.",
|
|
2239
|
+
"warn"
|
|
2240
|
+
);
|
|
2241
|
+
return { verifyFatal: false };
|
|
2242
|
+
}
|
|
2243
|
+
const configPath = await ensureLitestreamConfig(options, env);
|
|
2244
|
+
if (!configPath) return { verifyFatal: false };
|
|
2245
|
+
await restoreSessionDb(options, configPath, env);
|
|
2246
|
+
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
2247
|
+
return { verifyFatal: false };
|
|
2248
|
+
}
|
|
2249
|
+
return { verifyFatal: await verifySessionDb(options, configPath, env) };
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// src/lib/opencode/session-db-provenance.ts
|
|
2253
|
+
import { createRequire } from "module";
|
|
2254
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
2255
|
+
import { dirname as dirname3, join as join3 } from "path";
|
|
2256
|
+
var require2 = createRequire(import.meta.url);
|
|
2257
|
+
function readSessionDbMigrationIds(dbPath) {
|
|
2258
|
+
let db;
|
|
2259
|
+
try {
|
|
2260
|
+
const { DatabaseSync } = require2("node:sqlite");
|
|
2261
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
2262
|
+
const columns = db.prepare("PRAGMA table_info(migration)").all();
|
|
2263
|
+
const hasExpectedShape = columns.length === 2 && columns.some(
|
|
2264
|
+
(column) => column.name === "id" && typeof column.type === "string" && column.type.toUpperCase() === "TEXT" && column.pk === 1
|
|
2265
|
+
) && columns.some(
|
|
2266
|
+
(column) => column.name === "time_completed" && typeof column.type === "string" && column.type.toUpperCase() === "INTEGER" && column.notnull === 1 && column.pk === 0
|
|
2267
|
+
);
|
|
2268
|
+
if (!hasExpectedShape) {
|
|
2269
|
+
console.warn(`[readSessionDbMigrationIds] unsupported migration table shape in ${dbPath}`);
|
|
2270
|
+
return null;
|
|
2271
|
+
}
|
|
2272
|
+
const rows = db.prepare("SELECT id FROM migration ORDER BY id").all();
|
|
2273
|
+
if (rows.some((row) => typeof row.id !== "string")) return null;
|
|
2274
|
+
return rows.map((row) => row.id);
|
|
2275
|
+
} catch (error2) {
|
|
2276
|
+
console.warn(
|
|
2277
|
+
`[readSessionDbMigrationIds] could not read migration history from ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2278
|
+
);
|
|
2279
|
+
return null;
|
|
2280
|
+
} finally {
|
|
2281
|
+
try {
|
|
2282
|
+
db?.close();
|
|
2283
|
+
} catch (error2) {
|
|
2284
|
+
console.warn(
|
|
2285
|
+
`[readSessionDbMigrationIds] could not close ${dbPath}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2286
|
+
);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
2291
|
+
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
2292
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
2293
|
+
}
|
|
2294
|
+
function loadSessionDbProvenanceState(path) {
|
|
2295
|
+
let value;
|
|
2296
|
+
try {
|
|
2297
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
2298
|
+
} catch (error2) {
|
|
2299
|
+
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return {};
|
|
2300
|
+
console.error(
|
|
2301
|
+
`[session-db-provenance] could not read ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2302
|
+
);
|
|
2303
|
+
return {};
|
|
2304
|
+
}
|
|
2305
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2306
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2307
|
+
return {};
|
|
2308
|
+
}
|
|
2309
|
+
const state = {};
|
|
2310
|
+
for (const [dbPath, record] of Object.entries(value)) {
|
|
2311
|
+
if (!isSessionDbProvenanceRecord(record)) {
|
|
2312
|
+
console.error(`[session-db-provenance] ignored malformed state in ${path}`);
|
|
2313
|
+
return {};
|
|
2314
|
+
}
|
|
2315
|
+
state[dbPath] = record;
|
|
2316
|
+
}
|
|
2317
|
+
return state;
|
|
2318
|
+
}
|
|
2319
|
+
function saveSessionDbProvenanceState(path, state) {
|
|
2320
|
+
try {
|
|
2321
|
+
mkdirSync2(dirname3(path), { recursive: true });
|
|
2322
|
+
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
2323
|
+
`, "utf8");
|
|
2324
|
+
} catch (error2) {
|
|
2325
|
+
console.error(
|
|
2326
|
+
`[session-db-provenance] could not write ${path}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2327
|
+
);
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
function evaluateSessionDbProvenance(input) {
|
|
2331
|
+
const { currentVersion, currentIds, previous } = input;
|
|
2332
|
+
if (!previous) return { anomaly: false, reason: null };
|
|
2333
|
+
const current = new Set(currentIds);
|
|
2334
|
+
const prior = new Set(previous.migrationIds);
|
|
2335
|
+
for (const id of prior) {
|
|
2336
|
+
if (!current.has(id)) return { anomaly: true, reason: "migration-history-regressed" };
|
|
2337
|
+
}
|
|
2338
|
+
if (current.size > prior.size && previous.opencodeVersion === currentVersion) {
|
|
2339
|
+
return { anomaly: true, reason: "foreign-version-migrations" };
|
|
2340
|
+
}
|
|
2341
|
+
return { anomaly: false, reason: null };
|
|
2342
|
+
}
|
|
2343
|
+
function checkSessionDbProvenance(input) {
|
|
2344
|
+
const { dbPath, currentVersion, homeDir, env } = input;
|
|
2345
|
+
const path = sessionDbProvenanceStatePath(homeDir, env);
|
|
2346
|
+
const state = loadSessionDbProvenanceState(path);
|
|
2347
|
+
const previous = state[dbPath];
|
|
2348
|
+
const currentIds = readSessionDbMigrationIds(dbPath);
|
|
2349
|
+
if (currentIds === null) {
|
|
2350
|
+
return {
|
|
2351
|
+
anomaly: false,
|
|
2352
|
+
reason: null,
|
|
2353
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2354
|
+
migrationDelta: null
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
const decision = evaluateSessionDbProvenance({ currentVersion, currentIds, previous });
|
|
2358
|
+
const migrationDelta = previous ? new Set(currentIds).size - new Set(previous.migrationIds).size : null;
|
|
2359
|
+
state[dbPath] = {
|
|
2360
|
+
opencodeVersion: currentVersion,
|
|
2361
|
+
migrationIds: currentIds,
|
|
2362
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2363
|
+
};
|
|
2364
|
+
saveSessionDbProvenanceState(path, state);
|
|
2365
|
+
return {
|
|
2366
|
+
...decision,
|
|
2367
|
+
recordedVersion: previous?.opencodeVersion ?? null,
|
|
2368
|
+
migrationDelta
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
function isSessionDbProvenanceRecord(value) {
|
|
2372
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2373
|
+
const record = value;
|
|
2374
|
+
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
2375
|
+
}
|
|
2376
|
+
|
|
1669
2377
|
// src/lib/opencode/opencode-version-gate.ts
|
|
1670
2378
|
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
1671
2379
|
function isQueueValidatedVersion(version2) {
|
|
@@ -1680,7 +2388,63 @@ function buildOpenCodeVersionWarning(version2) {
|
|
|
1680
2388
|
}
|
|
1681
2389
|
|
|
1682
2390
|
// src/lib/opencode/process.ts
|
|
1683
|
-
import { execSync, spawn } from "child_process";
|
|
2391
|
+
import { execSync, spawn as spawn3 } from "child_process";
|
|
2392
|
+
|
|
2393
|
+
// src/lib/process-stop.ts
|
|
2394
|
+
async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
2395
|
+
if (!child.pid) {
|
|
2396
|
+
return { outcome: "not-running", code: child.exitCode, signal: child.signalCode };
|
|
2397
|
+
}
|
|
2398
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
2399
|
+
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
2400
|
+
}
|
|
2401
|
+
return new Promise((resolve4, reject) => {
|
|
2402
|
+
let forced = false;
|
|
2403
|
+
let settled = false;
|
|
2404
|
+
const timer = setTimeout(() => {
|
|
2405
|
+
forced = true;
|
|
2406
|
+
try {
|
|
2407
|
+
sendKill();
|
|
2408
|
+
} catch (error2) {
|
|
2409
|
+
if (error2.code === "ESRCH") {
|
|
2410
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2411
|
+
} else {
|
|
2412
|
+
fail(error2);
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
}, timeoutMs);
|
|
2416
|
+
const finish = (result) => {
|
|
2417
|
+
if (settled) return;
|
|
2418
|
+
settled = true;
|
|
2419
|
+
clearTimeout(timer);
|
|
2420
|
+
child.removeListener("exit", onExit);
|
|
2421
|
+
resolve4(result);
|
|
2422
|
+
};
|
|
2423
|
+
const fail = (error2) => {
|
|
2424
|
+
if (settled) return;
|
|
2425
|
+
settled = true;
|
|
2426
|
+
clearTimeout(timer);
|
|
2427
|
+
child.removeListener("exit", onExit);
|
|
2428
|
+
reject(error2);
|
|
2429
|
+
};
|
|
2430
|
+
const onExit = (code, signal) => {
|
|
2431
|
+
finish({ outcome: forced ? "killed" : "exited", code, signal });
|
|
2432
|
+
};
|
|
2433
|
+
child.once("exit", onExit);
|
|
2434
|
+
try {
|
|
2435
|
+
sendTerm();
|
|
2436
|
+
} catch (error2) {
|
|
2437
|
+
if (error2.code === "ESRCH") {
|
|
2438
|
+
finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
|
|
2439
|
+
} else {
|
|
2440
|
+
fail(error2);
|
|
2441
|
+
}
|
|
2442
|
+
return;
|
|
2443
|
+
}
|
|
2444
|
+
});
|
|
2445
|
+
}
|
|
2446
|
+
|
|
2447
|
+
// src/lib/opencode/process.ts
|
|
1684
2448
|
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
1685
2449
|
function getProcessCwd(pid) {
|
|
1686
2450
|
const platform = process.platform;
|
|
@@ -1836,39 +2600,45 @@ async function findHealthyOpenCodeInstances() {
|
|
|
1836
2600
|
}
|
|
1837
2601
|
return healthy;
|
|
1838
2602
|
}
|
|
1839
|
-
async function startOpenCode(port) {
|
|
2603
|
+
async function startOpenCode(port, options = {}) {
|
|
1840
2604
|
let command = "opencode";
|
|
1841
|
-
|
|
2605
|
+
const printLogs = options.inheritStdio ? ["--print-logs"] : [];
|
|
2606
|
+
let args = ["serve", "--port", port.toString(), "--hostname", "127.0.0.1", ...printLogs];
|
|
1842
2607
|
try {
|
|
1843
2608
|
execSync("which opencode", { stdio: "ignore" });
|
|
1844
2609
|
} catch {
|
|
1845
2610
|
command = "npx";
|
|
1846
|
-
args = [
|
|
1847
|
-
|
|
1848
|
-
|
|
2611
|
+
args = [
|
|
2612
|
+
"opencode",
|
|
2613
|
+
"serve",
|
|
2614
|
+
"--port",
|
|
2615
|
+
port.toString(),
|
|
2616
|
+
"--hostname",
|
|
2617
|
+
"127.0.0.1",
|
|
2618
|
+
...printLogs
|
|
2619
|
+
];
|
|
2620
|
+
}
|
|
2621
|
+
const child = spawn3(command, args, {
|
|
1849
2622
|
detached: true,
|
|
1850
|
-
stdio: "ignore",
|
|
2623
|
+
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
1851
2624
|
cwd: process.cwd()
|
|
1852
2625
|
});
|
|
1853
2626
|
return child;
|
|
1854
2627
|
}
|
|
1855
|
-
function
|
|
1856
|
-
|
|
1857
|
-
return;
|
|
1858
|
-
}
|
|
1859
|
-
try {
|
|
2628
|
+
function stopOpenCodeAndWait(opencodeProcess, timeoutMs) {
|
|
2629
|
+
const sendSignal = (signal) => {
|
|
1860
2630
|
if (process.platform === "win32") {
|
|
1861
|
-
opencodeProcess.kill(
|
|
2631
|
+
opencodeProcess.kill(signal);
|
|
1862
2632
|
} else {
|
|
1863
|
-
process.kill(-opencodeProcess.pid,
|
|
2633
|
+
process.kill(-opencodeProcess.pid, signal);
|
|
1864
2634
|
}
|
|
1865
|
-
}
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
2635
|
+
};
|
|
2636
|
+
return stopProcessAndWait(
|
|
2637
|
+
opencodeProcess,
|
|
2638
|
+
timeoutMs,
|
|
2639
|
+
() => sendSignal("SIGTERM"),
|
|
2640
|
+
() => sendSignal("SIGKILL")
|
|
2641
|
+
);
|
|
1872
2642
|
}
|
|
1873
2643
|
|
|
1874
2644
|
// src/lib/opencode/install.ts
|
|
@@ -2155,6 +2925,7 @@ async function createOpenCodeSession(port, directory) {
|
|
|
2155
2925
|
return data.id;
|
|
2156
2926
|
}
|
|
2157
2927
|
async function getModelAttachmentCapability(port, model) {
|
|
2928
|
+
const { model: baseModel } = splitModelVariant(model);
|
|
2158
2929
|
try {
|
|
2159
2930
|
const res = await timedFetch(`${opencodeBase(port)}/config/providers`);
|
|
2160
2931
|
if (!res.ok) {
|
|
@@ -2171,9 +2942,9 @@ async function getModelAttachmentCapability(port, model) {
|
|
|
2171
2942
|
);
|
|
2172
2943
|
return null;
|
|
2173
2944
|
}
|
|
2174
|
-
const slash =
|
|
2175
|
-
const providerId = slash > 0 ?
|
|
2176
|
-
let modelId = slash > 0 ?
|
|
2945
|
+
const slash = baseModel ? baseModel.indexOf("/") : -1;
|
|
2946
|
+
const providerId = slash > 0 ? baseModel.slice(0, slash) : void 0;
|
|
2947
|
+
let modelId = slash > 0 ? baseModel.slice(slash + 1) : void 0;
|
|
2177
2948
|
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
2178
2949
|
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
2179
2950
|
if (!provider && !providerId) {
|
|
@@ -2253,6 +3024,29 @@ async function buildFileParts(attachments, capable) {
|
|
|
2253
3024
|
}
|
|
2254
3025
|
return { parts, outcomes, capabilityUnknown };
|
|
2255
3026
|
}
|
|
3027
|
+
function splitModelVariant(raw) {
|
|
3028
|
+
const value = raw?.trim();
|
|
3029
|
+
if (!value) return {};
|
|
3030
|
+
const hashIndex = value.indexOf("#");
|
|
3031
|
+
if (hashIndex === -1) return { model: value };
|
|
3032
|
+
const model = value.slice(0, hashIndex).trim() || void 0;
|
|
3033
|
+
const variant = value.slice(hashIndex + 1).trim() || void 0;
|
|
3034
|
+
return { model, variant };
|
|
3035
|
+
}
|
|
3036
|
+
function applyModelOptions(body, options) {
|
|
3037
|
+
if (options?.agent) body.agent = options.agent;
|
|
3038
|
+
const { model, variant } = splitModelVariant(options?.model);
|
|
3039
|
+
if (model) {
|
|
3040
|
+
const slashIndex = model.indexOf("/");
|
|
3041
|
+
if (slashIndex !== -1) {
|
|
3042
|
+
body.model = {
|
|
3043
|
+
providerID: model.substring(0, slashIndex),
|
|
3044
|
+
modelID: model.substring(slashIndex + 1)
|
|
3045
|
+
};
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3048
|
+
if (variant) body.variant = variant;
|
|
3049
|
+
}
|
|
2256
3050
|
function messageText(m) {
|
|
2257
3051
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2258
3052
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
@@ -2277,18 +3071,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2277
3071
|
const body = {
|
|
2278
3072
|
parts
|
|
2279
3073
|
};
|
|
2280
|
-
|
|
2281
|
-
body.agent = options.agent;
|
|
2282
|
-
}
|
|
2283
|
-
if (options?.model) {
|
|
2284
|
-
const slashIndex = options.model.indexOf("/");
|
|
2285
|
-
if (slashIndex !== -1) {
|
|
2286
|
-
body.model = {
|
|
2287
|
-
providerID: options.model.substring(0, slashIndex),
|
|
2288
|
-
modelID: options.model.substring(slashIndex + 1)
|
|
2289
|
-
};
|
|
2290
|
-
}
|
|
2291
|
-
}
|
|
3074
|
+
applyModelOptions(body, options);
|
|
2292
3075
|
const res = await timedFetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
2293
3076
|
method: "POST",
|
|
2294
3077
|
headers: { "Content-Type": "application/json" },
|
|
@@ -2296,7 +3079,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2296
3079
|
});
|
|
2297
3080
|
if (res.status < 200 || res.status >= 300) {
|
|
2298
3081
|
const text = await res.text().catch(() => "");
|
|
2299
|
-
|
|
3082
|
+
const { variant } = splitModelVariant(options?.model);
|
|
3083
|
+
throw new Error(
|
|
3084
|
+
`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}${variant ? ` (variant: ${variant})` : ""}`
|
|
3085
|
+
);
|
|
2300
3086
|
}
|
|
2301
3087
|
const READ_BACK_ATTEMPTS = 5;
|
|
2302
3088
|
const READ_BACK_DELAY_MS = 150;
|
|
@@ -2320,7 +3106,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
2320
3106
|
}
|
|
2321
3107
|
}
|
|
2322
3108
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2323
|
-
await new Promise((
|
|
3109
|
+
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
2324
3110
|
}
|
|
2325
3111
|
}
|
|
2326
3112
|
return null;
|
|
@@ -2366,6 +3152,44 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2366
3152
|
}
|
|
2367
3153
|
return lastOk ?? last;
|
|
2368
3154
|
}
|
|
3155
|
+
function collectSubagentSessions(messages, userMessageId) {
|
|
3156
|
+
if (!messages || messages.length === 0) return [];
|
|
3157
|
+
const byParent = messages.filter(
|
|
3158
|
+
(message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
|
|
3159
|
+
);
|
|
3160
|
+
const assistants = byParent.length > 0 ? byParent : [];
|
|
3161
|
+
if (assistants.length === 0) {
|
|
3162
|
+
const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
|
|
3163
|
+
if (userIndex === -1) return [];
|
|
3164
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
3165
|
+
const message = messages[i];
|
|
3166
|
+
if (roleOf(message) === "user") break;
|
|
3167
|
+
if (roleOf(message) === "assistant") assistants.push(message);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
const refs = [];
|
|
3171
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3172
|
+
for (const message of assistants) {
|
|
3173
|
+
const parts = Array.isArray(message.parts) ? message.parts : [];
|
|
3174
|
+
for (const part of parts) {
|
|
3175
|
+
if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
|
|
3176
|
+
continue;
|
|
3177
|
+
const state = part.state;
|
|
3178
|
+
if (!state || typeof state !== "object") continue;
|
|
3179
|
+
const metadata = state.metadata;
|
|
3180
|
+
if (!metadata || typeof metadata !== "object") continue;
|
|
3181
|
+
const sessionId = metadata.sessionId;
|
|
3182
|
+
if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
|
|
3183
|
+
seen.add(sessionId);
|
|
3184
|
+
const start = state.time?.start;
|
|
3185
|
+
refs.push({
|
|
3186
|
+
sessionId,
|
|
3187
|
+
startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
return refs;
|
|
3192
|
+
}
|
|
2369
3193
|
function messageUsage(messages, userMessageId) {
|
|
2370
3194
|
if (!messages || messages.length === 0) return null;
|
|
2371
3195
|
const byParentAll = messages.filter(
|
|
@@ -2451,7 +3275,7 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
2451
3275
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
2452
3276
|
}
|
|
2453
3277
|
function isB2AbandonmentConfirmed(params) {
|
|
2454
|
-
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
3278
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false && params.rootOngoing === false;
|
|
2455
3279
|
}
|
|
2456
3280
|
function isAmbiguousTerminalFinish(m) {
|
|
2457
3281
|
if (completedOf(m) == null) return false;
|
|
@@ -2464,7 +3288,7 @@ function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
|
|
|
2464
3288
|
return isAmbiguousTerminalFinish(reply);
|
|
2465
3289
|
}
|
|
2466
3290
|
function isAmbiguousFinishResolved(params) {
|
|
2467
|
-
return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
|
|
3291
|
+
return params.sessionOngoing === false || params.sessionOngoing !== true && params.pinnedForMs >= params.maxPinnedMs;
|
|
2468
3292
|
}
|
|
2469
3293
|
function messageError(messages, userMessageId) {
|
|
2470
3294
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2494,8 +3318,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2494
3318
|
}
|
|
2495
3319
|
return false;
|
|
2496
3320
|
}
|
|
2497
|
-
function
|
|
2498
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3321
|
+
function classifyReplyAuthError(reply) {
|
|
2499
3322
|
const error2 = errorOf(reply);
|
|
2500
3323
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2501
3324
|
const e = error2;
|
|
@@ -2520,6 +3343,32 @@ function messageFailure(messages, userMessageId) {
|
|
|
2520
3343
|
}
|
|
2521
3344
|
return null;
|
|
2522
3345
|
}
|
|
3346
|
+
function messageFailure(messages, userMessageId) {
|
|
3347
|
+
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
3348
|
+
}
|
|
3349
|
+
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
3350
|
+
if (!messages || messages.length === 0) return null;
|
|
3351
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3352
|
+
const message = messages[i];
|
|
3353
|
+
if (roleOf(message) !== "assistant") continue;
|
|
3354
|
+
const created = createdOf(message);
|
|
3355
|
+
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
3356
|
+
const failure = classifyReplyAuthError(message);
|
|
3357
|
+
if (failure) {
|
|
3358
|
+
if (!failure.providerId) return null;
|
|
3359
|
+
return { providerId: failure.providerId, outcome: "failed", failure };
|
|
3360
|
+
}
|
|
3361
|
+
const providerId = message.info?.providerID;
|
|
3362
|
+
if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
|
|
3363
|
+
return { providerId, outcome: "succeeded" };
|
|
3364
|
+
}
|
|
3365
|
+
return null;
|
|
3366
|
+
}
|
|
3367
|
+
return null;
|
|
3368
|
+
}
|
|
3369
|
+
function findSubagentAuthOutcome(messages, sinceMs) {
|
|
3370
|
+
return findLatestSubagentAuthOutcome(messages, sinceMs);
|
|
3371
|
+
}
|
|
2523
3372
|
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2524
3373
|
if (classified != null) return classified;
|
|
2525
3374
|
if (hasConfiguredProvider !== false) return null;
|
|
@@ -2673,13 +3522,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2673
3522
|
}
|
|
2674
3523
|
|
|
2675
3524
|
// src/lib/opencode/session-db-size.ts
|
|
2676
|
-
import { statSync as
|
|
2677
|
-
import { join as
|
|
3525
|
+
import { statSync as statSync3 } from "fs";
|
|
3526
|
+
import { join as join4 } from "path";
|
|
2678
3527
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2679
3528
|
function statSessionDbBytes(homeDir) {
|
|
2680
|
-
const dbPath =
|
|
3529
|
+
const dbPath = join4(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2681
3530
|
try {
|
|
2682
|
-
return
|
|
3531
|
+
return statSync3(dbPath).size;
|
|
2683
3532
|
} catch (err) {
|
|
2684
3533
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2685
3534
|
if (!isMissingFile) {
|
|
@@ -2705,11 +3554,11 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
2705
3554
|
}
|
|
2706
3555
|
|
|
2707
3556
|
// src/lib/opencode/session-db-reclaim.ts
|
|
2708
|
-
import { statSync as
|
|
2709
|
-
import { dirname as
|
|
3557
|
+
import { statSync as statSync4, statfsSync } from "fs";
|
|
3558
|
+
import { dirname as dirname4 } from "path";
|
|
2710
3559
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2711
3560
|
try {
|
|
2712
|
-
const fsStats = statfsSync(
|
|
3561
|
+
const fsStats = statfsSync(dirname4(dbPath));
|
|
2713
3562
|
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2714
3563
|
if (availableBytes < requiredBytes) {
|
|
2715
3564
|
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
@@ -2778,7 +3627,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
2778
3627
|
);
|
|
2779
3628
|
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2780
3629
|
}
|
|
2781
|
-
const fileBytesForGuard =
|
|
3630
|
+
const fileBytesForGuard = statSync4(dbPath).size;
|
|
2782
3631
|
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2783
3632
|
if (skipReason !== null) {
|
|
2784
3633
|
console.warn(
|
|
@@ -2850,7 +3699,6 @@ var StreamForwarder = class {
|
|
|
2850
3699
|
handleFrame(frame) {
|
|
2851
3700
|
switch (frame.type) {
|
|
2852
3701
|
case "open":
|
|
2853
|
-
this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
|
|
2854
3702
|
void this.handleOpen(frame);
|
|
2855
3703
|
break;
|
|
2856
3704
|
case "req_data":
|
|
@@ -2886,12 +3734,21 @@ var StreamForwarder = class {
|
|
|
2886
3734
|
const { sid, method, path, headers, has_body } = frame;
|
|
2887
3735
|
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
2888
3736
|
const startedAt = Date.now();
|
|
3737
|
+
if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3738
|
+
this.callbacks.onOpen?.(sid, method, path);
|
|
3739
|
+
}
|
|
2889
3740
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
2890
3741
|
this.callbacks.onDrainPing?.();
|
|
2891
3742
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
2892
3743
|
this.send({ type: "res_end", sid });
|
|
2893
3744
|
return;
|
|
2894
3745
|
}
|
|
3746
|
+
if (path === TUNNEL_USAGE_REARM_PING_PATH) {
|
|
3747
|
+
this.callbacks.onUsageRearmPing?.();
|
|
3748
|
+
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
3749
|
+
this.send({ type: "res_end", sid });
|
|
3750
|
+
return;
|
|
3751
|
+
}
|
|
2895
3752
|
if (process.env.DEBUG) {
|
|
2896
3753
|
log("debug", "agent_request", {
|
|
2897
3754
|
correlation_id: correlationId,
|
|
@@ -2906,12 +3763,12 @@ var StreamForwarder = class {
|
|
|
2906
3763
|
let endBody;
|
|
2907
3764
|
if (has_body) {
|
|
2908
3765
|
const chunks = [];
|
|
2909
|
-
bodyPromise = new Promise((
|
|
3766
|
+
bodyPromise = new Promise((resolve4) => {
|
|
2910
3767
|
pushBody = (buf) => {
|
|
2911
3768
|
chunks.push(buf);
|
|
2912
3769
|
};
|
|
2913
3770
|
endBody = () => {
|
|
2914
|
-
|
|
3771
|
+
resolve4(Buffer.concat(chunks));
|
|
2915
3772
|
};
|
|
2916
3773
|
});
|
|
2917
3774
|
}
|
|
@@ -3036,11 +3893,12 @@ function connectTunnel(options) {
|
|
|
3036
3893
|
onResponse,
|
|
3037
3894
|
onInfo,
|
|
3038
3895
|
onWarning,
|
|
3039
|
-
onDrainPing
|
|
3896
|
+
onDrainPing,
|
|
3897
|
+
onUsageRearmPing
|
|
3040
3898
|
} = options;
|
|
3041
3899
|
const tunnelUrl = getTunnelUrlConfig();
|
|
3042
3900
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
3043
|
-
return new Promise((
|
|
3901
|
+
return new Promise((resolve4, reject) => {
|
|
3044
3902
|
const ws = new WebSocket2(url, {
|
|
3045
3903
|
headers: {
|
|
3046
3904
|
Authorization: authHeader
|
|
@@ -3048,7 +3906,8 @@ function connectTunnel(options) {
|
|
|
3048
3906
|
});
|
|
3049
3907
|
const forwarder = new StreamForwarder(ws, port, {
|
|
3050
3908
|
onHead: () => onResponse?.(),
|
|
3051
|
-
onDrainPing: () => onDrainPing?.()
|
|
3909
|
+
onDrainPing: () => onDrainPing?.(),
|
|
3910
|
+
onUsageRearmPing: () => onUsageRearmPing?.()
|
|
3052
3911
|
});
|
|
3053
3912
|
const connectionTimeout = setTimeout(() => {
|
|
3054
3913
|
ws.close();
|
|
@@ -3091,8 +3950,8 @@ function connectTunnel(options) {
|
|
|
3091
3950
|
try {
|
|
3092
3951
|
message = JSON.parse(data.toString());
|
|
3093
3952
|
} catch (error2) {
|
|
3094
|
-
const
|
|
3095
|
-
onError?.(`Failed to handle message: ${
|
|
3953
|
+
const errorMessage2 = error2 instanceof Error ? error2.message : "Unknown error";
|
|
3954
|
+
onError?.(`Failed to handle message: ${errorMessage2}`);
|
|
3096
3955
|
return;
|
|
3097
3956
|
}
|
|
3098
3957
|
if (isStreamFrame(message)) {
|
|
@@ -3104,7 +3963,7 @@ function connectTunnel(options) {
|
|
|
3104
3963
|
clearTimeout(connectionTimeout);
|
|
3105
3964
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
3106
3965
|
onConnected?.(connectedAgentId);
|
|
3107
|
-
|
|
3966
|
+
resolve4({
|
|
3108
3967
|
ws,
|
|
3109
3968
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
3110
3969
|
});
|
|
@@ -3209,6 +4068,7 @@ var RunnerConnection = class {
|
|
|
3209
4068
|
onError: (error2) => events.onError?.(error2),
|
|
3210
4069
|
onResponse: () => events.onResponse?.(),
|
|
3211
4070
|
onDrainPing: () => events.onDrainPing?.(),
|
|
4071
|
+
onUsageRearmPing: () => events.onUsageRearmPing?.(),
|
|
3212
4072
|
onInfo: (message) => events.onInfo?.(message),
|
|
3213
4073
|
onWarning: (message) => events.onWarning?.(message)
|
|
3214
4074
|
});
|
|
@@ -3235,10 +4095,10 @@ var RunnerConnection = class {
|
|
|
3235
4095
|
};
|
|
3236
4096
|
|
|
3237
4097
|
// src/lib/tunnel/ready-marker.ts
|
|
3238
|
-
import { writeFileSync } from "fs";
|
|
4098
|
+
import { writeFileSync as writeFileSync3 } from "fs";
|
|
3239
4099
|
function writeTunnelReadyMarker(path, agentId) {
|
|
3240
4100
|
try {
|
|
3241
|
-
|
|
4101
|
+
writeFileSync3(path, `${agentId}
|
|
3242
4102
|
`);
|
|
3243
4103
|
return { ok: true };
|
|
3244
4104
|
} catch (error2) {
|
|
@@ -3246,10 +4106,52 @@ function writeTunnelReadyMarker(path, agentId) {
|
|
|
3246
4106
|
}
|
|
3247
4107
|
}
|
|
3248
4108
|
|
|
4109
|
+
// src/lib/replication.ts
|
|
4110
|
+
import { spawn as spawn4 } from "child_process";
|
|
4111
|
+
function startSessionDbReplication(configPath) {
|
|
4112
|
+
return spawn4("litestream", ["replicate", "-config", configPath], {
|
|
4113
|
+
stdio: "inherit"
|
|
4114
|
+
});
|
|
4115
|
+
}
|
|
4116
|
+
async function stopSessionDbReplication(child, timeoutMs) {
|
|
4117
|
+
return stopProcessAndWait(
|
|
4118
|
+
child,
|
|
4119
|
+
timeoutMs,
|
|
4120
|
+
() => child.kill("SIGTERM"),
|
|
4121
|
+
() => child.kill("SIGKILL")
|
|
4122
|
+
);
|
|
4123
|
+
}
|
|
4124
|
+
|
|
4125
|
+
// src/lib/process-liveness.ts
|
|
4126
|
+
import { readFileSync as readFileSync4 } from "fs";
|
|
4127
|
+
function isProcessAlive(pid) {
|
|
4128
|
+
try {
|
|
4129
|
+
process.kill(pid, 0);
|
|
4130
|
+
} catch (error2) {
|
|
4131
|
+
const code = error2.code;
|
|
4132
|
+
if (code === "ESRCH") return false;
|
|
4133
|
+
if (code === "EPERM") return true;
|
|
4134
|
+
console.error(
|
|
4135
|
+
`[process-liveness] could not probe process ${pid}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4136
|
+
);
|
|
4137
|
+
return false;
|
|
4138
|
+
}
|
|
4139
|
+
if (process.platform !== "linux") return true;
|
|
4140
|
+
try {
|
|
4141
|
+
const status2 = readFileSync4(`/proc/${pid}/status`, "utf8");
|
|
4142
|
+
return !/^State:\s+Z(?:\s|$)/m.test(status2);
|
|
4143
|
+
} catch (error2) {
|
|
4144
|
+
console.error(
|
|
4145
|
+
`[process-liveness] could not inspect /proc/${pid}/status: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4146
|
+
);
|
|
4147
|
+
return true;
|
|
4148
|
+
}
|
|
4149
|
+
}
|
|
4150
|
+
|
|
3249
4151
|
// src/lib/openai-usage.ts
|
|
3250
|
-
import { readFileSync as
|
|
3251
|
-
import { homedir as
|
|
3252
|
-
import { join as
|
|
4152
|
+
import { readFileSync as readFileSync5 } from "fs";
|
|
4153
|
+
import { homedir as homedir3 } from "os";
|
|
4154
|
+
import { join as join5 } from "path";
|
|
3253
4155
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3254
4156
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3255
4157
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3263,7 +4165,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3263
4165
|
}
|
|
3264
4166
|
function readOpenCodeChatGptCredentials() {
|
|
3265
4167
|
try {
|
|
3266
|
-
const raw =
|
|
4168
|
+
const raw = readFileSync5(join5(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3267
4169
|
let parsed;
|
|
3268
4170
|
try {
|
|
3269
4171
|
parsed = JSON.parse(raw);
|
|
@@ -3285,6 +4187,23 @@ function readOpenCodeChatGptCredentials() {
|
|
|
3285
4187
|
return null;
|
|
3286
4188
|
}
|
|
3287
4189
|
}
|
|
4190
|
+
function parseChatGptIdentity(accessToken) {
|
|
4191
|
+
const segments = accessToken.split(".");
|
|
4192
|
+
if (segments.length !== 3) return null;
|
|
4193
|
+
let payload;
|
|
4194
|
+
try {
|
|
4195
|
+
const parsed = JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8"));
|
|
4196
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
4197
|
+
payload = parsed;
|
|
4198
|
+
} catch {
|
|
4199
|
+
return null;
|
|
4200
|
+
}
|
|
4201
|
+
const profile = payload["https://api.openai.com/profile"];
|
|
4202
|
+
const auth = payload["https://api.openai.com/auth"];
|
|
4203
|
+
const ownerEmail = typeof profile?.email === "string" ? profile.email.trim() || null : null;
|
|
4204
|
+
const planType = typeof auth?.chatgpt_plan_type === "string" ? auth.chatgpt_plan_type.trim() || null : null;
|
|
4205
|
+
return ownerEmail === null && planType === null ? null : { ownerEmail, planType };
|
|
4206
|
+
}
|
|
3288
4207
|
function toWindow2(headers, name) {
|
|
3289
4208
|
const utilizationHeader = headers.get(`x-codex-${name}-used-percent`);
|
|
3290
4209
|
const windowMinutesHeader = headers.get(`x-codex-${name}-window-minutes`);
|
|
@@ -3360,6 +4279,7 @@ async function getOpenAiUsage(port) {
|
|
|
3360
4279
|
"credentials_expired"
|
|
3361
4280
|
);
|
|
3362
4281
|
}
|
|
4282
|
+
const subscription = parseChatGptIdentity(credentials2.accessToken);
|
|
3363
4283
|
const models = await resolveProbeModels(port);
|
|
3364
4284
|
if (models.length === 0) {
|
|
3365
4285
|
throw new OpenAiUsageError("No supported OpenAI probe model is available.", "no_probe_model");
|
|
@@ -3392,7 +4312,7 @@ async function getOpenAiUsage(port) {
|
|
|
3392
4312
|
"no_usable_window"
|
|
3393
4313
|
);
|
|
3394
4314
|
}
|
|
3395
|
-
return usage;
|
|
4315
|
+
return { ...usage, subscription };
|
|
3396
4316
|
}
|
|
3397
4317
|
if (res.status === 401) {
|
|
3398
4318
|
throw new OpenAiUsageError(
|
|
@@ -3593,58 +4513,97 @@ function readDisk(homeDir) {
|
|
|
3593
4513
|
};
|
|
3594
4514
|
}
|
|
3595
4515
|
}
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
4516
|
+
var CPU_PEAK_WINDOW_MS = 6e4;
|
|
4517
|
+
var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
|
|
4518
|
+
var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4519
|
+
function createCpuPeakSampler() {
|
|
4520
|
+
const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4521
|
+
sampleHistory[0] = readCpuSample();
|
|
4522
|
+
let nextSampleIndex = 1;
|
|
4523
|
+
let sampleCount = 1;
|
|
4524
|
+
let peak = null;
|
|
4525
|
+
const timer = setInterval(() => {
|
|
3599
4526
|
const current = readCpuSample();
|
|
3600
|
-
const
|
|
3601
|
-
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
const warnings = [];
|
|
3607
|
-
if (disk.warning) warnings.push(disk.warning);
|
|
3608
|
-
if (ecsWarning) warnings.push(ecsWarning);
|
|
3609
|
-
let cpuPercent = hostCpuPercent;
|
|
3610
|
-
let cpuCount = hostCpuCount;
|
|
3611
|
-
let memoryTotalBytes = totalmem();
|
|
3612
|
-
let memoryAvailableBytes = freemem();
|
|
3613
|
-
if (limits !== null) {
|
|
3614
|
-
cpuCount = limits.cpuCount;
|
|
3615
|
-
memoryTotalBytes = limits.memoryTotalBytes;
|
|
3616
|
-
memoryAvailableBytes = clamp(
|
|
3617
|
-
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
3618
|
-
0,
|
|
3619
|
-
limits.memoryTotalBytes
|
|
3620
|
-
);
|
|
3621
|
-
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4527
|
+
const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
|
|
4528
|
+
if (sampleFromWindowAgo !== void 0) {
|
|
4529
|
+
const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
|
|
4530
|
+
if (percentage !== null) {
|
|
4531
|
+
peak = peak === null ? percentage : Math.max(peak, percentage);
|
|
4532
|
+
}
|
|
3622
4533
|
}
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
4534
|
+
sampleHistory[nextSampleIndex] = current;
|
|
4535
|
+
nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
|
|
4536
|
+
sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
|
|
4537
|
+
}, CPU_PEAK_SAMPLE_INTERVAL_MS);
|
|
4538
|
+
return {
|
|
4539
|
+
takeAndReset: () => {
|
|
4540
|
+
const currentPeak = peak;
|
|
4541
|
+
peak = null;
|
|
4542
|
+
return currentPeak;
|
|
4543
|
+
},
|
|
4544
|
+
stop: () => clearInterval(timer)
|
|
4545
|
+
};
|
|
4546
|
+
}
|
|
4547
|
+
function createResourceUsageCollector(homeDir) {
|
|
4548
|
+
let previous = readCpuSample();
|
|
4549
|
+
const cpuPeakSampler = createCpuPeakSampler();
|
|
4550
|
+
return {
|
|
4551
|
+
collect: async () => {
|
|
4552
|
+
const current = readCpuSample();
|
|
4553
|
+
const hostCpuPercent = cpuPercentBetween(previous, current);
|
|
4554
|
+
const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
|
|
4555
|
+
const hostCpuCount = cpus().length;
|
|
4556
|
+
previous = current;
|
|
4557
|
+
const disk = readDisk(homeDir);
|
|
4558
|
+
const opencodeDbBytes = statSessionDbBytes(homeDir);
|
|
4559
|
+
const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
|
|
4560
|
+
const warnings = [];
|
|
4561
|
+
if (disk.warning) warnings.push(disk.warning);
|
|
4562
|
+
if (ecsWarning) warnings.push(ecsWarning);
|
|
4563
|
+
let cpuPercent = hostCpuPercent;
|
|
4564
|
+
let cpuPeakPercent = hostCpuPeakPercent;
|
|
4565
|
+
let cpuCount = hostCpuCount;
|
|
4566
|
+
let memoryTotalBytes = totalmem();
|
|
4567
|
+
let memoryAvailableBytes = freemem();
|
|
4568
|
+
if (limits !== null) {
|
|
4569
|
+
cpuCount = limits.cpuCount;
|
|
4570
|
+
memoryTotalBytes = limits.memoryTotalBytes;
|
|
4571
|
+
memoryAvailableBytes = clamp(
|
|
4572
|
+
limits.memoryTotalBytes - (totalmem() - freemem()),
|
|
4573
|
+
0,
|
|
4574
|
+
limits.memoryTotalBytes
|
|
4575
|
+
);
|
|
4576
|
+
cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4577
|
+
cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
|
|
4578
|
+
}
|
|
4579
|
+
return {
|
|
4580
|
+
usage: {
|
|
4581
|
+
cpuPercent,
|
|
4582
|
+
cpuPeakPercent,
|
|
4583
|
+
cpuCount,
|
|
4584
|
+
memoryTotalBytes,
|
|
4585
|
+
memoryAvailableBytes,
|
|
4586
|
+
diskTotalBytes: disk.totalBytes,
|
|
4587
|
+
diskFreeBytes: disk.freeBytes,
|
|
4588
|
+
opencodeDbBytes
|
|
4589
|
+
},
|
|
4590
|
+
warnings
|
|
4591
|
+
};
|
|
4592
|
+
},
|
|
4593
|
+
stop: cpuPeakSampler.stop
|
|
3635
4594
|
};
|
|
3636
4595
|
}
|
|
3637
4596
|
|
|
3638
4597
|
// src/lib/channels/driver.ts
|
|
3639
|
-
import { homedir as
|
|
4598
|
+
import { homedir as homedir4 } from "os";
|
|
3640
4599
|
|
|
3641
4600
|
// src/lib/runner-file-sync.ts
|
|
3642
|
-
import { join as
|
|
4601
|
+
import { join as join7 } from "path";
|
|
3643
4602
|
|
|
3644
4603
|
// src/lib/file-push.ts
|
|
3645
4604
|
import { randomUUID } from "crypto";
|
|
3646
4605
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
3647
|
-
import { basename, dirname as
|
|
4606
|
+
import { basename, dirname as dirname5, isAbsolute, join as join6, relative, resolve as resolve2, sep } from "path";
|
|
3648
4607
|
var FILE_MODE = 384;
|
|
3649
4608
|
var DIRECTORY_MODE = 448;
|
|
3650
4609
|
async function writePushedFile(request) {
|
|
@@ -3675,9 +4634,9 @@ async function writePushedFile(request) {
|
|
|
3675
4634
|
}
|
|
3676
4635
|
try {
|
|
3677
4636
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
3678
|
-
|
|
4637
|
+
dirname5(candidate)
|
|
3679
4638
|
);
|
|
3680
|
-
const realTarget =
|
|
4639
|
+
const realTarget = join6(existingAncestor, ...missingSegments, basename(candidate));
|
|
3681
4640
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
3682
4641
|
if (allowedDirectory === null) {
|
|
3683
4642
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -3687,8 +4646,8 @@ async function writePushedFile(request) {
|
|
|
3687
4646
|
}
|
|
3688
4647
|
if (missingSegments.length > 0) {
|
|
3689
4648
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
3690
|
-
const realParent = await realpath(
|
|
3691
|
-
if (realParent !==
|
|
4649
|
+
const realParent = await realpath(dirname5(realTarget));
|
|
4650
|
+
if (realParent !== dirname5(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
3692
4651
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
3693
4652
|
path: realTarget,
|
|
3694
4653
|
bytes,
|
|
@@ -3713,7 +4672,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
3713
4672
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
3714
4673
|
return null;
|
|
3715
4674
|
}
|
|
3716
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
4675
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join6(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
3717
4676
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
3718
4677
|
return null;
|
|
3719
4678
|
}
|
|
@@ -3731,7 +4690,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
3731
4690
|
try {
|
|
3732
4691
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
3733
4692
|
} catch (err) {
|
|
3734
|
-
const parent =
|
|
4693
|
+
const parent = dirname5(current);
|
|
3735
4694
|
if (err.code !== "ENOENT" || parent === current) {
|
|
3736
4695
|
throw err;
|
|
3737
4696
|
}
|
|
@@ -3786,13 +4745,13 @@ function contains(realDirectory, realTarget) {
|
|
|
3786
4745
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
3787
4746
|
let current = existingAncestor;
|
|
3788
4747
|
for (const segment of missingSegments) {
|
|
3789
|
-
current =
|
|
4748
|
+
current = join6(current, segment);
|
|
3790
4749
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
3791
4750
|
await chmod(current, DIRECTORY_MODE);
|
|
3792
4751
|
}
|
|
3793
4752
|
}
|
|
3794
4753
|
async function writeAtomically(realTarget, content) {
|
|
3795
|
-
const temporaryPath =
|
|
4754
|
+
const temporaryPath = join6(dirname5(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
3796
4755
|
let handle;
|
|
3797
4756
|
try {
|
|
3798
4757
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -3922,12 +4881,12 @@ var NOT_APPLIED = {
|
|
|
3922
4881
|
opencodeAuthApplied: false
|
|
3923
4882
|
};
|
|
3924
4883
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
3925
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3926
|
-
return expanded ===
|
|
4884
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4885
|
+
return expanded === join7(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
3927
4886
|
}
|
|
3928
4887
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
3929
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3930
|
-
return expanded ===
|
|
4888
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join7(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
4889
|
+
return expanded === join7(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
3931
4890
|
}
|
|
3932
4891
|
async function applyOne(options, file) {
|
|
3933
4892
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -4454,6 +5413,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4454
5413
|
* and stops opencode.
|
|
4455
5414
|
*/
|
|
4456
5415
|
stopped = false;
|
|
5416
|
+
recycleRequestedFlag = false;
|
|
4457
5417
|
constructor(config) {
|
|
4458
5418
|
this.agentId = config.agentId;
|
|
4459
5419
|
this.port = config.port;
|
|
@@ -4473,7 +5433,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4473
5433
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
4474
5434
|
this.now = config.now ?? (() => Date.now());
|
|
4475
5435
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
4476
|
-
this.homeDir = config.homeDir ??
|
|
5436
|
+
this.homeDir = config.homeDir ?? homedir4();
|
|
4477
5437
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
4478
5438
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
4479
5439
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
@@ -4559,6 +5519,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4559
5519
|
let dispatched = 0;
|
|
4560
5520
|
try {
|
|
4561
5521
|
const conversations = await this.getPendingConversations();
|
|
5522
|
+
if (this.recycleRequestedFlag) {
|
|
5523
|
+
this.stop();
|
|
5524
|
+
}
|
|
4562
5525
|
if (conversations.length > 0) {
|
|
4563
5526
|
const total = conversations.reduce((sum, c) => sum + (c.pending_message_count ?? 0), 0);
|
|
4564
5527
|
this.log({
|
|
@@ -4687,6 +5650,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4687
5650
|
stop() {
|
|
4688
5651
|
this.stopped = true;
|
|
4689
5652
|
}
|
|
5653
|
+
/**
|
|
5654
|
+
* The server clears this request when a new MicroVM identity is recorded, so a
|
|
5655
|
+
* same-VM tunnel reconnect does not consume it. This is a plain read rather
|
|
5656
|
+
* than a consume; `run.ts` guards the action once-only.
|
|
5657
|
+
*/
|
|
5658
|
+
get recycleRequested() {
|
|
5659
|
+
return this.recycleRequestedFlag;
|
|
5660
|
+
}
|
|
4690
5661
|
/**
|
|
4691
5662
|
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
4692
5663
|
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
@@ -4824,7 +5795,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4824
5795
|
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
4825
5796
|
break;
|
|
4826
5797
|
}
|
|
4827
|
-
const
|
|
5798
|
+
const errorMessage2 = err instanceof Error ? err.message : String(err);
|
|
4828
5799
|
this.sessions.delete(conv.id);
|
|
4829
5800
|
this.supersede(conv.id, sessionId);
|
|
4830
5801
|
this.log({
|
|
@@ -4833,7 +5804,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4833
5804
|
conversation_id: conv.id,
|
|
4834
5805
|
message_id: message.id
|
|
4835
5806
|
});
|
|
4836
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5807
|
+
await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
|
|
4837
5808
|
this.log({
|
|
4838
5809
|
level: "warn",
|
|
4839
5810
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -4844,7 +5815,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4844
5815
|
});
|
|
4845
5816
|
this.log({
|
|
4846
5817
|
level: "error",
|
|
4847
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
5818
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage2}`,
|
|
4848
5819
|
conversation_id: conv.id,
|
|
4849
5820
|
message_id: message.id
|
|
4850
5821
|
});
|
|
@@ -4865,14 +5836,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4865
5836
|
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4866
5837
|
this.sessions.delete(conv.id);
|
|
4867
5838
|
this.supersede(conv.id, sessionId);
|
|
4868
|
-
const
|
|
5839
|
+
const errorMessage2 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
4869
5840
|
this.log({
|
|
4870
5841
|
level: "error",
|
|
4871
|
-
message:
|
|
5842
|
+
message: errorMessage2,
|
|
4872
5843
|
conversation_id: conv.id,
|
|
4873
5844
|
message_id: message.id
|
|
4874
5845
|
});
|
|
4875
|
-
await this.markFailed(conv.id, message.id, null,
|
|
5846
|
+
await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
|
|
4876
5847
|
this.log({
|
|
4877
5848
|
level: "warn",
|
|
4878
5849
|
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -5206,6 +6177,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5206
6177
|
});
|
|
5207
6178
|
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
5208
6179
|
}
|
|
6180
|
+
if (ocId !== null) {
|
|
6181
|
+
await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
|
|
6182
|
+
}
|
|
5209
6183
|
} catch (err) {
|
|
5210
6184
|
if (err instanceof ChannelAuthError) throw err;
|
|
5211
6185
|
this.log({
|
|
@@ -5740,6 +6714,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5740
6714
|
deliveryDeadlineAnchored: false,
|
|
5741
6715
|
b2PinnedSinceMs: 0,
|
|
5742
6716
|
b2LastDescendantCheckMs: 0,
|
|
6717
|
+
b2RootOngoingHeldLogged: false,
|
|
5743
6718
|
b2AbandonedSignalled: false,
|
|
5744
6719
|
ambiguousPinnedSinceMs: 0,
|
|
5745
6720
|
ambiguousResolved: false
|
|
@@ -5834,6 +6809,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5834
6809
|
deliveryDeadlineAnchored: false,
|
|
5835
6810
|
b2PinnedSinceMs: 0,
|
|
5836
6811
|
b2LastDescendantCheckMs: 0,
|
|
6812
|
+
b2RootOngoingHeldLogged: false,
|
|
5837
6813
|
b2AbandonedSignalled: false,
|
|
5838
6814
|
ambiguousPinnedSinceMs: 0,
|
|
5839
6815
|
ambiguousResolved: false
|
|
@@ -6168,6 +7144,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6168
7144
|
return;
|
|
6169
7145
|
}
|
|
6170
7146
|
inFlight.done = true;
|
|
7147
|
+
await this.reportSubagentAuthFailures(
|
|
7148
|
+
watcher.conv.id,
|
|
7149
|
+
inFlight.opencodeMessageId,
|
|
7150
|
+
inFlight.evidentMessageId,
|
|
7151
|
+
messages
|
|
7152
|
+
);
|
|
6171
7153
|
}
|
|
6172
7154
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6173
7155
|
return;
|
|
@@ -6187,6 +7169,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6187
7169
|
if (snapshotReadable) {
|
|
6188
7170
|
inFlight.b2PinnedSinceMs = 0;
|
|
6189
7171
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
7172
|
+
inFlight.b2RootOngoingHeldLogged = false;
|
|
6190
7173
|
inFlight.b2AbandonedSignalled = false;
|
|
6191
7174
|
}
|
|
6192
7175
|
} else {
|
|
@@ -6198,11 +7181,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6198
7181
|
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
6199
7182
|
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
6200
7183
|
inFlight.b2LastDescendantCheckMs = this.now();
|
|
6201
|
-
const descendantOngoing = await
|
|
7184
|
+
const [descendantOngoing, rootOngoing] = await Promise.all([
|
|
7185
|
+
this.isAnyDescendantSessionOngoing(sessionId),
|
|
7186
|
+
isSessionOngoing(this.port, sessionId)
|
|
7187
|
+
]);
|
|
6202
7188
|
if (isB2AbandonmentConfirmed({
|
|
6203
7189
|
pinnedForMs,
|
|
6204
7190
|
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
6205
|
-
descendantOngoing
|
|
7191
|
+
descendantOngoing,
|
|
7192
|
+
rootOngoing
|
|
6206
7193
|
})) {
|
|
6207
7194
|
inFlight.b2AbandonedSignalled = true;
|
|
6208
7195
|
this.log({
|
|
@@ -6211,12 +7198,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6211
7198
|
conversation_id: conv.id,
|
|
6212
7199
|
message_id: id
|
|
6213
7200
|
});
|
|
7201
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
6214
7202
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
6215
|
-
watched_for_ms: pinnedForMs
|
|
7203
|
+
watched_for_ms: pinnedForMs,
|
|
7204
|
+
finish: reply?.info?.finish ?? reply?.finish,
|
|
7205
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
7206
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
7207
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
6216
7208
|
});
|
|
6217
7209
|
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
6218
7210
|
return;
|
|
6219
7211
|
}
|
|
7212
|
+
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
7213
|
+
inFlight.b2RootOngoingHeldLogged = true;
|
|
7214
|
+
this.log({
|
|
7215
|
+
level: "warn",
|
|
7216
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with root_ongoing=${rootOngoing} and descendant_ongoing=${descendantOngoing} \u2014 holding until OpenCode confirms the root is idle`,
|
|
7217
|
+
conversation_id: conv.id,
|
|
7218
|
+
message_id: id
|
|
7219
|
+
});
|
|
7220
|
+
}
|
|
6220
7221
|
}
|
|
6221
7222
|
}
|
|
6222
7223
|
const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
|
|
@@ -6391,6 +7392,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6391
7392
|
return;
|
|
6392
7393
|
}
|
|
6393
7394
|
inFlight.done = true;
|
|
7395
|
+
await this.reportSubagentAuthFailures(
|
|
7396
|
+
watcher.conv.id,
|
|
7397
|
+
inFlight.opencodeMessageId,
|
|
7398
|
+
inFlight.evidentMessageId,
|
|
7399
|
+
messages
|
|
7400
|
+
);
|
|
6394
7401
|
}
|
|
6395
7402
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
6396
7403
|
}
|
|
@@ -6561,6 +7568,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6561
7568
|
});
|
|
6562
7569
|
return;
|
|
6563
7570
|
}
|
|
7571
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
6564
7572
|
this.dontRedispatch.delete(row.id);
|
|
6565
7573
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
6566
7574
|
return;
|
|
@@ -6722,6 +7730,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6722
7730
|
});
|
|
6723
7731
|
return;
|
|
6724
7732
|
}
|
|
7733
|
+
if (ocId !== null) {
|
|
7734
|
+
await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
|
|
7735
|
+
}
|
|
6725
7736
|
this.dontRedispatch.delete(row.id);
|
|
6726
7737
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
6727
7738
|
}
|
|
@@ -6815,14 +7826,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6815
7826
|
this.unconfirmedDispatchFailures.delete(row.id);
|
|
6816
7827
|
this.sessions.delete(readoptConv.id);
|
|
6817
7828
|
this.supersede(readoptConv.id, sessionId);
|
|
6818
|
-
const
|
|
7829
|
+
const errorMessage2 = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
6819
7830
|
this.log({
|
|
6820
7831
|
level: "error",
|
|
6821
|
-
message:
|
|
7832
|
+
message: errorMessage2,
|
|
6822
7833
|
conversation_id: row.conversation_id,
|
|
6823
7834
|
message_id: row.id
|
|
6824
7835
|
});
|
|
6825
|
-
await this.markFailed(row.conversation_id, row.id, null,
|
|
7836
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage2).catch((markErr) => {
|
|
6826
7837
|
this.log({
|
|
6827
7838
|
level: "warn",
|
|
6828
7839
|
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
@@ -7437,6 +8448,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7437
8448
|
throw new Error(`Failed to get pending conversations: HTTP ${res.status}`);
|
|
7438
8449
|
}
|
|
7439
8450
|
const data = await res.json();
|
|
8451
|
+
this.recycleRequestedFlag = data.recycle_requested === true;
|
|
7440
8452
|
let conversations = data.conversations;
|
|
7441
8453
|
if (this.conversationFilter) {
|
|
7442
8454
|
conversations = conversations.filter((c) => c.id === this.conversationFilter);
|
|
@@ -7672,6 +8684,111 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7672
8684
|
reply?.info?.modelID ?? null
|
|
7673
8685
|
);
|
|
7674
8686
|
}
|
|
8687
|
+
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
8688
|
+
const providerId = failure.providerId ?? "(unknown)";
|
|
8689
|
+
try {
|
|
8690
|
+
const res = await this.fetchImpl(
|
|
8691
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
|
|
8692
|
+
{
|
|
8693
|
+
method: "POST",
|
|
8694
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
8695
|
+
body: JSON.stringify({
|
|
8696
|
+
provider_id: failure.providerId,
|
|
8697
|
+
model_id: failure.modelId,
|
|
8698
|
+
reason: failure.reason
|
|
8699
|
+
})
|
|
8700
|
+
}
|
|
8701
|
+
);
|
|
8702
|
+
if (!res.ok) {
|
|
8703
|
+
this.log({
|
|
8704
|
+
level: "warn",
|
|
8705
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8706
|
+
conversation_id: conversationId,
|
|
8707
|
+
message_id: messageId
|
|
8708
|
+
});
|
|
8709
|
+
}
|
|
8710
|
+
} catch (err) {
|
|
8711
|
+
this.log({
|
|
8712
|
+
level: "warn",
|
|
8713
|
+
message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8714
|
+
conversation_id: conversationId,
|
|
8715
|
+
message_id: messageId
|
|
8716
|
+
});
|
|
8717
|
+
}
|
|
8718
|
+
}
|
|
8719
|
+
async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
|
|
8720
|
+
try {
|
|
8721
|
+
const res = await this.fetchImpl(
|
|
8722
|
+
`${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
|
|
8723
|
+
{
|
|
8724
|
+
method: "DELETE",
|
|
8725
|
+
headers: { Authorization: this.getAuthHeader() }
|
|
8726
|
+
}
|
|
8727
|
+
);
|
|
8728
|
+
if (!res.ok) {
|
|
8729
|
+
this.log({
|
|
8730
|
+
level: "warn",
|
|
8731
|
+
message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
|
|
8732
|
+
conversation_id: conversationId,
|
|
8733
|
+
message_id: messageId
|
|
8734
|
+
});
|
|
8735
|
+
}
|
|
8736
|
+
} catch (err) {
|
|
8737
|
+
this.log({
|
|
8738
|
+
level: "warn",
|
|
8739
|
+
message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8740
|
+
conversation_id: conversationId,
|
|
8741
|
+
message_id: messageId
|
|
8742
|
+
});
|
|
8743
|
+
}
|
|
8744
|
+
}
|
|
8745
|
+
async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
|
|
8746
|
+
const refs = collectSubagentSessions(messages, opencodeMessageId);
|
|
8747
|
+
if (refs.length === 0) return;
|
|
8748
|
+
const failedProviders = /* @__PURE__ */ new Map();
|
|
8749
|
+
const succeededProviders = /* @__PURE__ */ new Set();
|
|
8750
|
+
for (const ref of refs) {
|
|
8751
|
+
try {
|
|
8752
|
+
const childMessages = await getSessionMessages(this.port, ref.sessionId);
|
|
8753
|
+
if (childMessages === null) {
|
|
8754
|
+
this.log({
|
|
8755
|
+
level: "debug",
|
|
8756
|
+
message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
|
|
8757
|
+
conversation_id: conversationId,
|
|
8758
|
+
message_id: evidentMessageId
|
|
8759
|
+
});
|
|
8760
|
+
continue;
|
|
8761
|
+
}
|
|
8762
|
+
const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
|
|
8763
|
+
if (!outcome) continue;
|
|
8764
|
+
if (outcome.outcome === "failed") {
|
|
8765
|
+
failedProviders.set(outcome.providerId, outcome.failure);
|
|
8766
|
+
} else {
|
|
8767
|
+
succeededProviders.add(outcome.providerId);
|
|
8768
|
+
}
|
|
8769
|
+
} catch (err) {
|
|
8770
|
+
this.log({
|
|
8771
|
+
level: "warn",
|
|
8772
|
+
message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
|
|
8773
|
+
conversation_id: conversationId,
|
|
8774
|
+
message_id: evidentMessageId
|
|
8775
|
+
});
|
|
8776
|
+
}
|
|
8777
|
+
}
|
|
8778
|
+
for (const [providerId, failure] of failedProviders) {
|
|
8779
|
+
this.log({
|
|
8780
|
+
level: "warn",
|
|
8781
|
+
message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
|
|
8782
|
+
conversation_id: conversationId,
|
|
8783
|
+
message_id: evidentMessageId
|
|
8784
|
+
});
|
|
8785
|
+
await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
|
|
8786
|
+
}
|
|
8787
|
+
for (const providerId of succeededProviders) {
|
|
8788
|
+
if (failedProviders.has(providerId)) continue;
|
|
8789
|
+
await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
|
|
8790
|
+
}
|
|
8791
|
+
}
|
|
7675
8792
|
/**
|
|
7676
8793
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
7677
8794
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -7873,7 +8990,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
7873
8990
|
}
|
|
7874
8991
|
if (!ctx.interactive) {
|
|
7875
8992
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
7876
|
-
const proc = await startOpenCode(ctx.port);
|
|
8993
|
+
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
7877
8994
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
7878
8995
|
if (!health.healthy) {
|
|
7879
8996
|
return {
|
|
@@ -7941,7 +9058,7 @@ Port ${port} is already in use.`));
|
|
|
7941
9058
|
}
|
|
7942
9059
|
if (action === "start") {
|
|
7943
9060
|
const spinner = ora2("Starting OpenCode...").start();
|
|
7944
|
-
const proc = await startOpenCode(port);
|
|
9061
|
+
const proc = await startOpenCode(port, { inheritStdio: ctx.inheritStdio });
|
|
7945
9062
|
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
7946
9063
|
if (!health.healthy) {
|
|
7947
9064
|
spinner.fail("Failed to start OpenCode");
|
|
@@ -7953,12 +9070,558 @@ Port ${port} is already in use.`));
|
|
|
7953
9070
|
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
7954
9071
|
}
|
|
7955
9072
|
|
|
9073
|
+
// src/lib/runner-credentials.ts
|
|
9074
|
+
import { chmodSync as chmodSync2, writeFileSync as writeFileSync4 } from "fs";
|
|
9075
|
+
import { spawn as spawn5 } from "child_process";
|
|
9076
|
+
var RUNNER_SECRET_FETCH_TIMEOUT_MS = 6e4;
|
|
9077
|
+
var CREDENTIAL_RESTORE_TIMEOUT_MS = 6e4;
|
|
9078
|
+
var GITHUB_PROBE_TIMEOUT_MS = 1e4;
|
|
9079
|
+
var GIT_CONFIG_GLOBAL = "/tmp/gitconfig";
|
|
9080
|
+
var GIT_CREDENTIAL_HELPER = "/tmp/git-credential-helper.sh";
|
|
9081
|
+
function commandError2(result) {
|
|
9082
|
+
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
9083
|
+
}
|
|
9084
|
+
var runCommand2 = (command, args, opts) => {
|
|
9085
|
+
return new Promise((resolve4) => {
|
|
9086
|
+
let child;
|
|
9087
|
+
let stdout = "";
|
|
9088
|
+
let stderr = "";
|
|
9089
|
+
let settled = false;
|
|
9090
|
+
const timer = {};
|
|
9091
|
+
const finish = (result) => {
|
|
9092
|
+
if (settled) return;
|
|
9093
|
+
settled = true;
|
|
9094
|
+
if (timer.handle) clearTimeout(timer.handle);
|
|
9095
|
+
resolve4(result);
|
|
9096
|
+
};
|
|
9097
|
+
try {
|
|
9098
|
+
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
9099
|
+
} catch (error2) {
|
|
9100
|
+
finish({
|
|
9101
|
+
code: null,
|
|
9102
|
+
stdout,
|
|
9103
|
+
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
9104
|
+
timedOut: false
|
|
9105
|
+
});
|
|
9106
|
+
return;
|
|
9107
|
+
}
|
|
9108
|
+
child.stdout?.setEncoding("utf8");
|
|
9109
|
+
child.stdout?.on("data", (chunk) => {
|
|
9110
|
+
stdout += chunk;
|
|
9111
|
+
});
|
|
9112
|
+
child.stderr?.setEncoding("utf8");
|
|
9113
|
+
child.stderr?.on("data", (chunk) => {
|
|
9114
|
+
stderr += chunk;
|
|
9115
|
+
});
|
|
9116
|
+
child.once("error", (error2) => {
|
|
9117
|
+
finish({
|
|
9118
|
+
code: null,
|
|
9119
|
+
stdout,
|
|
9120
|
+
stderr: stderr === "" ? error2.message : `${stderr}
|
|
9121
|
+
${error2.message}`,
|
|
9122
|
+
timedOut: false
|
|
9123
|
+
});
|
|
9124
|
+
});
|
|
9125
|
+
child.once("close", (code) => finish({ code, stdout, stderr, timedOut: false }));
|
|
9126
|
+
timer.handle = setTimeout(
|
|
9127
|
+
() => {
|
|
9128
|
+
child.kill("SIGKILL");
|
|
9129
|
+
finish({ code: null, stdout, stderr, timedOut: true });
|
|
9130
|
+
},
|
|
9131
|
+
Math.max(0, opts.timeoutMs)
|
|
9132
|
+
);
|
|
9133
|
+
});
|
|
9134
|
+
};
|
|
9135
|
+
function isEnvironmentObject(value) {
|
|
9136
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9137
|
+
}
|
|
9138
|
+
function secretFailure(marker, detail, log3) {
|
|
9139
|
+
const message = `${marker}: ${detail}`;
|
|
9140
|
+
log3(message, "error");
|
|
9141
|
+
return new Error(message);
|
|
9142
|
+
}
|
|
9143
|
+
async function installRunnerSecret({
|
|
9144
|
+
env,
|
|
9145
|
+
log: log3,
|
|
9146
|
+
commandRunner
|
|
9147
|
+
}) {
|
|
9148
|
+
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
9149
|
+
if (!arn) {
|
|
9150
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
9151
|
+
return false;
|
|
9152
|
+
}
|
|
9153
|
+
const result = await (commandRunner ?? runCommand2)(
|
|
9154
|
+
"aws",
|
|
9155
|
+
[
|
|
9156
|
+
"secretsmanager",
|
|
9157
|
+
"get-secret-value",
|
|
9158
|
+
"--secret-id",
|
|
9159
|
+
arn,
|
|
9160
|
+
"--query",
|
|
9161
|
+
"SecretString",
|
|
9162
|
+
"--output",
|
|
9163
|
+
"text"
|
|
9164
|
+
],
|
|
9165
|
+
{ env, timeoutMs: RUNNER_SECRET_FETCH_TIMEOUT_MS }
|
|
9166
|
+
);
|
|
9167
|
+
if (result.timedOut) {
|
|
9168
|
+
throw secretFailure(
|
|
9169
|
+
"CREDENTIAL-RESTORE-TIMEOUT",
|
|
9170
|
+
`runner-secret did not finish within ${RUNNER_SECRET_FETCH_TIMEOUT_MS}ms`,
|
|
9171
|
+
log3
|
|
9172
|
+
);
|
|
9173
|
+
}
|
|
9174
|
+
if (result.code !== 0) {
|
|
9175
|
+
throw secretFailure("RUNNER-SECRET-UNREADABLE", commandError2(result), log3);
|
|
9176
|
+
}
|
|
9177
|
+
let payload;
|
|
9178
|
+
try {
|
|
9179
|
+
payload = JSON.parse(result.stdout);
|
|
9180
|
+
} catch (error2) {
|
|
9181
|
+
log3(
|
|
9182
|
+
`RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9183
|
+
"warn"
|
|
9184
|
+
);
|
|
9185
|
+
return false;
|
|
9186
|
+
}
|
|
9187
|
+
if (!isEnvironmentObject(payload)) {
|
|
9188
|
+
log3("RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object", "warn");
|
|
9189
|
+
return false;
|
|
9190
|
+
}
|
|
9191
|
+
let populated = 0;
|
|
9192
|
+
let skipped = 0;
|
|
9193
|
+
let githubTokenPopulated = false;
|
|
9194
|
+
for (const [key, value] of Object.entries(payload)) {
|
|
9195
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
9196
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
|
|
9197
|
+
log3(
|
|
9198
|
+
`RUNNER-SECRET-KEY-SKIPPED: ${JSON.stringify(key)} is not a valid environment variable name`,
|
|
9199
|
+
"warn"
|
|
9200
|
+
);
|
|
9201
|
+
skipped += 1;
|
|
9202
|
+
continue;
|
|
9203
|
+
}
|
|
9204
|
+
env[key] = value;
|
|
9205
|
+
populated += 1;
|
|
9206
|
+
if (key === "GH_TOKEN") githubTokenPopulated = true;
|
|
9207
|
+
}
|
|
9208
|
+
if (populated === 0) {
|
|
9209
|
+
log3(
|
|
9210
|
+
"RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md",
|
|
9211
|
+
"warn"
|
|
9212
|
+
);
|
|
9213
|
+
} else {
|
|
9214
|
+
log3(
|
|
9215
|
+
`RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names`
|
|
9216
|
+
);
|
|
9217
|
+
}
|
|
9218
|
+
return githubTokenPopulated;
|
|
9219
|
+
}
|
|
9220
|
+
function restoreFailure(operation, result, log3) {
|
|
9221
|
+
const message = `CREDENTIAL-RESTORE-FAILED: ${operation} failed: ${commandError2(result)}`;
|
|
9222
|
+
log3(message, "error");
|
|
9223
|
+
return new Error(message);
|
|
9224
|
+
}
|
|
9225
|
+
async function runCredentialRestore(args, operation, log3, synchroniserRunner) {
|
|
9226
|
+
const result = await synchroniserRunner(args, { timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS });
|
|
9227
|
+
if (result.timedOut) {
|
|
9228
|
+
log3(
|
|
9229
|
+
`CREDENTIAL-RESTORE-TIMEOUT: ${operation} did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9230
|
+
"warn"
|
|
9231
|
+
);
|
|
9232
|
+
return result;
|
|
9233
|
+
}
|
|
9234
|
+
if (result.code !== 0) throw restoreFailure(operation, result, log3);
|
|
9235
|
+
return result;
|
|
9236
|
+
}
|
|
9237
|
+
async function restoreCredentialStores({
|
|
9238
|
+
env,
|
|
9239
|
+
log: log3,
|
|
9240
|
+
synchroniserRunner = runSynchroniser
|
|
9241
|
+
}) {
|
|
9242
|
+
await runCredentialRestore(["restore", "claude"], "restore-claude", log3, synchroniserRunner);
|
|
9243
|
+
await runCredentialRestore(["restore", "opencode"], "restore-opencode", log3, synchroniserRunner);
|
|
9244
|
+
const result = await synchroniserRunner(["model-auth-ready"], {
|
|
9245
|
+
timeoutMs: CREDENTIAL_RESTORE_TIMEOUT_MS
|
|
9246
|
+
});
|
|
9247
|
+
if (result.timedOut) {
|
|
9248
|
+
log3(
|
|
9249
|
+
`CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within ${CREDENTIAL_RESTORE_TIMEOUT_MS}ms; continuing without it`,
|
|
9250
|
+
"warn"
|
|
9251
|
+
);
|
|
9252
|
+
return;
|
|
9253
|
+
}
|
|
9254
|
+
switch (result.code) {
|
|
9255
|
+
case 0:
|
|
9256
|
+
return;
|
|
9257
|
+
case 10:
|
|
9258
|
+
log3(
|
|
9259
|
+
`no model credentials under s3://${env.LITESTREAM_BUCKET ?? ""}/${env.LITESTREAM_PREFIX ?? ""}/ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs a model provider fails until one is connected. See 'Seeding a credential store' in infrastructure/evident-runner/MICROVM.md.`,
|
|
9260
|
+
"warn"
|
|
9261
|
+
);
|
|
9262
|
+
return;
|
|
9263
|
+
default:
|
|
9264
|
+
log3("could not determine whether this VM has model credentials", "warn");
|
|
9265
|
+
}
|
|
9266
|
+
}
|
|
9267
|
+
var GIT_CREDENTIAL_HELPER_CONTENT = [
|
|
9268
|
+
"#!/usr/bin/env bash",
|
|
9269
|
+
'[ "$1" = get ] || exit 0',
|
|
9270
|
+
"echo username=x-access-token",
|
|
9271
|
+
'echo "password=${GH_TOKEN}"',
|
|
9272
|
+
""
|
|
9273
|
+
].join("\n");
|
|
9274
|
+
async function probeGitHubAccess({ env, log: log3 }) {
|
|
9275
|
+
const auth = await runCommand2("gh", ["api", "user", "--jq", ".login"], {
|
|
9276
|
+
env,
|
|
9277
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9278
|
+
});
|
|
9279
|
+
if (auth.timedOut) {
|
|
9280
|
+
log3(`GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`, "warn");
|
|
9281
|
+
return;
|
|
9282
|
+
}
|
|
9283
|
+
if (auth.code !== 0) {
|
|
9284
|
+
log3(`GITHUB-AUTH-REJECTED: ${commandError2(auth)}`, "warn");
|
|
9285
|
+
return;
|
|
9286
|
+
}
|
|
9287
|
+
log3(`GITHUB-AUTH-OK: ${auth.stdout.trim()}`, "debug");
|
|
9288
|
+
const remote = await runCommand2("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
|
9289
|
+
env,
|
|
9290
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9291
|
+
});
|
|
9292
|
+
if (remote.code !== 0 || remote.timedOut) return;
|
|
9293
|
+
const repo = remote.stdout.trim().replace(/^(?:https:\/\/github\.com\/|git@github\.com:)/, "").replace(/\.git$/, "");
|
|
9294
|
+
if (!repo) return;
|
|
9295
|
+
const repository = await runCommand2("gh", ["api", `repos/${repo}`, "--jq", ".full_name"], {
|
|
9296
|
+
env,
|
|
9297
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9298
|
+
});
|
|
9299
|
+
if (repository.timedOut) {
|
|
9300
|
+
log3(
|
|
9301
|
+
`GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_TIMEOUT_MS}ms`,
|
|
9302
|
+
"warn"
|
|
9303
|
+
);
|
|
9304
|
+
} else if (repository.code !== 0) {
|
|
9305
|
+
log3(`GITHUB-REPO-INACCESSIBLE: ${repo}: ${commandError2(repository)}`, "warn");
|
|
9306
|
+
}
|
|
9307
|
+
}
|
|
9308
|
+
async function configureGitHubAccess({ env, log: log3 }) {
|
|
9309
|
+
if (!env.GH_TOKEN) {
|
|
9310
|
+
log3("GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above", "warn");
|
|
9311
|
+
return;
|
|
9312
|
+
}
|
|
9313
|
+
try {
|
|
9314
|
+
env.GIT_CONFIG_GLOBAL = GIT_CONFIG_GLOBAL;
|
|
9315
|
+
writeFileSync4(GIT_CONFIG_GLOBAL, "");
|
|
9316
|
+
writeFileSync4(GIT_CREDENTIAL_HELPER, GIT_CREDENTIAL_HELPER_CONTENT, { mode: 448 });
|
|
9317
|
+
chmodSync2(GIT_CREDENTIAL_HELPER, 448);
|
|
9318
|
+
const config = [
|
|
9319
|
+
["user.name", env.GIT_USER_NAME ?? "evident-bot"],
|
|
9320
|
+
["user.email", env.GIT_USER_EMAIL ?? "evident-bot@users.noreply.github.com"],
|
|
9321
|
+
["init.defaultBranch", "main"],
|
|
9322
|
+
["credential.https://github.com.helper", GIT_CREDENTIAL_HELPER]
|
|
9323
|
+
];
|
|
9324
|
+
for (const [key, value] of config) {
|
|
9325
|
+
const result = await runCommand2("git", ["config", "--global", key, value], {
|
|
9326
|
+
env,
|
|
9327
|
+
timeoutMs: GITHUB_PROBE_TIMEOUT_MS
|
|
9328
|
+
});
|
|
9329
|
+
if (result.timedOut || result.code !== 0) throw new Error(commandError2(result));
|
|
9330
|
+
}
|
|
9331
|
+
} catch (error2) {
|
|
9332
|
+
log3(
|
|
9333
|
+
`GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access (${error2 instanceof Error ? error2.message : String(error2)})`,
|
|
9334
|
+
"warn"
|
|
9335
|
+
);
|
|
9336
|
+
return;
|
|
9337
|
+
}
|
|
9338
|
+
void probeGitHubAccess({ env, log: log3 }).catch((error2) => {
|
|
9339
|
+
log3(
|
|
9340
|
+
`GITHUB-SETUP-FAILED: GitHub access probe failed: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
9341
|
+
"warn"
|
|
9342
|
+
);
|
|
9343
|
+
});
|
|
9344
|
+
}
|
|
9345
|
+
|
|
9346
|
+
// src/lib/opencode/config-overlay.ts
|
|
9347
|
+
import { execFileSync as execFileSync2 } from "child_process";
|
|
9348
|
+
import { copyFileSync, existsSync as existsSync2, statSync as statSync5 } from "fs";
|
|
9349
|
+
import { isAbsolute as isAbsolute2, join as join8, resolve as resolve3 } from "path";
|
|
9350
|
+
function isFile(filePath) {
|
|
9351
|
+
return existsSync2(filePath) && statSync5(filePath).isFile();
|
|
9352
|
+
}
|
|
9353
|
+
function applyRunnerOpenCodeConfig({
|
|
9354
|
+
overlayPath,
|
|
9355
|
+
cwd = process.cwd(),
|
|
9356
|
+
log: log3
|
|
9357
|
+
}) {
|
|
9358
|
+
if (!overlayPath) {
|
|
9359
|
+
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
9360
|
+
return;
|
|
9361
|
+
}
|
|
9362
|
+
const source = isAbsolute2(overlayPath) ? overlayPath : resolve3(cwd, overlayPath);
|
|
9363
|
+
const target = isFile(join8(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
9364
|
+
if (!isFile(source)) {
|
|
9365
|
+
log3(
|
|
9366
|
+
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
9367
|
+
"error"
|
|
9368
|
+
);
|
|
9369
|
+
return;
|
|
9370
|
+
}
|
|
9371
|
+
copyFileSync(source, join8(cwd, target));
|
|
9372
|
+
try {
|
|
9373
|
+
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
9374
|
+
stdio: "ignore"
|
|
9375
|
+
});
|
|
9376
|
+
} catch (error2) {
|
|
9377
|
+
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
9378
|
+
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
9379
|
+
}
|
|
9380
|
+
log3(`Applied runner OpenCode config ${source} to ${join8(cwd, target)}`);
|
|
9381
|
+
}
|
|
9382
|
+
|
|
9383
|
+
// src/lib/credential-sync.ts
|
|
9384
|
+
import { renameSync, writeFileSync as writeFileSync5 } from "fs";
|
|
9385
|
+
var CREDENTIAL_SYNC_TICK_TIMEOUT_MS = 3e4;
|
|
9386
|
+
var CREDENTIAL_FLUSH_DEADLINE_MS = 8e3;
|
|
9387
|
+
var CREDENTIAL_FLUSH_ABORT_GRACE_MS = 500;
|
|
9388
|
+
var DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS = 60;
|
|
9389
|
+
var STORES = ["claude", "opencode"];
|
|
9390
|
+
var MAX_FLUSH_PASSES = 2;
|
|
9391
|
+
function outcomesWith(outcome) {
|
|
9392
|
+
return { claude: outcome, opencode: outcome };
|
|
9393
|
+
}
|
|
9394
|
+
function errorMessage(error2) {
|
|
9395
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
9396
|
+
}
|
|
9397
|
+
function waitForSettlement(promise, timeoutMs) {
|
|
9398
|
+
return new Promise((resolve4) => {
|
|
9399
|
+
let settled = false;
|
|
9400
|
+
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
9401
|
+
const finish = (value) => {
|
|
9402
|
+
if (settled) return;
|
|
9403
|
+
settled = true;
|
|
9404
|
+
clearTimeout(timer);
|
|
9405
|
+
resolve4(value);
|
|
9406
|
+
};
|
|
9407
|
+
promise.then(
|
|
9408
|
+
() => finish(true),
|
|
9409
|
+
() => finish(true)
|
|
9410
|
+
);
|
|
9411
|
+
});
|
|
9412
|
+
}
|
|
9413
|
+
function writeMarker(markerPath, outcomes, log3) {
|
|
9414
|
+
const body = `${STORES.map((store) => `${store}=${outcomes[store]}`).join("\n")}
|
|
9415
|
+
`;
|
|
9416
|
+
const temporaryPath = `${markerPath}.tmp`;
|
|
9417
|
+
try {
|
|
9418
|
+
writeFileSync5(temporaryPath, body, { mode: 384 });
|
|
9419
|
+
renameSync(temporaryPath, markerPath);
|
|
9420
|
+
} catch (error2) {
|
|
9421
|
+
log3(`CREDENTIAL-FLUSH-MARKER-UNWRITABLE: ${markerPath}: ${errorMessage(error2)}`, "warn");
|
|
9422
|
+
}
|
|
9423
|
+
}
|
|
9424
|
+
function intervalSeconds(env, log3) {
|
|
9425
|
+
const raw = env.CREDS_SYNC_INTERVAL;
|
|
9426
|
+
if (raw === void 0 || /^[1-9][0-9]*$/.test(raw) && Number.isSafeInteger(Number(raw))) {
|
|
9427
|
+
return raw === void 0 ? DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS : Number(raw);
|
|
9428
|
+
}
|
|
9429
|
+
log3(
|
|
9430
|
+
`CREDENTIAL-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${raw}' is not a positive integer; using ${DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS}s`,
|
|
9431
|
+
"warn"
|
|
9432
|
+
);
|
|
9433
|
+
return DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS;
|
|
9434
|
+
}
|
|
9435
|
+
async function runFlushStore(store, env, synchroniserRunner, log3, deadlineAt) {
|
|
9436
|
+
const remainingMs = deadlineAt - Date.now();
|
|
9437
|
+
if (remainingMs <= 0) return { outcome: "timeout", orphaned: false };
|
|
9438
|
+
const controller = new AbortController();
|
|
9439
|
+
let result;
|
|
9440
|
+
let failed = false;
|
|
9441
|
+
const completion = Promise.resolve().then(
|
|
9442
|
+
() => synchroniserRunner(["sync-once", store], {
|
|
9443
|
+
timeoutMs: remainingMs,
|
|
9444
|
+
env,
|
|
9445
|
+
signal: controller.signal
|
|
9446
|
+
})
|
|
9447
|
+
).then(
|
|
9448
|
+
(value) => {
|
|
9449
|
+
result = value;
|
|
9450
|
+
},
|
|
9451
|
+
(error2) => {
|
|
9452
|
+
failed = true;
|
|
9453
|
+
log3(`CREDENTIAL-FLUSH-ERROR: ${store}: ${errorMessage(error2)}`, "warn");
|
|
9454
|
+
}
|
|
9455
|
+
);
|
|
9456
|
+
const abortTimer = setTimeout(() => controller.abort(), remainingMs);
|
|
9457
|
+
const settledBeforeDeadline = await waitForSettlement(completion, remainingMs);
|
|
9458
|
+
clearTimeout(abortTimer);
|
|
9459
|
+
if (!settledBeforeDeadline) {
|
|
9460
|
+
controller.abort();
|
|
9461
|
+
const settledAfterAbort = await waitForSettlement(completion, CREDENTIAL_FLUSH_ABORT_GRACE_MS);
|
|
9462
|
+
if (!settledAfterAbort) return { outcome: "timeout", orphaned: true };
|
|
9463
|
+
return { outcome: "timeout", orphaned: false };
|
|
9464
|
+
}
|
|
9465
|
+
if (failed || !result) return { outcome: "failed", orphaned: false };
|
|
9466
|
+
if (result.timedOut || Date.now() >= deadlineAt) {
|
|
9467
|
+
return { outcome: "timeout", orphaned: false };
|
|
9468
|
+
}
|
|
9469
|
+
return { outcome: result.code === 0 ? "ok" : "failed", orphaned: false };
|
|
9470
|
+
}
|
|
9471
|
+
function createCredentialSync({
|
|
9472
|
+
markerPath,
|
|
9473
|
+
env,
|
|
9474
|
+
log: log3,
|
|
9475
|
+
synchroniserRunner = runSynchroniser
|
|
9476
|
+
}) {
|
|
9477
|
+
const persistenceDisabled = !env.PERSISTENCE_BUCKET;
|
|
9478
|
+
let disabled = persistenceDisabled;
|
|
9479
|
+
let armed = false;
|
|
9480
|
+
let stopped = false;
|
|
9481
|
+
let timer;
|
|
9482
|
+
let inFlight;
|
|
9483
|
+
let activeTickAbort;
|
|
9484
|
+
let lastTickFailed;
|
|
9485
|
+
let flushPromise;
|
|
9486
|
+
const scheduleTick = (intervalMs, startTick2) => {
|
|
9487
|
+
if (stopped) return;
|
|
9488
|
+
timer = setTimeout(() => {
|
|
9489
|
+
timer = void 0;
|
|
9490
|
+
startTick2();
|
|
9491
|
+
}, intervalMs);
|
|
9492
|
+
};
|
|
9493
|
+
const startTick = (intervalMs) => {
|
|
9494
|
+
if (stopped) return;
|
|
9495
|
+
const controller = new AbortController();
|
|
9496
|
+
activeTickAbort = controller;
|
|
9497
|
+
const tick = (async () => {
|
|
9498
|
+
const outcomes = {
|
|
9499
|
+
claude: "failed",
|
|
9500
|
+
opencode: "failed"
|
|
9501
|
+
};
|
|
9502
|
+
for (const store of STORES) {
|
|
9503
|
+
if (controller.signal.aborted) break;
|
|
9504
|
+
try {
|
|
9505
|
+
const result = await synchroniserRunner(["sync-once", store], {
|
|
9506
|
+
timeoutMs: CREDENTIAL_SYNC_TICK_TIMEOUT_MS,
|
|
9507
|
+
env,
|
|
9508
|
+
signal: controller.signal
|
|
9509
|
+
});
|
|
9510
|
+
outcomes[store] = !result.timedOut && result.code === 0 ? "ok" : "failed";
|
|
9511
|
+
} catch (error2) {
|
|
9512
|
+
outcomes[store] = "failed";
|
|
9513
|
+
log3(`CREDENTIAL-SYNC-TICK-ERROR: ${store}: ${errorMessage(error2)}`, "debug");
|
|
9514
|
+
}
|
|
9515
|
+
}
|
|
9516
|
+
const failed = STORES.some((store) => outcomes[store] === "failed");
|
|
9517
|
+
log3(
|
|
9518
|
+
`CREDENTIAL-SYNC-TICK: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
9519
|
+
"debug"
|
|
9520
|
+
);
|
|
9521
|
+
if (failed && lastTickFailed !== true) {
|
|
9522
|
+
log3(
|
|
9523
|
+
"CREDENTIAL-SYNC-FAILED: an interval credential sync failed; retrying on the next tick",
|
|
9524
|
+
"warn"
|
|
9525
|
+
);
|
|
9526
|
+
} else if (!failed && lastTickFailed === true) {
|
|
9527
|
+
log3("CREDENTIAL-SYNC-RECOVERED: interval credential sync succeeded again", "warn");
|
|
9528
|
+
}
|
|
9529
|
+
lastTickFailed = failed;
|
|
9530
|
+
})().finally(() => {
|
|
9531
|
+
if (activeTickAbort === controller) activeTickAbort = void 0;
|
|
9532
|
+
if (inFlight === tick) inFlight = void 0;
|
|
9533
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9534
|
+
});
|
|
9535
|
+
inFlight = tick;
|
|
9536
|
+
};
|
|
9537
|
+
const performFlush = async () => {
|
|
9538
|
+
stopped = true;
|
|
9539
|
+
if (timer) {
|
|
9540
|
+
clearTimeout(timer);
|
|
9541
|
+
timer = void 0;
|
|
9542
|
+
}
|
|
9543
|
+
const deadlineAt = Date.now() + CREDENTIAL_FLUSH_DEADLINE_MS;
|
|
9544
|
+
if (inFlight) {
|
|
9545
|
+
const settled = await waitForSettlement(inFlight, CREDENTIAL_FLUSH_DEADLINE_MS);
|
|
9546
|
+
if (!settled) {
|
|
9547
|
+
activeTickAbort?.abort();
|
|
9548
|
+
const settledAfterAbort = await waitForSettlement(
|
|
9549
|
+
inFlight,
|
|
9550
|
+
CREDENTIAL_FLUSH_ABORT_GRACE_MS
|
|
9551
|
+
);
|
|
9552
|
+
if (!settledAfterAbort) {
|
|
9553
|
+
log3(
|
|
9554
|
+
"CREDENTIAL-FLUSH-ORPHANED-TICK: a sync-once child could not be proven settled; skipping the boundary flush rather than becoming a second writer",
|
|
9555
|
+
"warn"
|
|
9556
|
+
);
|
|
9557
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9558
|
+
}
|
|
9559
|
+
}
|
|
9560
|
+
}
|
|
9561
|
+
if (disabled) return { outcomes: outcomesWith("skipped"), orphaned: false };
|
|
9562
|
+
const outcomes = outcomesWith("timeout");
|
|
9563
|
+
for (const store of STORES) {
|
|
9564
|
+
const result = await runFlushStore(store, env, synchroniserRunner, log3, deadlineAt);
|
|
9565
|
+
if (result.orphaned) {
|
|
9566
|
+
log3(
|
|
9567
|
+
"CREDENTIAL-FLUSH-ORPHANED-CHILD: a flush sync-once child could not be proven settled; refusing another flush pass rather than becoming a second writer",
|
|
9568
|
+
"warn"
|
|
9569
|
+
);
|
|
9570
|
+
return { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9571
|
+
}
|
|
9572
|
+
outcomes[store] = result.outcome;
|
|
9573
|
+
}
|
|
9574
|
+
return { outcomes, orphaned: false };
|
|
9575
|
+
};
|
|
9576
|
+
let flushPasses = 0;
|
|
9577
|
+
let lastFlush;
|
|
9578
|
+
return {
|
|
9579
|
+
arm() {
|
|
9580
|
+
if (stopped || armed) return;
|
|
9581
|
+
armed = true;
|
|
9582
|
+
if (persistenceDisabled) {
|
|
9583
|
+
disabled = true;
|
|
9584
|
+
log3(
|
|
9585
|
+
"CREDENTIAL-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot",
|
|
9586
|
+
"warn"
|
|
9587
|
+
);
|
|
9588
|
+
return;
|
|
9589
|
+
}
|
|
9590
|
+
disabled = false;
|
|
9591
|
+
const intervalMs = intervalSeconds(env, log3) * 1e3;
|
|
9592
|
+
scheduleTick(intervalMs, () => startTick(intervalMs));
|
|
9593
|
+
},
|
|
9594
|
+
async stopAndFlush(publish) {
|
|
9595
|
+
let result;
|
|
9596
|
+
const runningFlush = flushPromise;
|
|
9597
|
+
if (runningFlush) {
|
|
9598
|
+
result = await runningFlush;
|
|
9599
|
+
} else if (flushPasses >= MAX_FLUSH_PASSES || lastFlush?.orphaned) {
|
|
9600
|
+
result = lastFlush ?? { outcomes: outcomesWith("timeout"), orphaned: true };
|
|
9601
|
+
} else {
|
|
9602
|
+
flushPasses++;
|
|
9603
|
+
const currentFlush = performFlush();
|
|
9604
|
+
flushPromise = currentFlush;
|
|
9605
|
+
try {
|
|
9606
|
+
result = await currentFlush;
|
|
9607
|
+
lastFlush = result;
|
|
9608
|
+
} finally {
|
|
9609
|
+
if (flushPromise === currentFlush) flushPromise = void 0;
|
|
9610
|
+
}
|
|
9611
|
+
}
|
|
9612
|
+
if (publish) writeMarker(markerPath, result.outcomes, log3);
|
|
9613
|
+
return result.outcomes;
|
|
9614
|
+
}
|
|
9615
|
+
};
|
|
9616
|
+
}
|
|
9617
|
+
|
|
7956
9618
|
// src/commands/run.ts
|
|
7957
9619
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
7958
9620
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
7959
9621
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
7960
9622
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
7961
9623
|
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
9624
|
+
var CHILD_STOP_TIMEOUT_MS = 1e4;
|
|
7962
9625
|
function resolveLogLevel(options) {
|
|
7963
9626
|
const accepted = Object.keys(LOG_LEVELS);
|
|
7964
9627
|
const validate = (value, source) => {
|
|
@@ -7989,11 +9652,11 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
7989
9652
|
if (trimmed === "") {
|
|
7990
9653
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
7991
9654
|
}
|
|
7992
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
7993
|
-
if (!
|
|
9655
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join9(homeDir, trimmed.slice(2)) : trimmed;
|
|
9656
|
+
if (!isAbsolute3(expanded)) {
|
|
7994
9657
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
7995
9658
|
}
|
|
7996
|
-
const normalized =
|
|
9659
|
+
const normalized = resolvePath2(expanded);
|
|
7997
9660
|
if (parse(normalized).root === normalized) {
|
|
7998
9661
|
throw new Error(
|
|
7999
9662
|
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
@@ -8109,7 +9772,7 @@ function logActivity(state, entry) {
|
|
|
8109
9772
|
}
|
|
8110
9773
|
function reportSessionDbRecovery(state) {
|
|
8111
9774
|
try {
|
|
8112
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
9775
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir5(), env: process.env });
|
|
8113
9776
|
for (const record of report.records) {
|
|
8114
9777
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
8115
9778
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -8127,6 +9790,16 @@ function reportSessionDbRecovery(state) {
|
|
|
8127
9790
|
);
|
|
8128
9791
|
}
|
|
8129
9792
|
}
|
|
9793
|
+
function reportSessionDbRecoveryRecord(state, record) {
|
|
9794
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
9795
|
+
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
9796
|
+
logActivity(state, {
|
|
9797
|
+
type: activity.level === "error" ? "error" : "info",
|
|
9798
|
+
level: activity.level,
|
|
9799
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
9800
|
+
metadata: activity.metadata
|
|
9801
|
+
});
|
|
9802
|
+
}
|
|
8130
9803
|
function displayStatus(state) {
|
|
8131
9804
|
if (!state.interactive) return;
|
|
8132
9805
|
const attempt = state.connection?.reconnectAttempt ?? 0;
|
|
@@ -8239,6 +9912,10 @@ async function driveChannels(state, driver) {
|
|
|
8239
9912
|
consecutiveDrainFailures = 0;
|
|
8240
9913
|
unreachableMs = 0;
|
|
8241
9914
|
state.messageCount += processed;
|
|
9915
|
+
if (driver.recycleRequested) {
|
|
9916
|
+
await beginGracefulShutdown(state, "recycle");
|
|
9917
|
+
return;
|
|
9918
|
+
}
|
|
8242
9919
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
8243
9920
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
8244
9921
|
const fileActivitySnapshot = driver.fileSyncActivity();
|
|
@@ -8281,8 +9958,8 @@ async function driveChannels(state, driver) {
|
|
|
8281
9958
|
state.running = false;
|
|
8282
9959
|
break;
|
|
8283
9960
|
}
|
|
8284
|
-
const
|
|
8285
|
-
logActivity(state, { type: "error", error: `Channel processing error: ${
|
|
9961
|
+
const errorMessage2 = error2 instanceof Error ? error2.message : String(error2);
|
|
9962
|
+
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage2}` });
|
|
8286
9963
|
if (state.interactive) displayStatus(state);
|
|
8287
9964
|
if (driver.hasInFlightWatchers()) {
|
|
8288
9965
|
consecutiveDrainFailures = 0;
|
|
@@ -8299,7 +9976,7 @@ async function driveChannels(state, driver) {
|
|
|
8299
9976
|
}
|
|
8300
9977
|
}
|
|
8301
9978
|
}
|
|
8302
|
-
await new Promise((
|
|
9979
|
+
await new Promise((resolve4) => setTimeout(resolve4, CHANNEL_POLL_INTERVAL_MS));
|
|
8303
9980
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
8304
9981
|
if (idleThisCycle) idleMs += cycleMs;
|
|
8305
9982
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -8322,7 +9999,43 @@ async function driveChannels(state, driver) {
|
|
|
8322
9999
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
8323
10000
|
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
8324
10001
|
function sessionDbPath() {
|
|
8325
|
-
return
|
|
10002
|
+
return join9(homedir5(), ".local", "share", "opencode", "opencode.db");
|
|
10003
|
+
}
|
|
10004
|
+
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
10005
|
+
const record = {
|
|
10006
|
+
v: 1,
|
|
10007
|
+
event: "session_db_recovery",
|
|
10008
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10009
|
+
stage: "verify",
|
|
10010
|
+
outcome: "schema_provenance_mismatch",
|
|
10011
|
+
severity: "error",
|
|
10012
|
+
reason: provenance.reason ?? "schema-provenance-mismatch",
|
|
10013
|
+
litestream_exit_code: null,
|
|
10014
|
+
attempt: null,
|
|
10015
|
+
replica_objects: null,
|
|
10016
|
+
replica_bytes: null,
|
|
10017
|
+
quarantine_destination: null,
|
|
10018
|
+
quarantined_objects: null,
|
|
10019
|
+
quarantine_failed_objects: null,
|
|
10020
|
+
quarantined_bytes: null,
|
|
10021
|
+
verified_restore_point: null,
|
|
10022
|
+
restore_points_tried: null,
|
|
10023
|
+
provenance_reason: provenance.reason,
|
|
10024
|
+
provenance_migration_delta: provenance.migrationDelta,
|
|
10025
|
+
replication_suspended: false,
|
|
10026
|
+
dbPath: sessionDbPath(),
|
|
10027
|
+
recorded_version: provenance.recordedVersion,
|
|
10028
|
+
current_version: currentVersion,
|
|
10029
|
+
provenance_pre_boot_migration_count: preBootMigrationCount
|
|
10030
|
+
};
|
|
10031
|
+
const activity = buildSessionDbRecoveryActivity(record);
|
|
10032
|
+
if (!activity) throw new Error("could not map session-DB provenance activity");
|
|
10033
|
+
logActivity(state, {
|
|
10034
|
+
type: activity.level === "error" ? "error" : "info",
|
|
10035
|
+
level: activity.level,
|
|
10036
|
+
...activity.level === "error" ? { error: activity.message } : { message: activity.message },
|
|
10037
|
+
metadata: activity.metadata
|
|
10038
|
+
});
|
|
8326
10039
|
}
|
|
8327
10040
|
async function runSweep(state, driver, config) {
|
|
8328
10041
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
@@ -8369,7 +10082,7 @@ async function runSweep(state, driver, config) {
|
|
|
8369
10082
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
8370
10083
|
dbPath: sessionDbPath(),
|
|
8371
10084
|
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
8372
|
-
allowFullVacuum: protectedNow.size === 0
|
|
10085
|
+
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
8373
10086
|
});
|
|
8374
10087
|
if (reclaimResult.ok) {
|
|
8375
10088
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
@@ -8405,7 +10118,7 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
8405
10118
|
for (const warning2 of config.warnings) {
|
|
8406
10119
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
8407
10120
|
}
|
|
8408
|
-
const dbBytes = statSessionDbBytes(
|
|
10121
|
+
const dbBytes = statSessionDbBytes(homedir5());
|
|
8409
10122
|
void (async () => {
|
|
8410
10123
|
const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
8411
10124
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -8606,7 +10319,8 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
8606
10319
|
});
|
|
8607
10320
|
return;
|
|
8608
10321
|
}
|
|
8609
|
-
const collect = createResourceUsageCollector(
|
|
10322
|
+
const { collect, stop } = createResourceUsageCollector(homedir5());
|
|
10323
|
+
state.stopResourceUsageSampling = stop;
|
|
8610
10324
|
let consecutiveFailures = 0;
|
|
8611
10325
|
const tick = async () => {
|
|
8612
10326
|
try {
|
|
@@ -8716,21 +10430,41 @@ async function cleanup(state, opts = {}) {
|
|
|
8716
10430
|
clearTimeout(state.resourceUsageTimer);
|
|
8717
10431
|
state.resourceUsageTimer = null;
|
|
8718
10432
|
}
|
|
10433
|
+
state.stopResourceUsageSampling?.();
|
|
10434
|
+
state.stopResourceUsageSampling = null;
|
|
10435
|
+
const credentialSync = state.credentialSync;
|
|
10436
|
+
const flushCredentials = credentialSync ? async (phase, publish) => {
|
|
10437
|
+
await timeShutdownPhase(state, durations, phase, async () => {
|
|
10438
|
+
const outcomes = await credentialSync.stopAndFlush(publish);
|
|
10439
|
+
const level = Object.values(outcomes).every((outcome) => outcome === "ok") ? "info" : "warn";
|
|
10440
|
+
log2(
|
|
10441
|
+
state,
|
|
10442
|
+
`Credential flush: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
|
|
10443
|
+
level
|
|
10444
|
+
);
|
|
10445
|
+
});
|
|
10446
|
+
} : void 0;
|
|
10447
|
+
let drainSettled = true;
|
|
8719
10448
|
if (opts.graceful && state.channelDriver) {
|
|
8720
10449
|
state.channelDriver.stop();
|
|
10450
|
+
}
|
|
10451
|
+
if (flushCredentials) {
|
|
10452
|
+
await flushCredentials("credential_flush", !(opts.graceful && state.channelDriver));
|
|
10453
|
+
}
|
|
10454
|
+
if (opts.graceful && state.channelDriver) {
|
|
8721
10455
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
8722
10456
|
if (state.interactive) {
|
|
8723
10457
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
8724
10458
|
displayStatus(state);
|
|
8725
10459
|
}
|
|
8726
10460
|
const driver = state.channelDriver;
|
|
8727
|
-
|
|
10461
|
+
drainSettled = await timeShutdownPhase(
|
|
8728
10462
|
state,
|
|
8729
10463
|
durations,
|
|
8730
10464
|
"drain",
|
|
8731
10465
|
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
8732
10466
|
);
|
|
8733
|
-
if (!
|
|
10467
|
+
if (!drainSettled) {
|
|
8734
10468
|
logActivity(state, {
|
|
8735
10469
|
type: "info",
|
|
8736
10470
|
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
@@ -8738,6 +10472,9 @@ async function cleanup(state, opts = {}) {
|
|
|
8738
10472
|
if (state.interactive) displayStatus(state);
|
|
8739
10473
|
}
|
|
8740
10474
|
}
|
|
10475
|
+
if (opts.graceful && state.channelDriver && flushCredentials && drainSettled) {
|
|
10476
|
+
await flushCredentials("credential_flush_final", true);
|
|
10477
|
+
}
|
|
8741
10478
|
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
8742
10479
|
if (state.connection) {
|
|
8743
10480
|
const connection = state.connection;
|
|
@@ -8746,24 +10483,83 @@ async function cleanup(state, opts = {}) {
|
|
|
8746
10483
|
}
|
|
8747
10484
|
if (state.opencodeProcess) {
|
|
8748
10485
|
const opencodeProcess = state.opencodeProcess;
|
|
8749
|
-
|
|
10486
|
+
const result = await timeShutdownPhase(
|
|
10487
|
+
state,
|
|
10488
|
+
durations,
|
|
10489
|
+
"opencode_stop",
|
|
10490
|
+
() => stopOpenCodeAndWait(opencodeProcess, CHILD_STOP_TIMEOUT_MS)
|
|
10491
|
+
);
|
|
8750
10492
|
if (state.interactive) {
|
|
8751
|
-
logActivity(state, { type: "info", message:
|
|
10493
|
+
logActivity(state, { type: "info", message: `Stopped OpenCode process (${result.outcome})` });
|
|
8752
10494
|
displayStatus(state);
|
|
8753
10495
|
} else {
|
|
8754
|
-
log2(state,
|
|
10496
|
+
log2(state, `Stopped OpenCode process (${result.outcome})`);
|
|
8755
10497
|
}
|
|
8756
10498
|
state.opencodeProcess = null;
|
|
8757
10499
|
}
|
|
10500
|
+
if (state.litestreamProcess) {
|
|
10501
|
+
const litestreamProcess = state.litestreamProcess;
|
|
10502
|
+
const result = await timeShutdownPhase(
|
|
10503
|
+
state,
|
|
10504
|
+
durations,
|
|
10505
|
+
"litestream_stop",
|
|
10506
|
+
() => stopSessionDbReplication(litestreamProcess, CHILD_STOP_TIMEOUT_MS)
|
|
10507
|
+
);
|
|
10508
|
+
log2(state, `Stopped litestream replication (${result.outcome})`);
|
|
10509
|
+
state.litestreamProcess = null;
|
|
10510
|
+
}
|
|
8758
10511
|
return durations;
|
|
8759
10512
|
}
|
|
10513
|
+
async function beginGracefulShutdown(state, trigger) {
|
|
10514
|
+
if (state.shuttingDown) return;
|
|
10515
|
+
state.shuttingDown = true;
|
|
10516
|
+
const shutdownStartedAt = Date.now();
|
|
10517
|
+
const shutdownMessage = trigger === "recycle" ? "Recycle requested \u2014 finishing in-flight work, then exiting" : "Shutting down...";
|
|
10518
|
+
if (state.interactive) {
|
|
10519
|
+
logActivity(state, { type: "info", message: shutdownMessage });
|
|
10520
|
+
displayStatus(state);
|
|
10521
|
+
} else {
|
|
10522
|
+
log2(state, shutdownMessage);
|
|
10523
|
+
}
|
|
10524
|
+
const durations = await cleanup(state, { graceful: true });
|
|
10525
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
10526
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
10527
|
+
let timer;
|
|
10528
|
+
const flushed = shutdownTelemetry().then(
|
|
10529
|
+
() => true,
|
|
10530
|
+
(error2) => {
|
|
10531
|
+
log2(
|
|
10532
|
+
state,
|
|
10533
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10534
|
+
"warn"
|
|
10535
|
+
);
|
|
10536
|
+
return true;
|
|
10537
|
+
}
|
|
10538
|
+
);
|
|
10539
|
+
const timedOut = new Promise((resolve4) => {
|
|
10540
|
+
timer = setTimeout(() => resolve4(false), telemetryBudgetMs);
|
|
10541
|
+
});
|
|
10542
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
10543
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
10544
|
+
}
|
|
10545
|
+
clearTimeout(timer);
|
|
10546
|
+
});
|
|
10547
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
10548
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
10549
|
+
process.exit(0);
|
|
10550
|
+
}
|
|
8760
10551
|
async function run(options) {
|
|
8761
10552
|
const interactive = isInteractive(options.json);
|
|
8762
10553
|
let logLevel;
|
|
8763
10554
|
let fileSyncDirectories;
|
|
8764
10555
|
try {
|
|
8765
10556
|
logLevel = resolveLogLevel(options);
|
|
8766
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
10557
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir5());
|
|
10558
|
+
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
10559
|
+
throw new Error(
|
|
10560
|
+
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
10561
|
+
);
|
|
10562
|
+
}
|
|
8767
10563
|
} catch (error2) {
|
|
8768
10564
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
8769
10565
|
if (options.json) {
|
|
@@ -8787,7 +10583,9 @@ async function run(options) {
|
|
|
8787
10583
|
connected: false,
|
|
8788
10584
|
opencodeConnected: false,
|
|
8789
10585
|
opencodeVersion: null,
|
|
10586
|
+
sessionDbProvenanceAnomaly: false,
|
|
8790
10587
|
opencodeProcess: null,
|
|
10588
|
+
litestreamProcess: null,
|
|
8791
10589
|
connection: null,
|
|
8792
10590
|
channelDriver: null,
|
|
8793
10591
|
running: true,
|
|
@@ -8801,9 +10599,24 @@ async function run(options) {
|
|
|
8801
10599
|
openaiUsageTimer: null,
|
|
8802
10600
|
openaiUsageRearm: null,
|
|
8803
10601
|
resourceUsageTimer: null,
|
|
10602
|
+
stopResourceUsageSampling: null,
|
|
10603
|
+
credentialSync: null,
|
|
8804
10604
|
authHeader: ""
|
|
8805
10605
|
};
|
|
8806
10606
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
10607
|
+
if (options.credentialSyncMarker) {
|
|
10608
|
+
state.credentialSync = createCredentialSync({
|
|
10609
|
+
markerPath: options.credentialSyncMarker,
|
|
10610
|
+
env: process.env,
|
|
10611
|
+
log: (message, level = "info") => {
|
|
10612
|
+
if (level === "error") {
|
|
10613
|
+
logActivity(state, { type: "error", error: message });
|
|
10614
|
+
} else {
|
|
10615
|
+
logActivity(state, { type: "info", level, message });
|
|
10616
|
+
}
|
|
10617
|
+
}
|
|
10618
|
+
});
|
|
10619
|
+
}
|
|
8807
10620
|
if (fileSyncDirectories.length > 0) {
|
|
8808
10621
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
8809
10622
|
} else {
|
|
@@ -8829,43 +10642,7 @@ async function run(options) {
|
|
|
8829
10642
|
"warn"
|
|
8830
10643
|
);
|
|
8831
10644
|
}
|
|
8832
|
-
const handleSignal =
|
|
8833
|
-
if (state.shuttingDown) return;
|
|
8834
|
-
state.shuttingDown = true;
|
|
8835
|
-
const shutdownStartedAt = Date.now();
|
|
8836
|
-
if (state.interactive) {
|
|
8837
|
-
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
8838
|
-
displayStatus(state);
|
|
8839
|
-
} else {
|
|
8840
|
-
log2(state, "Shutting down...");
|
|
8841
|
-
}
|
|
8842
|
-
const durations = await cleanup(state, { graceful: true });
|
|
8843
|
-
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
8844
|
-
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
8845
|
-
let timer;
|
|
8846
|
-
const flushed = shutdownTelemetry().then(
|
|
8847
|
-
() => true,
|
|
8848
|
-
(error2) => {
|
|
8849
|
-
log2(
|
|
8850
|
-
state,
|
|
8851
|
-
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8852
|
-
"warn"
|
|
8853
|
-
);
|
|
8854
|
-
return true;
|
|
8855
|
-
}
|
|
8856
|
-
);
|
|
8857
|
-
const timedOut = new Promise((resolve3) => {
|
|
8858
|
-
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
8859
|
-
});
|
|
8860
|
-
if (!await Promise.race([flushed, timedOut])) {
|
|
8861
|
-
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
8862
|
-
}
|
|
8863
|
-
clearTimeout(timer);
|
|
8864
|
-
});
|
|
8865
|
-
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
8866
|
-
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
8867
|
-
process.exit(0);
|
|
8868
|
-
};
|
|
10645
|
+
const handleSignal = () => beginGracefulShutdown(state, "signal");
|
|
8869
10646
|
process.on("SIGINT", handleSignal);
|
|
8870
10647
|
process.on("SIGTERM", handleSignal);
|
|
8871
10648
|
try {
|
|
@@ -8995,7 +10772,68 @@ async function run(options) {
|
|
|
8995
10772
|
} else {
|
|
8996
10773
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
8997
10774
|
}
|
|
10775
|
+
if (options.restoreRunnerCredentials) {
|
|
10776
|
+
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
10777
|
+
const credentialContext = {
|
|
10778
|
+
env: process.env,
|
|
10779
|
+
log: (message, level = "info") => {
|
|
10780
|
+
if (level === "error") {
|
|
10781
|
+
logActivity(state, { type: "error", error: message });
|
|
10782
|
+
} else {
|
|
10783
|
+
logActivity(state, { type: "info", level, message });
|
|
10784
|
+
}
|
|
10785
|
+
}
|
|
10786
|
+
};
|
|
10787
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
10788
|
+
await restoreCredentialStores(credentialContext);
|
|
10789
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
10790
|
+
}
|
|
10791
|
+
state.credentialSync?.arm();
|
|
10792
|
+
let sessionDbVerifyFatal = false;
|
|
10793
|
+
if (!options.restoreSessionDb) {
|
|
10794
|
+
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
10795
|
+
} else {
|
|
10796
|
+
const health = await checkOpenCodeHealth(state.port);
|
|
10797
|
+
if (health.healthy) {
|
|
10798
|
+
log2(
|
|
10799
|
+
state,
|
|
10800
|
+
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
10801
|
+
"debug"
|
|
10802
|
+
);
|
|
10803
|
+
} else {
|
|
10804
|
+
const result = await restoreAndVerifySessionDb({
|
|
10805
|
+
dbPath: sessionDbPath(),
|
|
10806
|
+
litestreamConfig: options.litestreamConfig,
|
|
10807
|
+
noReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
10808
|
+
env: process.env,
|
|
10809
|
+
log: (message, level = "info") => {
|
|
10810
|
+
if (level === "error") {
|
|
10811
|
+
logActivity(state, { type: "error", error: message });
|
|
10812
|
+
} else {
|
|
10813
|
+
logActivity(state, { type: "info", level, message });
|
|
10814
|
+
}
|
|
10815
|
+
},
|
|
10816
|
+
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
10817
|
+
});
|
|
10818
|
+
sessionDbVerifyFatal = result.verifyFatal;
|
|
10819
|
+
}
|
|
10820
|
+
}
|
|
8998
10821
|
reportSessionDbRecovery(state);
|
|
10822
|
+
if (sessionDbVerifyFatal) {
|
|
10823
|
+
throw new Error(
|
|
10824
|
+
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
10825
|
+
);
|
|
10826
|
+
}
|
|
10827
|
+
applyRunnerOpenCodeConfig({
|
|
10828
|
+
overlayPath: options.opencodeConfigOverlay,
|
|
10829
|
+
log: (message, level = "info") => {
|
|
10830
|
+
if (level === "error") {
|
|
10831
|
+
logActivity(state, { type: "error", error: message });
|
|
10832
|
+
} else {
|
|
10833
|
+
logActivity(state, { type: "info", level, message });
|
|
10834
|
+
}
|
|
10835
|
+
}
|
|
10836
|
+
});
|
|
8999
10837
|
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
9000
10838
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
9001
10839
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
@@ -9004,6 +10842,7 @@ async function run(options) {
|
|
|
9004
10842
|
for (const warning2 of maxActiveSessionsWarnings) {
|
|
9005
10843
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
9006
10844
|
}
|
|
10845
|
+
const preBootMigrationIds = readSessionDbMigrationIds(sessionDbPath());
|
|
9007
10846
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
9008
10847
|
try {
|
|
9009
10848
|
const oc = await ensureOpenCodeRunning({
|
|
@@ -9011,11 +10850,41 @@ async function run(options) {
|
|
|
9011
10850
|
interactive: state.interactive,
|
|
9012
10851
|
agentId: state.agentId,
|
|
9013
10852
|
log: (message) => log2(state, message),
|
|
9014
|
-
startTimeoutMs: opencodeStartTimeoutMs
|
|
10853
|
+
startTimeoutMs: opencodeStartTimeoutMs,
|
|
10854
|
+
inheritStdio: Boolean(options.opencodePidFile)
|
|
9015
10855
|
});
|
|
9016
10856
|
state.port = oc.port;
|
|
9017
|
-
state.opencodeProcess = oc.process;
|
|
10857
|
+
state.opencodeProcess = options.opencodePidFile ? null : oc.process;
|
|
9018
10858
|
state.opencodeVersion = oc.version;
|
|
10859
|
+
if (options.opencodePidFile && oc.process?.pid !== void 0) {
|
|
10860
|
+
try {
|
|
10861
|
+
writeFileSync6(options.opencodePidFile, `${oc.process.pid}
|
|
10862
|
+
`, { mode: 384 });
|
|
10863
|
+
chmodSync3(options.opencodePidFile, 384);
|
|
10864
|
+
} catch (error2) {
|
|
10865
|
+
logActivity(state, {
|
|
10866
|
+
type: "error",
|
|
10867
|
+
error: `Failed to write OpenCode pid file ${options.opencodePidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10868
|
+
});
|
|
10869
|
+
}
|
|
10870
|
+
}
|
|
10871
|
+
if (state.opencodeVersion !== null) {
|
|
10872
|
+
const provenance = checkSessionDbProvenance({
|
|
10873
|
+
dbPath: sessionDbPath(),
|
|
10874
|
+
currentVersion: state.opencodeVersion,
|
|
10875
|
+
homeDir: homedir5(),
|
|
10876
|
+
env: process.env
|
|
10877
|
+
});
|
|
10878
|
+
if (provenance.anomaly) {
|
|
10879
|
+
state.sessionDbProvenanceAnomaly = true;
|
|
10880
|
+
logSessionDbProvenanceMismatch(
|
|
10881
|
+
state,
|
|
10882
|
+
provenance,
|
|
10883
|
+
state.opencodeVersion,
|
|
10884
|
+
preBootMigrationIds?.length ?? null
|
|
10885
|
+
);
|
|
10886
|
+
}
|
|
10887
|
+
}
|
|
9019
10888
|
state.opencodeConnected = oc.notReadyReason === null;
|
|
9020
10889
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
9021
10890
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
@@ -9052,6 +10921,108 @@ async function run(options) {
|
|
|
9052
10921
|
ocSpinner?.fail(error2.message);
|
|
9053
10922
|
throw error2;
|
|
9054
10923
|
}
|
|
10924
|
+
if (options.litestreamPidFile) {
|
|
10925
|
+
if (options.sessionDbNoReplicateMarker && existsSync3(options.sessionDbNoReplicateMarker)) {
|
|
10926
|
+
log2(
|
|
10927
|
+
state,
|
|
10928
|
+
`Skipping Litestream replication because ${options.sessionDbNoReplicateMarker} marks this session database unsafe to replicate`
|
|
10929
|
+
);
|
|
10930
|
+
} else if (!options.litestreamConfig) {
|
|
10931
|
+
logActivity(state, {
|
|
10932
|
+
type: "info",
|
|
10933
|
+
level: "warn",
|
|
10934
|
+
message: "Skipping Litestream replication because no configuration file was provided"
|
|
10935
|
+
});
|
|
10936
|
+
} else {
|
|
10937
|
+
let existingPid;
|
|
10938
|
+
if (existsSync3(options.litestreamPidFile)) {
|
|
10939
|
+
try {
|
|
10940
|
+
const rawPid = readFileSync6(options.litestreamPidFile, "utf8").trim();
|
|
10941
|
+
const parsedPid = Number(rawPid);
|
|
10942
|
+
if (/^\d+$/.test(rawPid) && Number.isSafeInteger(parsedPid) && parsedPid > 0) {
|
|
10943
|
+
existingPid = parsedPid;
|
|
10944
|
+
}
|
|
10945
|
+
} catch (error2) {
|
|
10946
|
+
logActivity(state, {
|
|
10947
|
+
type: "info",
|
|
10948
|
+
level: "warn",
|
|
10949
|
+
message: `Could not read Litestream pid file ${options.litestreamPidFile}; checking for a new process: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10950
|
+
});
|
|
10951
|
+
}
|
|
10952
|
+
}
|
|
10953
|
+
if (existingPid !== void 0 && isProcessAlive(existingPid)) {
|
|
10954
|
+
log2(state, `Litestream replication is already running with pid ${existingPid}`);
|
|
10955
|
+
} else {
|
|
10956
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
10957
|
+
state.litestreamProcess = null;
|
|
10958
|
+
let failureHandled = false;
|
|
10959
|
+
const reportImageOwnedReplicationFailure = (message) => {
|
|
10960
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
10961
|
+
failureHandled = true;
|
|
10962
|
+
logActivity(state, { type: "error", error: message });
|
|
10963
|
+
if (state.interactive) displayStatus(state);
|
|
10964
|
+
};
|
|
10965
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
10966
|
+
reportImageOwnedReplicationFailure(
|
|
10967
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
10968
|
+
);
|
|
10969
|
+
});
|
|
10970
|
+
litestreamProcess.on("error", (error2) => {
|
|
10971
|
+
reportImageOwnedReplicationFailure(
|
|
10972
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10973
|
+
);
|
|
10974
|
+
});
|
|
10975
|
+
try {
|
|
10976
|
+
if (litestreamProcess.pid !== void 0) {
|
|
10977
|
+
writeFileSync6(options.litestreamPidFile, `${litestreamProcess.pid}
|
|
10978
|
+
`, {
|
|
10979
|
+
mode: 384
|
|
10980
|
+
});
|
|
10981
|
+
chmodSync3(options.litestreamPidFile, 384);
|
|
10982
|
+
}
|
|
10983
|
+
} catch (error2) {
|
|
10984
|
+
logActivity(state, {
|
|
10985
|
+
type: "error",
|
|
10986
|
+
error: `Failed to write Litestream pid file ${options.litestreamPidFile}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
10987
|
+
});
|
|
10988
|
+
}
|
|
10989
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
10990
|
+
}
|
|
10991
|
+
}
|
|
10992
|
+
} else if (options.litestreamConfig) {
|
|
10993
|
+
const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
|
|
10994
|
+
state.litestreamProcess = litestreamProcess;
|
|
10995
|
+
let failureHandled = false;
|
|
10996
|
+
const failRunForReplication = (message) => {
|
|
10997
|
+
if (failureHandled || state.shuttingDown || !state.running) return;
|
|
10998
|
+
failureHandled = true;
|
|
10999
|
+
state.shuttingDown = true;
|
|
11000
|
+
logActivity(state, { type: "error", error: message });
|
|
11001
|
+
if (state.interactive) displayStatus(state);
|
|
11002
|
+
void (async () => {
|
|
11003
|
+
try {
|
|
11004
|
+
await cleanup(state);
|
|
11005
|
+
await shutdownTelemetry();
|
|
11006
|
+
} catch (error2) {
|
|
11007
|
+
console.error(
|
|
11008
|
+
`[run] replication failure cleanup failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11009
|
+
);
|
|
11010
|
+
}
|
|
11011
|
+
process.exit(1);
|
|
11012
|
+
})();
|
|
11013
|
+
};
|
|
11014
|
+
litestreamProcess.on("exit", (code, signal) => {
|
|
11015
|
+
failRunForReplication(
|
|
11016
|
+
`Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
|
|
11017
|
+
);
|
|
11018
|
+
});
|
|
11019
|
+
litestreamProcess.on("error", (error2) => {
|
|
11020
|
+
failRunForReplication(
|
|
11021
|
+
`Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
11022
|
+
);
|
|
11023
|
+
});
|
|
11024
|
+
log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
|
|
11025
|
+
}
|
|
9055
11026
|
const tunnelSpinner = interactive && !state.json ? ora3("Connecting tunnel...").start() : null;
|
|
9056
11027
|
const channelDriver = new ChannelDriver({
|
|
9057
11028
|
agentId: state.agentId,
|
|
@@ -9063,7 +11034,7 @@ async function run(options) {
|
|
|
9063
11034
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
9064
11035
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
9065
11036
|
fileSyncDirectories,
|
|
9066
|
-
homeDir:
|
|
11037
|
+
homeDir: homedir5(),
|
|
9067
11038
|
maxActiveSessions,
|
|
9068
11039
|
log: (entry) => (
|
|
9069
11040
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -9189,6 +11160,18 @@ async function run(options) {
|
|
|
9189
11160
|
if (state.interactive) displayStatus(state);
|
|
9190
11161
|
});
|
|
9191
11162
|
},
|
|
11163
|
+
// Both loops are rearmed because `rearm()` is idempotent for the
|
|
11164
|
+
// provider that did not just connect, and is a no-op when reporting is off.
|
|
11165
|
+
onUsageRearmPing: () => {
|
|
11166
|
+
if (!state.running) return;
|
|
11167
|
+
logActivity(state, {
|
|
11168
|
+
type: "info",
|
|
11169
|
+
level: "debug",
|
|
11170
|
+
message: "Usage rearm ping received"
|
|
11171
|
+
});
|
|
11172
|
+
state.claudeUsageRearm?.();
|
|
11173
|
+
state.openaiUsageRearm?.();
|
|
11174
|
+
},
|
|
9192
11175
|
onInfo: (message) => logActivity(state, { type: "info", message })
|
|
9193
11176
|
}
|
|
9194
11177
|
});
|
|
@@ -9209,7 +11192,17 @@ async function run(options) {
|
|
|
9209
11192
|
setTimer: (timer) => {
|
|
9210
11193
|
state.openaiUsageTimer = timer;
|
|
9211
11194
|
},
|
|
9212
|
-
fetchUsage: () =>
|
|
11195
|
+
fetchUsage: async () => {
|
|
11196
|
+
const usage = await getOpenAiUsage(state.port);
|
|
11197
|
+
if (usage.subscription === null) {
|
|
11198
|
+
logActivity(state, {
|
|
11199
|
+
type: "info",
|
|
11200
|
+
level: "debug",
|
|
11201
|
+
message: "OpenAI usage subscription could not be identified from the local credential"
|
|
11202
|
+
});
|
|
11203
|
+
}
|
|
11204
|
+
return usage;
|
|
11205
|
+
},
|
|
9213
11206
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
9214
11207
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
9215
11208
|
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
@@ -9255,7 +11248,7 @@ async function run(options) {
|
|
|
9255
11248
|
}
|
|
9256
11249
|
|
|
9257
11250
|
// src/index.ts
|
|
9258
|
-
var { version } =
|
|
11251
|
+
var { version } = createRequire2(import.meta.url)("../package.json");
|
|
9259
11252
|
var program = new Command();
|
|
9260
11253
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
9261
11254
|
"--endpoint <url>",
|
|
@@ -9312,6 +11305,30 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9312
11305
|
).option(
|
|
9313
11306
|
"--tunnel-ready-file <path>",
|
|
9314
11307
|
"Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
|
|
11308
|
+
).option(
|
|
11309
|
+
"--litestream-config <path>",
|
|
11310
|
+
"Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
|
|
11311
|
+
).option(
|
|
11312
|
+
"--opencode-pid-file <path>",
|
|
11313
|
+
"Record the OpenCode pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11314
|
+
).option(
|
|
11315
|
+
"--litestream-pid-file <path>",
|
|
11316
|
+
"Record the Litestream pid here and leave the process running at shutdown for the runner image's lifecycle hooks to stop."
|
|
11317
|
+
).option(
|
|
11318
|
+
"--session-db-no-replicate-marker <path>",
|
|
11319
|
+
"Read this marker as a gate before starting Litestream; the session-DB restore writes it when this boot's database is not safe to replicate."
|
|
11320
|
+
).option(
|
|
11321
|
+
"--restore-session-db",
|
|
11322
|
+
"Restore and verify the OpenCode session database before starting OpenCode; requires --litestream-config for the replica and --session-db-no-replicate-marker for give-up records."
|
|
11323
|
+
).option(
|
|
11324
|
+
"--restore-runner-credentials",
|
|
11325
|
+
"Restore the hosted runner secret and persisted credential stores before starting OpenCode."
|
|
11326
|
+
).option(
|
|
11327
|
+
"--opencode-config-overlay <path>",
|
|
11328
|
+
"Apply this runner-provided OpenCode config before starting OpenCode."
|
|
11329
|
+
).option(
|
|
11330
|
+
"--credential-sync-marker <path>",
|
|
11331
|
+
"Own the interval credential sync and write this marker once the shutdown flush has finished, so the runner image's lifecycle hooks can wait on it."
|
|
9315
11332
|
).action(
|
|
9316
11333
|
(options) => {
|
|
9317
11334
|
run({
|
|
@@ -9343,7 +11360,15 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
9343
11360
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
9344
11361
|
// resolveFileSyncDirectories.
|
|
9345
11362
|
enableFileSyncTo: options.enableFileSyncTo,
|
|
9346
|
-
tunnelReadyFile: options.tunnelReadyFile
|
|
11363
|
+
tunnelReadyFile: options.tunnelReadyFile,
|
|
11364
|
+
litestreamConfig: options.litestreamConfig,
|
|
11365
|
+
opencodePidFile: options.opencodePidFile,
|
|
11366
|
+
litestreamPidFile: options.litestreamPidFile,
|
|
11367
|
+
sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
|
|
11368
|
+
restoreSessionDb: options.restoreSessionDb,
|
|
11369
|
+
restoreRunnerCredentials: options.restoreRunnerCredentials,
|
|
11370
|
+
opencodeConfigOverlay: options.opencodeConfigOverlay,
|
|
11371
|
+
credentialSyncMarker: options.credentialSyncMarker
|
|
9347
11372
|
});
|
|
9348
11373
|
}
|
|
9349
11374
|
);
|