@evident-ai/cli 3.5.2-dev.2bb04b2 → 3.5.2-dev.35de651
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/index.js +910 -255
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -703,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
703
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
704
704
|
});
|
|
705
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
706
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
707
732
|
const apiUrl = getApiUrlConfig();
|
|
708
733
|
try {
|
|
@@ -877,6 +902,76 @@ async function status(options = {}) {
|
|
|
877
902
|
process.exit(result.exitCode);
|
|
878
903
|
}
|
|
879
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// src/lib/claude-usage.ts
|
|
881
976
|
import { execFileSync } from "node:child_process";
|
|
882
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1227,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1227
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1228
1323
|
|
|
1229
1324
|
// src/lib/telemetry.ts
|
|
1230
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.35de651" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1231
1326
|
function getCliVersion() {
|
|
1232
1327
|
return CLI_VERSION;
|
|
1233
1328
|
}
|
|
@@ -1388,20 +1483,47 @@ var EventTypes = {
|
|
|
1388
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1389
1484
|
};
|
|
1390
1485
|
|
|
1391
|
-
// src/lib/runner-activity-telemetry.ts
|
|
1392
|
-
import { homedir as homedir2 } from "node:os";
|
|
1393
|
-
|
|
1394
1486
|
// src/lib/opencode/session-db-reclaim.ts
|
|
1395
1487
|
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1396
1488
|
import { dirname as dirname2 } from "node:path";
|
|
1397
|
-
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1491
|
+
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1518
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1398
1519
|
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1399
1520
|
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1521
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1400
1522
|
function errorMessage(error2) {
|
|
1401
1523
|
if (!(error2 instanceof Error)) return String(error2);
|
|
1402
1524
|
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1403
1525
|
}
|
|
1404
|
-
function
|
|
1526
|
+
function errorMessages(error2) {
|
|
1405
1527
|
const messages = [];
|
|
1406
1528
|
const visited = /* @__PURE__ */ new Set();
|
|
1407
1529
|
let current = error2;
|
|
@@ -1425,12 +1547,81 @@ function isSessionStoreWriteFailure(error2) {
|
|
|
1425
1547
|
}
|
|
1426
1548
|
break;
|
|
1427
1549
|
}
|
|
1428
|
-
return messages
|
|
1550
|
+
return messages;
|
|
1551
|
+
}
|
|
1552
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1553
|
+
return errorMessages(error2).some(
|
|
1429
1554
|
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1430
1555
|
message
|
|
1431
1556
|
)
|
|
1432
1557
|
);
|
|
1433
1558
|
}
|
|
1559
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1560
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1561
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1562
|
+
function isCauseBearingText(text) {
|
|
1563
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1564
|
+
}
|
|
1565
|
+
function structuredLogFields(line) {
|
|
1566
|
+
const fields = [];
|
|
1567
|
+
for (let index = 0; index < line.length; index++) {
|
|
1568
|
+
if (line[index] === '"') {
|
|
1569
|
+
index++;
|
|
1570
|
+
while (index < line.length) {
|
|
1571
|
+
if (line[index] === "\\") {
|
|
1572
|
+
index++;
|
|
1573
|
+
} else if (line[index] === '"') {
|
|
1574
|
+
break;
|
|
1575
|
+
}
|
|
1576
|
+
index++;
|
|
1577
|
+
}
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1581
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1582
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1583
|
+
const valueStart = index + key.length;
|
|
1584
|
+
if (line[valueStart] !== '"') {
|
|
1585
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1586
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1587
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
let valueEnd = valueStart + 1;
|
|
1591
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1592
|
+
if (line[valueEnd] === "\\") {
|
|
1593
|
+
valueEnd++;
|
|
1594
|
+
} else if (line[valueEnd] === '"') {
|
|
1595
|
+
break;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1599
|
+
index = valueEnd;
|
|
1600
|
+
}
|
|
1601
|
+
return fields;
|
|
1602
|
+
}
|
|
1603
|
+
function structuredCauseMessages(text) {
|
|
1604
|
+
const fields = structuredLogFields(text);
|
|
1605
|
+
if (fields.length > 0) return fields;
|
|
1606
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1607
|
+
}
|
|
1608
|
+
function causeTokenFromText(text) {
|
|
1609
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1610
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1611
|
+
const match = pattern.exec(text);
|
|
1612
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1613
|
+
return match[1];
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
return null;
|
|
1617
|
+
}
|
|
1618
|
+
function sessionStoreCauseToken(value) {
|
|
1619
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1620
|
+
const token = causeTokenFromText(message);
|
|
1621
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1622
|
+
}
|
|
1623
|
+
return null;
|
|
1624
|
+
}
|
|
1434
1625
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1435
1626
|
try {
|
|
1436
1627
|
const fsStats = statfsSync(dirname2(dbPath));
|
|
@@ -1452,6 +1643,7 @@ function readFreelistCount(db) {
|
|
|
1452
1643
|
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1453
1644
|
}
|
|
1454
1645
|
function readCheckpointResult(db) {
|
|
1646
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1455
1647
|
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1456
1648
|
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1457
1649
|
}
|
|
@@ -1469,7 +1661,7 @@ async function probeReclaimAvailability(input) {
|
|
|
1469
1661
|
try {
|
|
1470
1662
|
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1471
1663
|
try {
|
|
1472
|
-
db.exec(
|
|
1664
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1473
1665
|
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1474
1666
|
} finally {
|
|
1475
1667
|
db.close();
|
|
@@ -1489,7 +1681,8 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1489
1681
|
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1490
1682
|
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1491
1683
|
now = () => performance.now(),
|
|
1492
|
-
|
|
1684
|
+
hasLiveTurn = false,
|
|
1685
|
+
hasProvenanceAnomaly = false
|
|
1493
1686
|
} = input;
|
|
1494
1687
|
let sqlite;
|
|
1495
1688
|
try {
|
|
@@ -1505,17 +1698,19 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1505
1698
|
let db;
|
|
1506
1699
|
try {
|
|
1507
1700
|
db = new DatabaseSync(dbPath);
|
|
1508
|
-
db.exec(
|
|
1701
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1509
1702
|
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1510
1703
|
if (autoVacuum === 0) {
|
|
1511
|
-
if (
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1704
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1705
|
+
const detail = [
|
|
1706
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1707
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1708
|
+
].join(" and ");
|
|
1709
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1515
1710
|
return {
|
|
1516
1711
|
ok: false,
|
|
1517
1712
|
skipped: "full-vacuum-blocked",
|
|
1518
|
-
detail
|
|
1713
|
+
detail
|
|
1519
1714
|
};
|
|
1520
1715
|
}
|
|
1521
1716
|
const fileBytesForGuard = statSync2(dbPath).size;
|
|
@@ -1533,7 +1728,9 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1533
1728
|
db.exec("VACUUM");
|
|
1534
1729
|
const afterBytes = readLogicalBytes(db);
|
|
1535
1730
|
const freelistAfter = readFreelistCount(db);
|
|
1731
|
+
const checkpointStartedAt = now();
|
|
1536
1732
|
const checkpoint = readCheckpointResult(db);
|
|
1733
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1537
1734
|
return {
|
|
1538
1735
|
ok: true,
|
|
1539
1736
|
mode: "convert",
|
|
@@ -1544,10 +1741,18 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1544
1741
|
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1545
1742
|
stoppedBy: "freelist-drained",
|
|
1546
1743
|
elapsedMs: Math.max(0, now() - startedAt),
|
|
1744
|
+
checkpointElapsedMs,
|
|
1547
1745
|
checkpoint
|
|
1548
1746
|
};
|
|
1549
1747
|
}
|
|
1550
1748
|
if (autoVacuum === 2) {
|
|
1749
|
+
if (hasLiveTurn) {
|
|
1750
|
+
const detail = "a session turn is live";
|
|
1751
|
+
console.warn(
|
|
1752
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1753
|
+
);
|
|
1754
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1755
|
+
}
|
|
1551
1756
|
const beforeBytes = readLogicalBytes(db);
|
|
1552
1757
|
const freelistBefore = readFreelistCount(db);
|
|
1553
1758
|
const startedAt = now();
|
|
@@ -1572,7 +1777,9 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1572
1777
|
freelistAfter = readFreelistCount(db);
|
|
1573
1778
|
}
|
|
1574
1779
|
const afterBytes = readLogicalBytes(db);
|
|
1780
|
+
const checkpointStartedAt = now();
|
|
1575
1781
|
const checkpoint = readCheckpointResult(db);
|
|
1782
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1576
1783
|
return {
|
|
1577
1784
|
ok: true,
|
|
1578
1785
|
mode: "incremental",
|
|
@@ -1583,6 +1790,7 @@ async function reclaimSessionDbSpace(input) {
|
|
|
1583
1790
|
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1584
1791
|
stoppedBy,
|
|
1585
1792
|
elapsedMs: Math.max(0, now() - startedAt),
|
|
1793
|
+
checkpointElapsedMs,
|
|
1586
1794
|
checkpoint
|
|
1587
1795
|
};
|
|
1588
1796
|
}
|
|
@@ -1664,16 +1872,22 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
|
|
|
1664
1872
|
}
|
|
1665
1873
|
};
|
|
1666
1874
|
const flushLine = (line) => {
|
|
1667
|
-
|
|
1875
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1876
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1877
|
+
const cause = sessionStoreCauseToken(line);
|
|
1878
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1668
1879
|
try {
|
|
1669
|
-
onSessionStoreWriteFailure(
|
|
1880
|
+
onSessionStoreWriteFailure({
|
|
1881
|
+
isDiskFailure,
|
|
1882
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1883
|
+
cause
|
|
1884
|
+
});
|
|
1670
1885
|
} catch (err) {
|
|
1671
1886
|
reportTelemetryDiagnostic(
|
|
1672
1887
|
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1673
1888
|
);
|
|
1674
1889
|
}
|
|
1675
1890
|
}
|
|
1676
|
-
const parsed = parseOpenCodeLogLine(line);
|
|
1677
1891
|
if (!parsed) return;
|
|
1678
1892
|
const now = Date.now();
|
|
1679
1893
|
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
@@ -1722,34 +1936,21 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
|
|
|
1722
1936
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1723
1937
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1724
1938
|
var MAX_METADATA_ENTRIES = 20;
|
|
1725
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1726
|
-
function redactHomeDirectory(message) {
|
|
1727
|
-
const home = homedir2();
|
|
1728
|
-
if (!home || home === "/") {
|
|
1729
|
-
return message;
|
|
1730
|
-
}
|
|
1731
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1732
|
-
let result = "";
|
|
1733
|
-
let searchFrom = 0;
|
|
1734
|
-
let homeIndex;
|
|
1735
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1736
|
-
const homeEnd = homeIndex + home.length;
|
|
1737
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1738
|
-
const nextCharacter = message[homeEnd];
|
|
1739
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1740
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1741
|
-
result += atPathBoundary ? "~" : home;
|
|
1742
|
-
searchFrom = homeEnd;
|
|
1743
|
-
}
|
|
1744
|
-
return result + message.slice(searchFrom);
|
|
1745
|
-
}
|
|
1746
|
-
function redactDiagnosticText(message) {
|
|
1747
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1748
|
-
return redactHomeDirectory(redacted);
|
|
1749
|
-
}
|
|
1750
1939
|
function truncate(message) {
|
|
1751
1940
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1752
|
-
|
|
1941
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1942
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1943
|
+
let previousDropped = -1;
|
|
1944
|
+
while (dropped !== previousDropped) {
|
|
1945
|
+
previousDropped = dropped;
|
|
1946
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1947
|
+
dropped = nextDropped;
|
|
1948
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1949
|
+
}
|
|
1950
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1951
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1952
|
+
const tailLength = remaining - headLength;
|
|
1953
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1753
1954
|
}
|
|
1754
1955
|
function sanitiseMetadata(metadata) {
|
|
1755
1956
|
if (!metadata) return {};
|
|
@@ -1959,12 +2160,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1959
2160
|
metadata: withoutContractFields(record),
|
|
1960
2161
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1961
2162
|
};
|
|
1962
|
-
case "session_db_boot_refused":
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
2163
|
+
case "session_db_boot_refused": {
|
|
2164
|
+
const message = (() => {
|
|
2165
|
+
switch (record.reason) {
|
|
2166
|
+
case "local_discard_failed":
|
|
2167
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2168
|
+
case "replica_separation_unproven":
|
|
2169
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2170
|
+
case "verification_timeout_before_separation":
|
|
2171
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2172
|
+
case "verification_timeout_during_separation":
|
|
2173
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2174
|
+
default:
|
|
2175
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2176
|
+
}
|
|
2177
|
+
})();
|
|
2178
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2179
|
+
}
|
|
1968
2180
|
case "schema_provenance_mismatch":
|
|
1969
2181
|
return {
|
|
1970
2182
|
level,
|
|
@@ -2250,8 +2462,12 @@ function errorOf(m) {
|
|
|
2250
2462
|
if (!m || typeof m !== "object") return void 0;
|
|
2251
2463
|
return m.info?.error ?? m.error;
|
|
2252
2464
|
}
|
|
2465
|
+
function isTerminalStopAnswer(m) {
|
|
2466
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2467
|
+
}
|
|
2253
2468
|
function isAssistantInFlight(m) {
|
|
2254
2469
|
if (completedOf(m) == null) return true;
|
|
2470
|
+
if (errorOf(m) != null) return false;
|
|
2255
2471
|
return finishOf(m) === "tool-calls";
|
|
2256
2472
|
}
|
|
2257
2473
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2365,7 +2581,18 @@ async function deleteSession(port, id, client) {
|
|
|
2365
2581
|
{ method: "DELETE" },
|
|
2366
2582
|
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2367
2583
|
);
|
|
2368
|
-
|
|
2584
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2585
|
+
return { ok: true, status: res.status };
|
|
2586
|
+
}
|
|
2587
|
+
try {
|
|
2588
|
+
const body = await res.text();
|
|
2589
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2590
|
+
} catch (error2) {
|
|
2591
|
+
console.error(
|
|
2592
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2593
|
+
);
|
|
2594
|
+
return { ok: false, status: res.status };
|
|
2595
|
+
}
|
|
2369
2596
|
} catch (error2) {
|
|
2370
2597
|
console.error(
|
|
2371
2598
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
@@ -2608,11 +2835,31 @@ function messageText(m) {
|
|
|
2608
2835
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2609
2836
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2610
2837
|
}
|
|
2611
|
-
|
|
2838
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2839
|
+
if (!messages || messages.length === 0) return null;
|
|
2840
|
+
let best = null;
|
|
2841
|
+
for (const message of messages) {
|
|
2842
|
+
if (roleOf2(message) !== "user") continue;
|
|
2843
|
+
const id = idOf2(message);
|
|
2844
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2845
|
+
if (messageText(message) !== content) continue;
|
|
2846
|
+
const created = createdOf(message) ?? 0;
|
|
2847
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2848
|
+
}
|
|
2849
|
+
return best?.id ?? null;
|
|
2850
|
+
}
|
|
2851
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2612
2852
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2613
2853
|
const knownUserIds = new Set(
|
|
2614
2854
|
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2615
2855
|
);
|
|
2856
|
+
if (callbacks) {
|
|
2857
|
+
try {
|
|
2858
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2859
|
+
} catch (error2) {
|
|
2860
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2616
2863
|
const parts = [{ type: "text", text: content }];
|
|
2617
2864
|
let pendingOutcomes = null;
|
|
2618
2865
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2651,22 +2898,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2651
2898
|
const READ_BACK_DELAY_MS = 150;
|
|
2652
2899
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2653
2900
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
const id = idOf2(m);
|
|
2659
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2660
|
-
if (messageText(m) !== content) continue;
|
|
2661
|
-
const created = createdOf(m) ?? 0;
|
|
2662
|
-
if (best === null || created > best.created) {
|
|
2663
|
-
best = { id, created };
|
|
2664
|
-
}
|
|
2665
|
-
}
|
|
2666
|
-
if (best) {
|
|
2667
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2668
|
-
return best.id;
|
|
2669
|
-
}
|
|
2901
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2902
|
+
if (dispatchedUserMessageId !== null) {
|
|
2903
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2904
|
+
return dispatchedUserMessageId;
|
|
2670
2905
|
}
|
|
2671
2906
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2672
2907
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2690,30 +2925,41 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
2690
2925
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2691
2926
|
if (!messages || messages.length === 0) return null;
|
|
2692
2927
|
let lastCorrelated = null;
|
|
2693
|
-
let lastNonErrored = null;
|
|
2694
2928
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2695
2929
|
const m = messages[i];
|
|
2696
2930
|
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2931
|
+
lastCorrelated = m;
|
|
2932
|
+
break;
|
|
2933
|
+
}
|
|
2934
|
+
if (lastCorrelated) {
|
|
2935
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2936
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2937
|
+
const m = messages[i];
|
|
2938
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2939
|
+
return m;
|
|
2940
|
+
}
|
|
2701
2941
|
}
|
|
2942
|
+
return lastCorrelated;
|
|
2702
2943
|
}
|
|
2703
|
-
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
2704
2944
|
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2705
2945
|
if (userIndex === -1) return null;
|
|
2706
2946
|
let last = null;
|
|
2707
|
-
let
|
|
2947
|
+
let lastAssistantIndex = -1;
|
|
2708
2948
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2709
2949
|
const role = roleOf2(messages[i]);
|
|
2710
2950
|
if (role === "user") break;
|
|
2711
2951
|
if (role === "assistant") {
|
|
2712
2952
|
last = messages[i];
|
|
2713
|
-
|
|
2953
|
+
lastAssistantIndex = i;
|
|
2714
2954
|
}
|
|
2715
2955
|
}
|
|
2716
|
-
|
|
2956
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2957
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2958
|
+
const m = messages[i];
|
|
2959
|
+
if (roleOf2(m) === "user") break;
|
|
2960
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2961
|
+
}
|
|
2962
|
+
return last;
|
|
2717
2963
|
}
|
|
2718
2964
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2719
2965
|
if (!messages || messages.length === 0) return [];
|
|
@@ -2819,8 +3065,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2819
3065
|
}
|
|
2820
3066
|
function sumAssistantUsage(messages) {
|
|
2821
3067
|
if (!messages || messages.length === 0) return null;
|
|
2822
|
-
const
|
|
2823
|
-
const
|
|
3068
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3069
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3070
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2824
3071
|
let sawAnyUsage = false;
|
|
2825
3072
|
let inputSum = 0;
|
|
2826
3073
|
let outputSum = 0;
|
|
@@ -2876,11 +3123,9 @@ function messageUsage(messages, userMessageId) {
|
|
|
2876
3123
|
const byParentAll = messages.filter(
|
|
2877
3124
|
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2878
3125
|
);
|
|
2879
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2880
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2881
3126
|
let correlated;
|
|
2882
|
-
if (
|
|
2883
|
-
correlated =
|
|
3127
|
+
if (byParentAll.length > 0) {
|
|
3128
|
+
correlated = byParentAll;
|
|
2884
3129
|
} else {
|
|
2885
3130
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2886
3131
|
correlated = reply ? [reply] : [];
|
|
@@ -2926,14 +3171,38 @@ function messageError(messages, userMessageId) {
|
|
|
2926
3171
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2927
3172
|
const error2 = errorOf(reply);
|
|
2928
3173
|
if (error2 == null) return null;
|
|
2929
|
-
|
|
2930
|
-
if (typeof error2 === "
|
|
3174
|
+
let rendered;
|
|
3175
|
+
if (typeof error2 === "string") {
|
|
3176
|
+
rendered = error2;
|
|
3177
|
+
} else if (typeof error2 === "object") {
|
|
2931
3178
|
const e = error2;
|
|
2932
3179
|
const dataMessage = e.data?.message;
|
|
2933
|
-
if (typeof dataMessage === "string")
|
|
2934
|
-
|
|
3180
|
+
if (typeof dataMessage === "string") {
|
|
3181
|
+
rendered = dataMessage;
|
|
3182
|
+
} else if (typeof e.message === "string") {
|
|
3183
|
+
rendered = e.message;
|
|
3184
|
+
} else {
|
|
3185
|
+
rendered = "The agent run failed.";
|
|
3186
|
+
}
|
|
3187
|
+
} else {
|
|
3188
|
+
rendered = "The agent run failed.";
|
|
2935
3189
|
}
|
|
2936
|
-
|
|
3190
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3191
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3192
|
+
) : void 0;
|
|
3193
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3194
|
+
}
|
|
3195
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3196
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3197
|
+
if (!reply) return null;
|
|
3198
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3199
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3200
|
+
return {
|
|
3201
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3202
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3203
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3204
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3205
|
+
};
|
|
2937
3206
|
}
|
|
2938
3207
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2939
3208
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -3508,7 +3777,18 @@ async function deleteV2Session(client, sessionId) {
|
|
|
3508
3777
|
{ method: "DELETE" },
|
|
3509
3778
|
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3510
3779
|
);
|
|
3511
|
-
|
|
3780
|
+
if (response.status >= 200 && response.status < 300) {
|
|
3781
|
+
return { ok: true, status: response.status };
|
|
3782
|
+
}
|
|
3783
|
+
try {
|
|
3784
|
+
const body = await response.text();
|
|
3785
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
3786
|
+
} catch (error2) {
|
|
3787
|
+
console.error(
|
|
3788
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3789
|
+
);
|
|
3790
|
+
return { ok: false, status: response.status };
|
|
3791
|
+
}
|
|
3512
3792
|
} catch (error2) {
|
|
3513
3793
|
console.error(
|
|
3514
3794
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
@@ -4217,9 +4497,12 @@ function runSynchroniser(args, opts) {
|
|
|
4217
4497
|
|
|
4218
4498
|
// src/lib/opencode/session-db-boot.ts
|
|
4219
4499
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
4220
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4500
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
4221
4501
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
4222
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4502
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4503
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4504
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4505
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
4223
4506
|
function commandError(result) {
|
|
4224
4507
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
4225
4508
|
}
|
|
@@ -4530,20 +4813,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4530
4813
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4531
4814
|
env: {
|
|
4532
4815
|
...env,
|
|
4533
|
-
// The synchroniser reads
|
|
4534
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4816
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4535
4817
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4536
4818
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4819
|
+
),
|
|
4820
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4821
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4822
|
+
),
|
|
4823
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4824
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4825
|
+
),
|
|
4826
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4827
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4537
4828
|
)
|
|
4538
4829
|
}
|
|
4539
4830
|
});
|
|
4540
4831
|
logSynchroniserDiagnostics(result, options);
|
|
4541
4832
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4833
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4834
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4835
|
+
options.log(
|
|
4836
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4837
|
+
"error"
|
|
4838
|
+
);
|
|
4839
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4840
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4841
|
+
}
|
|
4542
4842
|
options.log(
|
|
4543
4843
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4544
4844
|
"warn"
|
|
4545
4845
|
);
|
|
4546
|
-
return false;
|
|
4846
|
+
return { verifyFatal: false };
|
|
4547
4847
|
}
|
|
4548
4848
|
if (result.code === 34) {
|
|
4549
4849
|
reportRecord(
|
|
@@ -4553,14 +4853,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4553
4853
|
null,
|
|
4554
4854
|
options
|
|
4555
4855
|
);
|
|
4556
|
-
return true;
|
|
4856
|
+
return { verifyFatal: true };
|
|
4557
4857
|
}
|
|
4558
4858
|
if (result.code === 33) {
|
|
4559
4859
|
options.log(
|
|
4560
4860
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4561
4861
|
"warn"
|
|
4562
4862
|
);
|
|
4563
|
-
return false;
|
|
4863
|
+
return { verifyFatal: false };
|
|
4564
4864
|
}
|
|
4565
4865
|
if (result.code !== 0) {
|
|
4566
4866
|
options.log(
|
|
@@ -4568,13 +4868,13 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4568
4868
|
"warn"
|
|
4569
4869
|
);
|
|
4570
4870
|
}
|
|
4571
|
-
return false;
|
|
4871
|
+
return { verifyFatal: false };
|
|
4572
4872
|
}
|
|
4573
4873
|
options.log(
|
|
4574
4874
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4575
4875
|
"debug"
|
|
4576
4876
|
);
|
|
4577
|
-
return false;
|
|
4877
|
+
return { verifyFatal: false };
|
|
4578
4878
|
}
|
|
4579
4879
|
function fileExists(path) {
|
|
4580
4880
|
try {
|
|
@@ -4661,7 +4961,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4661
4961
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4662
4962
|
return { verifyFatal: false };
|
|
4663
4963
|
}
|
|
4664
|
-
return
|
|
4964
|
+
return verifySessionDb(options, configPath, env);
|
|
4665
4965
|
}
|
|
4666
4966
|
|
|
4667
4967
|
// src/lib/opencode/session-db-provenance.ts
|
|
@@ -6442,6 +6742,7 @@ function createSessionStoreHealth(options = {}) {
|
|
|
6442
6742
|
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6443
6743
|
let lastUnwritableAt = null;
|
|
6444
6744
|
let lastWritableAt = null;
|
|
6745
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6445
6746
|
return {
|
|
6446
6747
|
observeUnwritable: () => {
|
|
6447
6748
|
lastUnwritableAt = now();
|
|
@@ -6453,6 +6754,23 @@ function createSessionStoreHealth(options = {}) {
|
|
|
6453
6754
|
if (lastUnwritableAt === null) return false;
|
|
6454
6755
|
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6455
6756
|
return now() - lastUnwritableAt < stalenessMs;
|
|
6757
|
+
},
|
|
6758
|
+
observeCause: (sessionId, cause) => {
|
|
6759
|
+
if (sessionId === null) return;
|
|
6760
|
+
causes.delete(sessionId);
|
|
6761
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6762
|
+
while (causes.size > 256) {
|
|
6763
|
+
const oldest = causes.keys().next().value;
|
|
6764
|
+
if (typeof oldest !== "string") break;
|
|
6765
|
+
causes.delete(oldest);
|
|
6766
|
+
}
|
|
6767
|
+
},
|
|
6768
|
+
readCause: (sessionId) => {
|
|
6769
|
+
const observed = causes.get(sessionId);
|
|
6770
|
+
if (!observed) return null;
|
|
6771
|
+
causes.delete(sessionId);
|
|
6772
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6773
|
+
return observed.cause;
|
|
6456
6774
|
}
|
|
6457
6775
|
};
|
|
6458
6776
|
}
|
|
@@ -6893,6 +7211,11 @@ function messageIdOf(m) {
|
|
|
6893
7211
|
const infoId = m.info?.id;
|
|
6894
7212
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6895
7213
|
}
|
|
7214
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7215
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7216
|
+
if (replyShape === null) return base;
|
|
7217
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7218
|
+
}
|
|
6896
7219
|
function cleanImageMime(contentType) {
|
|
6897
7220
|
if (!contentType) return null;
|
|
6898
7221
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6913,6 +7236,10 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6913
7236
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6914
7237
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6915
7238
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7239
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7240
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7241
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7242
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6916
7243
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6917
7244
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
6918
7245
|
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
@@ -7070,6 +7397,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7070
7397
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
7071
7398
|
*/
|
|
7072
7399
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7400
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
7073
7401
|
/**
|
|
7074
7402
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
7075
7403
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -7102,10 +7430,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7102
7430
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
7103
7431
|
*/
|
|
7104
7432
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7433
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7434
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
7105
7435
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
7106
7436
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
7107
7437
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
7108
7438
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7439
|
+
/**
|
|
7440
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7441
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7442
|
+
* identical-text turn, so it never produces a correlation.
|
|
7443
|
+
*/
|
|
7444
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
7109
7445
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
7110
7446
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
7111
7447
|
/**
|
|
@@ -7191,21 +7527,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7191
7527
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
7192
7528
|
*/
|
|
7193
7529
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
7194
|
-
/**
|
|
7195
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
7196
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
7197
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
7198
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
7199
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
7200
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
7201
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
7202
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
7203
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
7204
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
7205
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
7206
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
7207
|
-
*/
|
|
7208
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
7209
7530
|
/**
|
|
7210
7531
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
7211
7532
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -7223,8 +7544,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7223
7544
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
7224
7545
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
7225
7546
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
7226
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
7227
|
-
*
|
|
7547
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7548
|
+
* the terminal-report fence and is never sent again).
|
|
7228
7549
|
*/
|
|
7229
7550
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
7230
7551
|
/**
|
|
@@ -7542,6 +7863,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7542
7863
|
}
|
|
7543
7864
|
return ids;
|
|
7544
7865
|
}
|
|
7866
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7867
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7868
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7869
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7870
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7871
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7872
|
+
}
|
|
7873
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7874
|
+
}
|
|
7875
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7876
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7877
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7878
|
+
this.awaitingReadopt.delete(messageId);
|
|
7879
|
+
this.sessions.delete(conversationId);
|
|
7880
|
+
this.supersede(conversationId, sessionId);
|
|
7881
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7882
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7883
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7884
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7885
|
+
this.log({
|
|
7886
|
+
level: "error",
|
|
7887
|
+
message: errorMessage3,
|
|
7888
|
+
conversation_id: conversationId,
|
|
7889
|
+
message_id: messageId
|
|
7890
|
+
});
|
|
7891
|
+
const reportError = await this.reportUntrackableAck(
|
|
7892
|
+
conversationId,
|
|
7893
|
+
messageId,
|
|
7894
|
+
null,
|
|
7895
|
+
errorMessage3
|
|
7896
|
+
);
|
|
7897
|
+
if (reportError !== null) {
|
|
7898
|
+
this.log({
|
|
7899
|
+
level: "warn",
|
|
7900
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7901
|
+
conversation_id: conversationId,
|
|
7902
|
+
message_id: messageId
|
|
7903
|
+
});
|
|
7904
|
+
}
|
|
7905
|
+
if (source === "readopt") {
|
|
7906
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7907
|
+
}
|
|
7908
|
+
}
|
|
7545
7909
|
/**
|
|
7546
7910
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7547
7911
|
*
|
|
@@ -7721,6 +8085,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7721
8085
|
}
|
|
7722
8086
|
break;
|
|
7723
8087
|
}
|
|
8088
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8089
|
+
if (correlation) {
|
|
8090
|
+
correlation.attempts += 1;
|
|
8091
|
+
let correlatedId = null;
|
|
8092
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8093
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8094
|
+
conv,
|
|
8095
|
+
message,
|
|
8096
|
+
correlation.sessionId
|
|
8097
|
+
);
|
|
8098
|
+
if (polled.ok)
|
|
8099
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8100
|
+
} else if (correlation.knownUserIds === null) {
|
|
8101
|
+
this.log({
|
|
8102
|
+
level: "debug",
|
|
8103
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8104
|
+
conversation_id: conv.id,
|
|
8105
|
+
message_id: message.id
|
|
8106
|
+
});
|
|
8107
|
+
}
|
|
8108
|
+
if (correlatedId !== null) {
|
|
8109
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8110
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8111
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8112
|
+
this.dispatched.add(message.id);
|
|
8113
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8114
|
+
dispatched += 1;
|
|
8115
|
+
this.log({
|
|
8116
|
+
level: "info",
|
|
8117
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8118
|
+
conversation_id: conv.id,
|
|
8119
|
+
message_id: message.id
|
|
8120
|
+
});
|
|
8121
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8122
|
+
opencode_message_id: correlatedId
|
|
8123
|
+
});
|
|
8124
|
+
continue;
|
|
8125
|
+
}
|
|
8126
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8127
|
+
await this.handoffUntrackableCorrelation(
|
|
8128
|
+
conv.id,
|
|
8129
|
+
message.id,
|
|
8130
|
+
correlation.sessionId,
|
|
8131
|
+
correlation.attempts,
|
|
8132
|
+
"fresh"
|
|
8133
|
+
);
|
|
8134
|
+
break;
|
|
8135
|
+
}
|
|
8136
|
+
break;
|
|
8137
|
+
}
|
|
7724
8138
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7725
8139
|
if (effectiveOpencodeMessageId) {
|
|
7726
8140
|
const outcome = await this.resolveRedrive(
|
|
@@ -7742,6 +8156,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7742
8156
|
model: message.opencode_model ?? void 0
|
|
7743
8157
|
};
|
|
7744
8158
|
let opencodeMessageId;
|
|
8159
|
+
let preDispatchUserIds = null;
|
|
7745
8160
|
try {
|
|
7746
8161
|
this.log({
|
|
7747
8162
|
level: "info",
|
|
@@ -7771,7 +8186,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7771
8186
|
message.content,
|
|
7772
8187
|
options,
|
|
7773
8188
|
sendAttachments,
|
|
7774
|
-
this.openCodeClient
|
|
8189
|
+
this.openCodeClient,
|
|
8190
|
+
{
|
|
8191
|
+
onKnownUserIds: (ids) => {
|
|
8192
|
+
preDispatchUserIds = ids;
|
|
8193
|
+
},
|
|
8194
|
+
onKnownUserIdsError: (error2) => {
|
|
8195
|
+
this.log({
|
|
8196
|
+
level: "warn",
|
|
8197
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8198
|
+
conversation_id: conv.id,
|
|
8199
|
+
message_id: message.id
|
|
8200
|
+
});
|
|
8201
|
+
}
|
|
8202
|
+
}
|
|
7775
8203
|
)
|
|
7776
8204
|
);
|
|
7777
8205
|
} catch (err) {
|
|
@@ -7779,6 +8207,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7779
8207
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7780
8208
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7781
8209
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8210
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7782
8211
|
this.log({
|
|
7783
8212
|
level: "error",
|
|
7784
8213
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7854,39 +8283,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7854
8283
|
if (this.isV2) {
|
|
7855
8284
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7856
8285
|
}
|
|
7857
|
-
|
|
7858
|
-
|
|
7859
|
-
|
|
7860
|
-
|
|
7861
|
-
|
|
7862
|
-
|
|
7863
|
-
message_id: message.id
|
|
7864
|
-
});
|
|
7865
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7866
|
-
continue;
|
|
7867
|
-
}
|
|
7868
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7869
|
-
this.sessions.delete(conv.id);
|
|
7870
|
-
this.supersede(conv.id, sessionId);
|
|
7871
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8286
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8287
|
+
sessionId,
|
|
8288
|
+
content: message.content,
|
|
8289
|
+
attempts: 0,
|
|
8290
|
+
knownUserIds: preDispatchUserIds
|
|
8291
|
+
});
|
|
7872
8292
|
this.log({
|
|
7873
|
-
level: "
|
|
7874
|
-
message:
|
|
8293
|
+
level: "warn",
|
|
8294
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7875
8295
|
conversation_id: conv.id,
|
|
7876
8296
|
message_id: message.id
|
|
7877
8297
|
});
|
|
7878
|
-
|
|
7879
|
-
this.log({
|
|
7880
|
-
level: "warn",
|
|
7881
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7882
|
-
conversation_id: conv.id,
|
|
7883
|
-
message_id: message.id
|
|
7884
|
-
});
|
|
7885
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7886
|
-
});
|
|
8298
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7887
8299
|
break;
|
|
7888
8300
|
}
|
|
7889
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7890
8301
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7891
8302
|
this.releasedOpencodeIds.delete(message.id);
|
|
7892
8303
|
this.dispatched.add(message.id);
|
|
@@ -8491,23 +8902,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8491
8902
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8492
8903
|
return 1;
|
|
8493
8904
|
}
|
|
8494
|
-
/**
|
|
8495
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8496
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8497
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8498
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8499
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8500
|
-
* count, since a new session is a genuinely different attempt.
|
|
8501
|
-
*/
|
|
8502
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8503
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8504
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8505
|
-
existing.count += 1;
|
|
8506
|
-
return existing.count;
|
|
8507
|
-
}
|
|
8508
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8509
|
-
return 1;
|
|
8510
|
-
}
|
|
8511
8905
|
/** Record one queued give-up toward the never-started bound. */
|
|
8512
8906
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8513
8907
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8844,6 +9238,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8844
9238
|
this.bufferedSessionErrors.delete(oldest);
|
|
8845
9239
|
}
|
|
8846
9240
|
}
|
|
9241
|
+
retainSessionError(event) {
|
|
9242
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9243
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9244
|
+
reason: event.reason,
|
|
9245
|
+
observedAt: this.now()
|
|
9246
|
+
});
|
|
9247
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9248
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9249
|
+
if (typeof oldest !== "string") break;
|
|
9250
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9251
|
+
}
|
|
9252
|
+
}
|
|
9253
|
+
readRetainedSessionError(sessionId) {
|
|
9254
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9255
|
+
if (!retained) return null;
|
|
9256
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9257
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9258
|
+
return null;
|
|
9259
|
+
}
|
|
9260
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9261
|
+
return retained.reason;
|
|
9262
|
+
}
|
|
8847
9263
|
/**
|
|
8848
9264
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8849
9265
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -9170,9 +9586,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9170
9586
|
return;
|
|
9171
9587
|
}
|
|
9172
9588
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9589
|
+
this.retainSessionError(event);
|
|
9173
9590
|
this.log({
|
|
9174
|
-
level: "
|
|
9175
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9591
|
+
level: "warn",
|
|
9592
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
9176
9593
|
conversation_id: watcher.conv.id
|
|
9177
9594
|
});
|
|
9178
9595
|
return;
|
|
@@ -9202,9 +9619,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9202
9619
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
9203
9620
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
9204
9621
|
if (state !== "queued") {
|
|
9622
|
+
this.retainSessionError(event);
|
|
9205
9623
|
this.log({
|
|
9206
|
-
level: "
|
|
9207
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9624
|
+
level: "warn",
|
|
9625
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
9208
9626
|
conversation_id: watcher.conv.id,
|
|
9209
9627
|
message_id: inFlight.evidentMessageId
|
|
9210
9628
|
});
|
|
@@ -9433,7 +9851,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9433
9851
|
if (state === "failed") {
|
|
9434
9852
|
this.anchorDeliveryDeadline(inFlight);
|
|
9435
9853
|
if (!inFlight.done) {
|
|
9436
|
-
const
|
|
9854
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9855
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9856
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9857
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9858
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9859
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9860
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9861
|
+
(detail) => detail !== null
|
|
9862
|
+
);
|
|
9863
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9437
9864
|
this.log({
|
|
9438
9865
|
level: "error",
|
|
9439
9866
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9523,17 +9950,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9523
9950
|
}
|
|
9524
9951
|
} else {
|
|
9525
9952
|
if (inFlight.b2AbandonedSignalled) {
|
|
9953
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9954
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9526
9955
|
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9527
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9528
9956
|
await this.settleMessageAbandonedWithoutAnswer(
|
|
9529
9957
|
sessionId,
|
|
9530
9958
|
watcher,
|
|
9531
9959
|
inFlight,
|
|
9532
9960
|
messages,
|
|
9533
|
-
|
|
9961
|
+
finish
|
|
9534
9962
|
);
|
|
9535
9963
|
} else {
|
|
9536
|
-
await this.settleMessageDone(
|
|
9964
|
+
await this.settleMessageDone(
|
|
9965
|
+
sessionId,
|
|
9966
|
+
watcher,
|
|
9967
|
+
inFlight,
|
|
9968
|
+
messages,
|
|
9969
|
+
composeAbandonmentNote(
|
|
9970
|
+
finish,
|
|
9971
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
9972
|
+
"with-answer"
|
|
9973
|
+
)
|
|
9974
|
+
);
|
|
9537
9975
|
}
|
|
9538
9976
|
return;
|
|
9539
9977
|
}
|
|
@@ -9596,7 +10034,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9596
10034
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9597
10035
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9598
10036
|
});
|
|
9599
|
-
await this.settleMessageDone(
|
|
10037
|
+
await this.settleMessageDone(
|
|
10038
|
+
sessionId,
|
|
10039
|
+
watcher,
|
|
10040
|
+
inFlight,
|
|
10041
|
+
messages,
|
|
10042
|
+
composeAbandonmentNote(
|
|
10043
|
+
finish,
|
|
10044
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10045
|
+
"with-answer"
|
|
10046
|
+
)
|
|
10047
|
+
);
|
|
9600
10048
|
return;
|
|
9601
10049
|
}
|
|
9602
10050
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9796,15 +10244,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9796
10244
|
}
|
|
9797
10245
|
}
|
|
9798
10246
|
/**
|
|
9799
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9800
|
-
*
|
|
9801
|
-
*
|
|
9802
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9803
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9804
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9805
|
-
* and risking the two copies silently drifting apart.
|
|
10247
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10248
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10249
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9806
10250
|
*/
|
|
9807
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10251
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9808
10252
|
const conv = watcher.conv;
|
|
9809
10253
|
this.anchorDeliveryDeadline(inFlight);
|
|
9810
10254
|
if (!inFlight.done) {
|
|
@@ -9831,7 +10275,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9831
10275
|
title,
|
|
9832
10276
|
usage,
|
|
9833
10277
|
usageAgentName,
|
|
9834
|
-
subagentInvocations
|
|
10278
|
+
subagentInvocations,
|
|
10279
|
+
note
|
|
9835
10280
|
);
|
|
9836
10281
|
} catch (err) {
|
|
9837
10282
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9876,7 +10321,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9876
10321
|
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
9877
10322
|
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
9878
10323
|
const conv = watcher.conv;
|
|
9879
|
-
const error2 =
|
|
10324
|
+
const error2 = composeAbandonmentNote(
|
|
10325
|
+
finish,
|
|
10326
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10327
|
+
"without-answer"
|
|
10328
|
+
);
|
|
9880
10329
|
this.anchorDeliveryDeadline(inFlight);
|
|
9881
10330
|
this.log({
|
|
9882
10331
|
level: "warn",
|
|
@@ -10003,7 +10452,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10003
10452
|
*/
|
|
10004
10453
|
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10005
10454
|
const conv = watcher.conv;
|
|
10006
|
-
const error2 =
|
|
10455
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10007
10456
|
this.log({
|
|
10008
10457
|
level: "error",
|
|
10009
10458
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
@@ -10094,11 +10543,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10094
10543
|
}
|
|
10095
10544
|
}
|
|
10096
10545
|
}
|
|
10097
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10546
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
10098
10547
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10548
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10549
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10550
|
+
this.awaitingCorrelation.delete(id);
|
|
10551
|
+
this.log({
|
|
10552
|
+
level: "debug",
|
|
10553
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10554
|
+
message_id: id
|
|
10555
|
+
});
|
|
10556
|
+
}
|
|
10557
|
+
}
|
|
10099
10558
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
10100
10559
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10101
10560
|
this.untrackableAck.delete(id);
|
|
10561
|
+
this.untrackableAckSessionIds.delete(id);
|
|
10102
10562
|
this.untrackableAckSignalled.delete(id);
|
|
10103
10563
|
this.untrackableAckWarned.delete(id);
|
|
10104
10564
|
this.log({
|
|
@@ -10138,7 +10598,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10138
10598
|
bySession.set(row.opencode_session_id, list);
|
|
10139
10599
|
}
|
|
10140
10600
|
for (const [sessionId, sessionRows] of bySession) {
|
|
10141
|
-
let messages;
|
|
10601
|
+
let messages = null;
|
|
10142
10602
|
try {
|
|
10143
10603
|
if (this.isV2) {
|
|
10144
10604
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -10147,9 +10607,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10147
10607
|
level: "warn",
|
|
10148
10608
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
10149
10609
|
});
|
|
10150
|
-
|
|
10610
|
+
} else {
|
|
10611
|
+
messages = snapshot;
|
|
10151
10612
|
}
|
|
10152
|
-
messages = snapshot;
|
|
10153
10613
|
} else {
|
|
10154
10614
|
const polled = await pollSessionMessagesForRedrive(
|
|
10155
10615
|
this.port,
|
|
@@ -10162,15 +10622,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10162
10622
|
level: "warn",
|
|
10163
10623
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
10164
10624
|
});
|
|
10165
|
-
|
|
10625
|
+
} else {
|
|
10626
|
+
messages = polled.messages;
|
|
10166
10627
|
}
|
|
10167
|
-
messages = polled.messages;
|
|
10168
10628
|
}
|
|
10169
10629
|
} catch (err) {
|
|
10170
10630
|
this.log({
|
|
10171
10631
|
level: "warn",
|
|
10172
10632
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
10173
10633
|
});
|
|
10634
|
+
}
|
|
10635
|
+
if (messages === null) {
|
|
10636
|
+
for (const row of sessionRows) {
|
|
10637
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10638
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10639
|
+
}
|
|
10640
|
+
}
|
|
10174
10641
|
continue;
|
|
10175
10642
|
}
|
|
10176
10643
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -10180,6 +10647,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10180
10647
|
}
|
|
10181
10648
|
}
|
|
10182
10649
|
}
|
|
10650
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10651
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10652
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10653
|
+
if (!correlation) return false;
|
|
10654
|
+
correlation.attempts += 1;
|
|
10655
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10656
|
+
if (correlatedId !== null) {
|
|
10657
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10658
|
+
const conv = this.convForRow(sessionId, row);
|
|
10659
|
+
const message = this.queuedMessageForRow(row);
|
|
10660
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10661
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10662
|
+
this.dispatched.add(row.id);
|
|
10663
|
+
this.readopted.add(row.id);
|
|
10664
|
+
this.ensureWatcherRunning(sessionId);
|
|
10665
|
+
this.log({
|
|
10666
|
+
level: "info",
|
|
10667
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10668
|
+
conversation_id: row.conversation_id,
|
|
10669
|
+
message_id: row.id
|
|
10670
|
+
});
|
|
10671
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10672
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10673
|
+
opencode_message_id: correlatedId
|
|
10674
|
+
});
|
|
10675
|
+
return true;
|
|
10676
|
+
}
|
|
10677
|
+
if (correlation.knownUserIds === null) {
|
|
10678
|
+
this.log({
|
|
10679
|
+
level: "debug",
|
|
10680
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10681
|
+
conversation_id: row.conversation_id,
|
|
10682
|
+
message_id: row.id
|
|
10683
|
+
});
|
|
10684
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10685
|
+
this.log({
|
|
10686
|
+
level: "warn",
|
|
10687
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10688
|
+
conversation_id: row.conversation_id,
|
|
10689
|
+
message_id: row.id
|
|
10690
|
+
});
|
|
10691
|
+
}
|
|
10692
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10693
|
+
await this.handoffUntrackableCorrelation(
|
|
10694
|
+
row.conversation_id,
|
|
10695
|
+
row.id,
|
|
10696
|
+
correlation.sessionId,
|
|
10697
|
+
correlation.attempts,
|
|
10698
|
+
"readopt"
|
|
10699
|
+
);
|
|
10700
|
+
}
|
|
10701
|
+
return true;
|
|
10702
|
+
}
|
|
10183
10703
|
/**
|
|
10184
10704
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
10185
10705
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -10187,8 +10707,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10187
10707
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
10188
10708
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
10189
10709
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
10190
|
-
* before the restart)
|
|
10191
|
-
*
|
|
10710
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10711
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10712
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10713
|
+
* `forceReadoptRun`):
|
|
10192
10714
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
10193
10715
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
10194
10716
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -10197,7 +10719,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10197
10719
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
10198
10720
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
10199
10721
|
* tracking the stored id so the reply correlates by it;
|
|
10200
|
-
* - `unknown`/null id →
|
|
10722
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10723
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
10201
10724
|
*
|
|
10202
10725
|
* Only `ChannelAuthError` propagates.
|
|
10203
10726
|
*/
|
|
@@ -10211,6 +10734,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10211
10734
|
});
|
|
10212
10735
|
return;
|
|
10213
10736
|
}
|
|
10737
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
10214
10738
|
const ocId = row.opencode_message_id;
|
|
10215
10739
|
const state = messageRunState(messages, ocId ?? "");
|
|
10216
10740
|
if (state === "done") {
|
|
@@ -10277,12 +10801,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10277
10801
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
10278
10802
|
this.dontRedispatch.delete(row.id);
|
|
10279
10803
|
this.untrackableAck.delete(row.id);
|
|
10804
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
10280
10805
|
this.untrackableAckSignalled.delete(row.id);
|
|
10281
10806
|
this.untrackableAckWarned.delete(row.id);
|
|
10282
10807
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
10283
10808
|
return;
|
|
10284
10809
|
}
|
|
10285
|
-
if (this.
|
|
10810
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10811
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10812
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10813
|
+
this.untrackableAckWarned.add(row.id);
|
|
10814
|
+
this.log({
|
|
10815
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10816
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10817
|
+
conversation_id: row.conversation_id,
|
|
10818
|
+
message_id: row.id
|
|
10819
|
+
});
|
|
10820
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10821
|
+
const reportError = await this.reportUntrackableAck(
|
|
10822
|
+
row.conversation_id,
|
|
10823
|
+
row.id,
|
|
10824
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10825
|
+
errorMessage3
|
|
10826
|
+
);
|
|
10827
|
+
if (reportError !== null) {
|
|
10828
|
+
this.log({
|
|
10829
|
+
level: "warn",
|
|
10830
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10831
|
+
conversation_id: row.conversation_id,
|
|
10832
|
+
message_id: row.id
|
|
10833
|
+
});
|
|
10834
|
+
}
|
|
10835
|
+
} else {
|
|
10836
|
+
this.log({
|
|
10837
|
+
level: "debug",
|
|
10838
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10839
|
+
conversation_id: row.conversation_id,
|
|
10840
|
+
message_id: row.id
|
|
10841
|
+
});
|
|
10842
|
+
}
|
|
10843
|
+
return;
|
|
10844
|
+
}
|
|
10845
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
10286
10846
|
this.log({
|
|
10287
10847
|
level: "debug",
|
|
10288
10848
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -10460,6 +11020,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10460
11020
|
}
|
|
10461
11021
|
this.dontRedispatch.delete(row.id);
|
|
10462
11022
|
this.untrackableAck.delete(row.id);
|
|
11023
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
10463
11024
|
this.untrackableAckSignalled.delete(row.id);
|
|
10464
11025
|
this.untrackableAckWarned.delete(row.id);
|
|
10465
11026
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -10479,8 +11040,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10479
11040
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
10480
11041
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
10481
11042
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
10482
|
-
* re-entry)
|
|
10483
|
-
*
|
|
11043
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11044
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11045
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
10484
11046
|
*
|
|
10485
11047
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
10486
11048
|
* `processed_at` (Invariant 1).
|
|
@@ -10543,6 +11105,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10543
11105
|
);
|
|
10544
11106
|
}
|
|
10545
11107
|
let ocId;
|
|
11108
|
+
let preDispatchUserIds = null;
|
|
10546
11109
|
try {
|
|
10547
11110
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10548
11111
|
sessionId,
|
|
@@ -10552,7 +11115,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10552
11115
|
row.content,
|
|
10553
11116
|
options,
|
|
10554
11117
|
sendAttachments,
|
|
10555
|
-
this.openCodeClient
|
|
11118
|
+
this.openCodeClient,
|
|
11119
|
+
{
|
|
11120
|
+
onKnownUserIds: (ids) => {
|
|
11121
|
+
preDispatchUserIds = ids;
|
|
11122
|
+
},
|
|
11123
|
+
onKnownUserIdsError: (error2) => {
|
|
11124
|
+
this.log({
|
|
11125
|
+
level: "warn",
|
|
11126
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11127
|
+
conversation_id: row.conversation_id,
|
|
11128
|
+
message_id: row.id
|
|
11129
|
+
});
|
|
11130
|
+
}
|
|
11131
|
+
}
|
|
10556
11132
|
)
|
|
10557
11133
|
);
|
|
10558
11134
|
} catch (err) {
|
|
@@ -10585,6 +11161,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10585
11161
|
}
|
|
10586
11162
|
} else {
|
|
10587
11163
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11164
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10588
11165
|
}
|
|
10589
11166
|
this.log({
|
|
10590
11167
|
level: "error",
|
|
@@ -10633,39 +11210,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10633
11210
|
}
|
|
10634
11211
|
if (ocId === null) {
|
|
10635
11212
|
this.awaitingReadopt.delete(row.id);
|
|
10636
|
-
|
|
10637
|
-
|
|
10638
|
-
|
|
10639
|
-
|
|
10640
|
-
|
|
10641
|
-
|
|
10642
|
-
this.log({
|
|
10643
|
-
level: "error",
|
|
10644
|
-
message: errorMessage3,
|
|
10645
|
-
conversation_id: row.conversation_id,
|
|
10646
|
-
message_id: row.id
|
|
10647
|
-
});
|
|
10648
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10649
|
-
this.log({
|
|
10650
|
-
level: "warn",
|
|
10651
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10652
|
-
conversation_id: row.conversation_id,
|
|
10653
|
-
message_id: row.id
|
|
10654
|
-
});
|
|
10655
|
-
});
|
|
10656
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10657
|
-
return;
|
|
10658
|
-
}
|
|
11213
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11214
|
+
sessionId,
|
|
11215
|
+
content: row.content,
|
|
11216
|
+
attempts: 0,
|
|
11217
|
+
knownUserIds: preDispatchUserIds
|
|
11218
|
+
});
|
|
10659
11219
|
this.log({
|
|
10660
11220
|
level: "warn",
|
|
10661
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11221
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10662
11222
|
conversation_id: row.conversation_id,
|
|
10663
11223
|
message_id: row.id
|
|
10664
11224
|
});
|
|
10665
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10666
11225
|
return;
|
|
10667
11226
|
}
|
|
10668
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10669
11227
|
const processedAtMs = this.processedAtMs(row);
|
|
10670
11228
|
this.registerReadopted(
|
|
10671
11229
|
readoptConv,
|
|
@@ -11571,7 +12129,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11571
12129
|
* watcher retries next tick within the
|
|
11572
12130
|
* deadline, Finding 4).
|
|
11573
12131
|
*/
|
|
11574
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12132
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11575
12133
|
const res = await this.fetchImpl(
|
|
11576
12134
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11577
12135
|
{
|
|
@@ -11589,7 +12147,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11589
12147
|
...title ? { title } : {},
|
|
11590
12148
|
...usage ? usage : {},
|
|
11591
12149
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11592
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12150
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12151
|
+
...error2 ? { error: error2 } : {}
|
|
11593
12152
|
})
|
|
11594
12153
|
}
|
|
11595
12154
|
);
|
|
@@ -12317,7 +12876,9 @@ async function installRunnerSecret({
|
|
|
12317
12876
|
}) {
|
|
12318
12877
|
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
12319
12878
|
if (!arn) {
|
|
12320
|
-
|
|
12879
|
+
if (!env.GH_TOKEN) {
|
|
12880
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
12881
|
+
}
|
|
12321
12882
|
return false;
|
|
12322
12883
|
}
|
|
12323
12884
|
const result = await (commandRunner ?? runCommand2)(
|
|
@@ -13261,6 +13822,9 @@ async function driveChannels(state, driver) {
|
|
|
13261
13822
|
}
|
|
13262
13823
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
13263
13824
|
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
13825
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
13826
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
13827
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
13264
13828
|
function shouldWarnForReclaimSkip(reason) {
|
|
13265
13829
|
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
13266
13830
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
@@ -13348,8 +13912,12 @@ async function runSweep(state, driver, config) {
|
|
|
13348
13912
|
protectedIds: protectedNow
|
|
13349
13913
|
}
|
|
13350
13914
|
);
|
|
13351
|
-
const
|
|
13352
|
-
|
|
13915
|
+
const eligibleToDelete = toDelete.filter(
|
|
13916
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
13917
|
+
);
|
|
13918
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13919
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
13920
|
+
const answeredFailures = [];
|
|
13353
13921
|
protectedNow = driver.protectedSessionIds();
|
|
13354
13922
|
for (const id of deleteCandidates) {
|
|
13355
13923
|
if (protectedNow.has(id)) {
|
|
@@ -13372,8 +13940,44 @@ async function runSweep(state, driver, config) {
|
|
|
13372
13940
|
if (statusClass === 4) failed4xx++;
|
|
13373
13941
|
else if (statusClass === 5) failed5xx++;
|
|
13374
13942
|
else failedOtherStatus++;
|
|
13943
|
+
answeredFailures.push({
|
|
13944
|
+
id,
|
|
13945
|
+
status: outcome.status,
|
|
13946
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
13947
|
+
});
|
|
13375
13948
|
}
|
|
13376
13949
|
}
|
|
13950
|
+
const givenUpFailures = [];
|
|
13951
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
13952
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
13953
|
+
} else {
|
|
13954
|
+
for (const failure of answeredFailures) {
|
|
13955
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
13956
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
13957
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
13958
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
13959
|
+
givenUpFailures.push(failure);
|
|
13960
|
+
}
|
|
13961
|
+
}
|
|
13962
|
+
}
|
|
13963
|
+
for (const failure of givenUpFailures) {
|
|
13964
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
13965
|
+
0,
|
|
13966
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
13967
|
+
);
|
|
13968
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
13969
|
+
type: "info",
|
|
13970
|
+
level: "warn",
|
|
13971
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
13972
|
+
metadata: {
|
|
13973
|
+
outcome: "session-delete-given-up",
|
|
13974
|
+
session_id: failure.id,
|
|
13975
|
+
status: failure.status,
|
|
13976
|
+
body_excerpt: excerpt,
|
|
13977
|
+
given_up_count: givenUpFailures.length
|
|
13978
|
+
}
|
|
13979
|
+
});
|
|
13980
|
+
}
|
|
13377
13981
|
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13378
13982
|
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13379
13983
|
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
@@ -13394,21 +13998,31 @@ async function runSweep(state, driver, config) {
|
|
|
13394
13998
|
sessions_remaining: remaining,
|
|
13395
13999
|
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13396
14000
|
};
|
|
14001
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
13397
14002
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
13398
14003
|
dbPath: sessionDbPath(),
|
|
13399
14004
|
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13400
14005
|
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13401
14006
|
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
13402
|
-
|
|
14007
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14008
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
13403
14009
|
});
|
|
13404
14010
|
if (reclaimResult.ok) {
|
|
13405
14011
|
state.sessionStoreHealth.observeWritable();
|
|
13406
14012
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
13407
14013
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
13408
14014
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
14015
|
+
const loopElapsedMs = Math.max(
|
|
14016
|
+
0,
|
|
14017
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14018
|
+
);
|
|
14019
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14020
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14021
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
13409
14022
|
logSessionCleanupActivity(state, "sweep", {
|
|
13410
14023
|
type: "info",
|
|
13411
|
-
|
|
14024
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14025
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
13412
14026
|
metadata: {
|
|
13413
14027
|
outcome: "reclaimed",
|
|
13414
14028
|
mode: reclaimResult.mode,
|
|
@@ -13421,6 +14035,7 @@ async function runSweep(state, driver, config) {
|
|
|
13421
14035
|
freelist_after: reclaimResult.freelistAfter,
|
|
13422
14036
|
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13423
14037
|
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14038
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
13424
14039
|
...sessionMetadata
|
|
13425
14040
|
}
|
|
13426
14041
|
});
|
|
@@ -13964,6 +14579,7 @@ async function run(options) {
|
|
|
13964
14579
|
messageCount: 0,
|
|
13965
14580
|
lastProxiedActivityAt: null,
|
|
13966
14581
|
sessionCleanupTimers: [],
|
|
14582
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13967
14583
|
claudeUsageTimer: null,
|
|
13968
14584
|
claudeUsageRearm: null,
|
|
13969
14585
|
openaiUsageTimer: null,
|
|
@@ -14144,29 +14760,61 @@ async function run(options) {
|
|
|
14144
14760
|
} else {
|
|
14145
14761
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
14146
14762
|
}
|
|
14147
|
-
if (options.restoreRunnerCredentials) {
|
|
14148
|
-
|
|
14149
|
-
|
|
14150
|
-
|
|
14151
|
-
|
|
14152
|
-
if (level === "error") {
|
|
14153
|
-
logActivity(state, { type: "error", error: message });
|
|
14154
|
-
} else {
|
|
14155
|
-
logActivity(state, { type: "info", level, message });
|
|
14156
|
-
}
|
|
14763
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14764
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14765
|
+
log2(state, message, level);
|
|
14766
|
+
if (state.interactive && !state.json) {
|
|
14767
|
+
logActivity(state, { type: "info", level, message });
|
|
14157
14768
|
}
|
|
14158
14769
|
};
|
|
14159
|
-
const
|
|
14770
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14771
|
+
if (vaultResolution.error) {
|
|
14772
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14773
|
+
logCredentialVaultResolution(
|
|
14774
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14775
|
+
"warn"
|
|
14776
|
+
);
|
|
14777
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14778
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14779
|
+
logCredentialVaultResolution(
|
|
14780
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14781
|
+
"info"
|
|
14782
|
+
);
|
|
14783
|
+
} else {
|
|
14784
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14785
|
+
logCredentialVaultResolution(
|
|
14786
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14787
|
+
"debug"
|
|
14788
|
+
);
|
|
14789
|
+
}
|
|
14790
|
+
}
|
|
14791
|
+
const credentialContext = {
|
|
14792
|
+
env: process.env,
|
|
14793
|
+
log: (message, level = "info") => {
|
|
14794
|
+
if (level === "error") {
|
|
14795
|
+
logActivity(state, { type: "error", error: message });
|
|
14796
|
+
} else {
|
|
14797
|
+
logActivity(state, { type: "info", level, message });
|
|
14798
|
+
}
|
|
14799
|
+
}
|
|
14800
|
+
};
|
|
14801
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
14802
|
+
if (options.restoreRunnerCredentials) {
|
|
14160
14803
|
await restoreCredentialStores(credentialContext);
|
|
14161
|
-
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
14162
14804
|
}
|
|
14805
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
14163
14806
|
state.credentialSync?.arm();
|
|
14164
14807
|
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14165
14808
|
() => ({
|
|
14166
14809
|
agentId: state.agentId,
|
|
14167
14810
|
authHeader: state.authHeader
|
|
14168
14811
|
}),
|
|
14169
|
-
|
|
14812
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14813
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14814
|
+
if (cause !== null && sessionId !== null) {
|
|
14815
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14816
|
+
}
|
|
14817
|
+
}
|
|
14170
14818
|
);
|
|
14171
14819
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14172
14820
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
@@ -14177,22 +14825,27 @@ async function run(options) {
|
|
|
14177
14825
|
openCodeLogTail.stop();
|
|
14178
14826
|
};
|
|
14179
14827
|
let sessionDbVerifyFatal = false;
|
|
14828
|
+
let sessionDbVerifyFatalReason;
|
|
14180
14829
|
if (!options.restoreSessionDb) {
|
|
14181
|
-
|
|
14830
|
+
logActivity(state, {
|
|
14831
|
+
type: "info",
|
|
14832
|
+
level: "warn",
|
|
14833
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14834
|
+
});
|
|
14182
14835
|
} else {
|
|
14183
14836
|
const health = await checkOpenCodeHealth(state.port);
|
|
14184
14837
|
if (health.healthy) {
|
|
14185
|
-
|
|
14186
|
-
|
|
14187
|
-
|
|
14188
|
-
"
|
|
14189
|
-
);
|
|
14838
|
+
logActivity(state, {
|
|
14839
|
+
type: "info",
|
|
14840
|
+
level: "warn",
|
|
14841
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14842
|
+
});
|
|
14190
14843
|
} else if (health.authFailed) {
|
|
14191
|
-
|
|
14192
|
-
|
|
14193
|
-
|
|
14194
|
-
"
|
|
14195
|
-
);
|
|
14844
|
+
logActivity(state, {
|
|
14845
|
+
type: "info",
|
|
14846
|
+
level: "warn",
|
|
14847
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14848
|
+
});
|
|
14196
14849
|
} else {
|
|
14197
14850
|
const result = await restoreAndVerifySessionDb({
|
|
14198
14851
|
dbPath: sessionDbPath(),
|
|
@@ -14209,12 +14862,13 @@ async function run(options) {
|
|
|
14209
14862
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
14210
14863
|
});
|
|
14211
14864
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14865
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
14212
14866
|
}
|
|
14213
14867
|
}
|
|
14214
14868
|
reportSessionDbRecovery(state);
|
|
14215
14869
|
if (sessionDbVerifyFatal) {
|
|
14216
14870
|
throw new Error(
|
|
14217
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14871
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
14218
14872
|
);
|
|
14219
14873
|
}
|
|
14220
14874
|
applyRunnerOpenCodeConfig({
|
|
@@ -14541,14 +15195,14 @@ async function run(options) {
|
|
|
14541
15195
|
},
|
|
14542
15196
|
onDisconnected: (code, reason, transition) => {
|
|
14543
15197
|
state.connected = false;
|
|
14544
|
-
const
|
|
15198
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
14545
15199
|
logActivity(state, {
|
|
14546
15200
|
type: "info",
|
|
14547
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15201
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14548
15202
|
});
|
|
14549
15203
|
emitAgentDisconnected(state.agentId, {
|
|
14550
15204
|
code,
|
|
14551
|
-
reason:
|
|
15205
|
+
reason: safeReason2,
|
|
14552
15206
|
connection_generation: transition.generation
|
|
14553
15207
|
});
|
|
14554
15208
|
if (state.interactive) displayStatus(state);
|
|
@@ -14711,6 +15365,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14711
15365
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14712
15366
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14713
15367
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15368
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14714
15369
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14715
15370
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14716
15371
|
"-a, --agent [id]",
|