@evident-ai/cli 3.5.2-dev.df1ae49 → 3.5.2-dev.fbf25b4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +950 -242
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -703,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
703
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
704
704
|
});
|
|
705
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
706
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
707
732
|
const apiUrl = getApiUrlConfig();
|
|
708
733
|
try {
|
|
@@ -877,6 +902,76 @@ async function status(options = {}) {
|
|
|
877
902
|
process.exit(result.exitCode);
|
|
878
903
|
}
|
|
879
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// src/lib/claude-usage.ts
|
|
881
976
|
import { execFileSync } from "node:child_process";
|
|
882
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1069,6 +1164,76 @@ import chalk7 from "chalk";
|
|
|
1069
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1070
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1071
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1072
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1073
1238
|
var TelemetryEventTypes = {
|
|
1074
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1157,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1157
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1158
1323
|
|
|
1159
1324
|
// src/lib/telemetry.ts
|
|
1160
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.fbf25b4" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1161
1326
|
function getCliVersion() {
|
|
1162
1327
|
return CLI_VERSION;
|
|
1163
1328
|
}
|
|
@@ -1318,12 +1483,38 @@ var EventTypes = {
|
|
|
1318
1483
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1319
1484
|
};
|
|
1320
1485
|
|
|
1321
|
-
// src/lib/runner-activity-telemetry.ts
|
|
1322
|
-
import { homedir as homedir2 } from "node:os";
|
|
1323
|
-
|
|
1324
1486
|
// src/lib/opencode/session-db-reclaim.ts
|
|
1325
1487
|
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1326
1488
|
import { dirname as dirname2 } from "node:path";
|
|
1489
|
+
|
|
1490
|
+
// src/lib/diagnostic-redaction.ts
|
|
1491
|
+
import { homedir as homedir2 } from "node:os";
|
|
1492
|
+
function redactHomeDirectory(message) {
|
|
1493
|
+
const home = homedir2();
|
|
1494
|
+
if (!home || home === "/") {
|
|
1495
|
+
return message;
|
|
1496
|
+
}
|
|
1497
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1498
|
+
let result = "";
|
|
1499
|
+
let searchFrom = 0;
|
|
1500
|
+
let homeIndex;
|
|
1501
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1502
|
+
const homeEnd = homeIndex + home.length;
|
|
1503
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1504
|
+
const nextCharacter = message[homeEnd];
|
|
1505
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1506
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1507
|
+
result += atPathBoundary ? "~" : home;
|
|
1508
|
+
searchFrom = homeEnd;
|
|
1509
|
+
}
|
|
1510
|
+
return result + message.slice(searchFrom);
|
|
1511
|
+
}
|
|
1512
|
+
function redactDiagnosticText(message) {
|
|
1513
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1514
|
+
return redactHomeDirectory(redacted);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1327
1518
|
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1328
1519
|
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1329
1520
|
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
@@ -1331,7 +1522,7 @@ function errorMessage(error2) {
|
|
|
1331
1522
|
if (!(error2 instanceof Error)) return String(error2);
|
|
1332
1523
|
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1333
1524
|
}
|
|
1334
|
-
function
|
|
1525
|
+
function errorMessages(error2) {
|
|
1335
1526
|
const messages = [];
|
|
1336
1527
|
const visited = /* @__PURE__ */ new Set();
|
|
1337
1528
|
let current = error2;
|
|
@@ -1355,12 +1546,81 @@ function isSessionStoreWriteFailure(error2) {
|
|
|
1355
1546
|
}
|
|
1356
1547
|
break;
|
|
1357
1548
|
}
|
|
1358
|
-
return messages
|
|
1549
|
+
return messages;
|
|
1550
|
+
}
|
|
1551
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1552
|
+
return errorMessages(error2).some(
|
|
1359
1553
|
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1360
1554
|
message
|
|
1361
1555
|
)
|
|
1362
1556
|
);
|
|
1363
1557
|
}
|
|
1558
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1559
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1560
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1561
|
+
function isCauseBearingText(text) {
|
|
1562
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1563
|
+
}
|
|
1564
|
+
function structuredLogFields(line) {
|
|
1565
|
+
const fields = [];
|
|
1566
|
+
for (let index = 0; index < line.length; index++) {
|
|
1567
|
+
if (line[index] === '"') {
|
|
1568
|
+
index++;
|
|
1569
|
+
while (index < line.length) {
|
|
1570
|
+
if (line[index] === "\\") {
|
|
1571
|
+
index++;
|
|
1572
|
+
} else if (line[index] === '"') {
|
|
1573
|
+
break;
|
|
1574
|
+
}
|
|
1575
|
+
index++;
|
|
1576
|
+
}
|
|
1577
|
+
continue;
|
|
1578
|
+
}
|
|
1579
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1580
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1581
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1582
|
+
const valueStart = index + key.length;
|
|
1583
|
+
if (line[valueStart] !== '"') {
|
|
1584
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1585
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1586
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
let valueEnd = valueStart + 1;
|
|
1590
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1591
|
+
if (line[valueEnd] === "\\") {
|
|
1592
|
+
valueEnd++;
|
|
1593
|
+
} else if (line[valueEnd] === '"') {
|
|
1594
|
+
break;
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1598
|
+
index = valueEnd;
|
|
1599
|
+
}
|
|
1600
|
+
return fields;
|
|
1601
|
+
}
|
|
1602
|
+
function structuredCauseMessages(text) {
|
|
1603
|
+
const fields = structuredLogFields(text);
|
|
1604
|
+
if (fields.length > 0) return fields;
|
|
1605
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1606
|
+
}
|
|
1607
|
+
function causeTokenFromText(text) {
|
|
1608
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1609
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1610
|
+
const match = pattern.exec(text);
|
|
1611
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1612
|
+
return match[1];
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
return null;
|
|
1616
|
+
}
|
|
1617
|
+
function sessionStoreCauseToken(value) {
|
|
1618
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1619
|
+
const token = causeTokenFromText(message);
|
|
1620
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1621
|
+
}
|
|
1622
|
+
return null;
|
|
1623
|
+
}
|
|
1364
1624
|
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1365
1625
|
try {
|
|
1366
1626
|
const fsStats = statfsSync(dirname2(dbPath));
|
|
@@ -1549,11 +1809,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1549
1809
|
if (!levelMatch) return null;
|
|
1550
1810
|
const level = levelMatch[1].toUpperCase();
|
|
1551
1811
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1552
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1812
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1813
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1553
1814
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1554
1815
|
const parsed = {
|
|
1555
1816
|
level: level === "WARN" ? "warn" : "error",
|
|
1556
|
-
sessionID: sessionMatch?.[1]
|
|
1817
|
+
sessionID: sessionMatch?.[1],
|
|
1818
|
+
messageID: messageIDMatch?.[1]
|
|
1557
1819
|
};
|
|
1558
1820
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1559
1821
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1567,6 +1829,7 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
|
|
|
1567
1829
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1568
1830
|
const metadata = {
|
|
1569
1831
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1832
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1570
1833
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1571
1834
|
};
|
|
1572
1835
|
forwardRunnerActivity(
|
|
@@ -1591,19 +1854,25 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
|
|
|
1591
1854
|
}
|
|
1592
1855
|
};
|
|
1593
1856
|
const flushLine = (line) => {
|
|
1594
|
-
|
|
1857
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1858
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
1859
|
+
const cause = sessionStoreCauseToken(line);
|
|
1860
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
1595
1861
|
try {
|
|
1596
|
-
onSessionStoreWriteFailure(
|
|
1862
|
+
onSessionStoreWriteFailure({
|
|
1863
|
+
isDiskFailure,
|
|
1864
|
+
sessionId: parsed?.sessionID ?? null,
|
|
1865
|
+
cause
|
|
1866
|
+
});
|
|
1597
1867
|
} catch (err) {
|
|
1598
1868
|
reportTelemetryDiagnostic(
|
|
1599
1869
|
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1600
1870
|
);
|
|
1601
1871
|
}
|
|
1602
1872
|
}
|
|
1603
|
-
const parsed = parseOpenCodeLogLine(line);
|
|
1604
1873
|
if (!parsed) return;
|
|
1605
1874
|
const now = Date.now();
|
|
1606
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1875
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1607
1876
|
expireRepeatStates(now, key);
|
|
1608
1877
|
if (key === void 0) {
|
|
1609
1878
|
forwardLine(line, parsed);
|
|
@@ -1649,34 +1918,21 @@ function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure)
|
|
|
1649
1918
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1650
1919
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1651
1920
|
var MAX_METADATA_ENTRIES = 20;
|
|
1652
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1653
|
-
function redactHomeDirectory(message) {
|
|
1654
|
-
const home = homedir2();
|
|
1655
|
-
if (!home || home === "/") {
|
|
1656
|
-
return message;
|
|
1657
|
-
}
|
|
1658
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1659
|
-
let result = "";
|
|
1660
|
-
let searchFrom = 0;
|
|
1661
|
-
let homeIndex;
|
|
1662
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1663
|
-
const homeEnd = homeIndex + home.length;
|
|
1664
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1665
|
-
const nextCharacter = message[homeEnd];
|
|
1666
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1667
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1668
|
-
result += atPathBoundary ? "~" : home;
|
|
1669
|
-
searchFrom = homeEnd;
|
|
1670
|
-
}
|
|
1671
|
-
return result + message.slice(searchFrom);
|
|
1672
|
-
}
|
|
1673
|
-
function redactDiagnosticText(message) {
|
|
1674
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1675
|
-
return redactHomeDirectory(redacted);
|
|
1676
|
-
}
|
|
1677
1921
|
function truncate(message) {
|
|
1678
1922
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1679
|
-
|
|
1923
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
1924
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1925
|
+
let previousDropped = -1;
|
|
1926
|
+
while (dropped !== previousDropped) {
|
|
1927
|
+
previousDropped = dropped;
|
|
1928
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
1929
|
+
dropped = nextDropped;
|
|
1930
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
1931
|
+
}
|
|
1932
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
1933
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
1934
|
+
const tailLength = remaining - headLength;
|
|
1935
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1680
1936
|
}
|
|
1681
1937
|
function sanitiseMetadata(metadata) {
|
|
1682
1938
|
if (!metadata) return {};
|
|
@@ -1886,12 +2142,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1886
2142
|
metadata: withoutContractFields(record),
|
|
1887
2143
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1888
2144
|
};
|
|
1889
|
-
case "session_db_boot_refused":
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
2145
|
+
case "session_db_boot_refused": {
|
|
2146
|
+
const message = (() => {
|
|
2147
|
+
switch (record.reason) {
|
|
2148
|
+
case "local_discard_failed":
|
|
2149
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2150
|
+
case "replica_separation_unproven":
|
|
2151
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2152
|
+
case "verification_timeout_before_separation":
|
|
2153
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2154
|
+
case "verification_timeout_during_separation":
|
|
2155
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2156
|
+
default:
|
|
2157
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2158
|
+
}
|
|
2159
|
+
})();
|
|
2160
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2161
|
+
}
|
|
1895
2162
|
case "schema_provenance_mismatch":
|
|
1896
2163
|
return {
|
|
1897
2164
|
level,
|
|
@@ -2141,7 +2408,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
2141
2408
|
return null;
|
|
2142
2409
|
}
|
|
2143
2410
|
}
|
|
2144
|
-
function
|
|
2411
|
+
function roleOf2(m) {
|
|
2145
2412
|
if (!m || typeof m !== "object") return void 0;
|
|
2146
2413
|
if (typeof m.role === "string") return m.role;
|
|
2147
2414
|
const infoRole = m.info?.role;
|
|
@@ -2155,13 +2422,13 @@ function createdOf(m) {
|
|
|
2155
2422
|
if (!m || typeof m !== "object") return void 0;
|
|
2156
2423
|
return m.info?.time?.created ?? m.time?.created;
|
|
2157
2424
|
}
|
|
2158
|
-
function
|
|
2425
|
+
function idOf2(m) {
|
|
2159
2426
|
if (!m || typeof m !== "object") return void 0;
|
|
2160
2427
|
if (typeof m.id === "string") return m.id;
|
|
2161
2428
|
const infoId = m.info?.id;
|
|
2162
2429
|
return typeof infoId === "string" ? infoId : void 0;
|
|
2163
2430
|
}
|
|
2164
|
-
function
|
|
2431
|
+
function parentIdOf2(m) {
|
|
2165
2432
|
if (!m || typeof m !== "object") return void 0;
|
|
2166
2433
|
if (typeof m.parentID === "string") return m.parentID;
|
|
2167
2434
|
const infoParent = m.info?.parentID;
|
|
@@ -2177,8 +2444,12 @@ function errorOf(m) {
|
|
|
2177
2444
|
if (!m || typeof m !== "object") return void 0;
|
|
2178
2445
|
return m.info?.error ?? m.error;
|
|
2179
2446
|
}
|
|
2447
|
+
function isTerminalStopAnswer(m) {
|
|
2448
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2449
|
+
}
|
|
2180
2450
|
function isAssistantInFlight(m) {
|
|
2181
2451
|
if (completedOf(m) == null) return true;
|
|
2452
|
+
if (errorOf(m) != null) return false;
|
|
2182
2453
|
return finishOf(m) === "tool-calls";
|
|
2183
2454
|
}
|
|
2184
2455
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2226,7 +2497,7 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2226
2497
|
function isSessionActivelyGenerating(messages) {
|
|
2227
2498
|
if (!messages || messages.length === 0) return false;
|
|
2228
2499
|
const last = messages[messages.length - 1];
|
|
2229
|
-
if (
|
|
2500
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2230
2501
|
return completedOf(last) == null;
|
|
2231
2502
|
}
|
|
2232
2503
|
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
@@ -2535,11 +2806,31 @@ function messageText(m) {
|
|
|
2535
2806
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2536
2807
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2537
2808
|
}
|
|
2538
|
-
|
|
2809
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2810
|
+
if (!messages || messages.length === 0) return null;
|
|
2811
|
+
let best = null;
|
|
2812
|
+
for (const message of messages) {
|
|
2813
|
+
if (roleOf2(message) !== "user") continue;
|
|
2814
|
+
const id = idOf2(message);
|
|
2815
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2816
|
+
if (messageText(message) !== content) continue;
|
|
2817
|
+
const created = createdOf(message) ?? 0;
|
|
2818
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2819
|
+
}
|
|
2820
|
+
return best?.id ?? null;
|
|
2821
|
+
}
|
|
2822
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2539
2823
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2540
2824
|
const knownUserIds = new Set(
|
|
2541
|
-
(before ?? []).filter((m) =>
|
|
2825
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2542
2826
|
);
|
|
2827
|
+
if (callbacks) {
|
|
2828
|
+
try {
|
|
2829
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2830
|
+
} catch (error2) {
|
|
2831
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2543
2834
|
const parts = [{ type: "text", text: content }];
|
|
2544
2835
|
let pendingOutcomes = null;
|
|
2545
2836
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2578,22 +2869,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2578
2869
|
const READ_BACK_DELAY_MS = 150;
|
|
2579
2870
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2580
2871
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
const id = idOf(m);
|
|
2586
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2587
|
-
if (messageText(m) !== content) continue;
|
|
2588
|
-
const created = createdOf(m) ?? 0;
|
|
2589
|
-
if (best === null || created > best.created) {
|
|
2590
|
-
best = { id, created };
|
|
2591
|
-
}
|
|
2592
|
-
}
|
|
2593
|
-
if (best) {
|
|
2594
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2595
|
-
return best.id;
|
|
2596
|
-
}
|
|
2872
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2873
|
+
if (dispatchedUserMessageId !== null) {
|
|
2874
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2875
|
+
return dispatchedUserMessageId;
|
|
2597
2876
|
}
|
|
2598
2877
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2599
2878
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2604,57 +2883,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2604
2883
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2605
2884
|
if (!messages || messages.length === 0) return null;
|
|
2606
2885
|
const byParent = messages.find(
|
|
2607
|
-
(m) =>
|
|
2886
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2608
2887
|
);
|
|
2609
2888
|
if (byParent) return byParent;
|
|
2610
|
-
const userIndex = messages.findIndex((m) =>
|
|
2889
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2611
2890
|
if (userIndex === -1) return null;
|
|
2612
2891
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2613
|
-
if (
|
|
2892
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2614
2893
|
}
|
|
2615
2894
|
return null;
|
|
2616
2895
|
}
|
|
2617
2896
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2618
2897
|
if (!messages || messages.length === 0) return null;
|
|
2619
2898
|
let lastCorrelated = null;
|
|
2620
|
-
let lastNonErrored = null;
|
|
2621
2899
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2622
2900
|
const m = messages[i];
|
|
2623
|
-
if (
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2901
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2902
|
+
lastCorrelated = m;
|
|
2903
|
+
break;
|
|
2904
|
+
}
|
|
2905
|
+
if (lastCorrelated) {
|
|
2906
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
2907
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2908
|
+
const m = messages[i];
|
|
2909
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
2910
|
+
return m;
|
|
2911
|
+
}
|
|
2628
2912
|
}
|
|
2913
|
+
return lastCorrelated;
|
|
2629
2914
|
}
|
|
2630
|
-
|
|
2631
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
2915
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2632
2916
|
if (userIndex === -1) return null;
|
|
2633
2917
|
let last = null;
|
|
2634
|
-
let
|
|
2918
|
+
let lastAssistantIndex = -1;
|
|
2635
2919
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2636
|
-
const role =
|
|
2920
|
+
const role = roleOf2(messages[i]);
|
|
2637
2921
|
if (role === "user") break;
|
|
2638
2922
|
if (role === "assistant") {
|
|
2639
2923
|
last = messages[i];
|
|
2640
|
-
|
|
2924
|
+
lastAssistantIndex = i;
|
|
2641
2925
|
}
|
|
2642
2926
|
}
|
|
2643
|
-
|
|
2927
|
+
if (last === null || errorOf(last) == null) return last;
|
|
2928
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
2929
|
+
const m = messages[i];
|
|
2930
|
+
if (roleOf2(m) === "user") break;
|
|
2931
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
2932
|
+
}
|
|
2933
|
+
return last;
|
|
2644
2934
|
}
|
|
2645
2935
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2646
2936
|
if (!messages || messages.length === 0) return [];
|
|
2647
2937
|
const byParent = messages.filter(
|
|
2648
|
-
(message) =>
|
|
2938
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2649
2939
|
);
|
|
2650
2940
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2651
2941
|
if (assistants.length === 0) {
|
|
2652
|
-
const userIndex = messages.findIndex((message) =>
|
|
2942
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2653
2943
|
if (userIndex === -1) return [];
|
|
2654
2944
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2655
2945
|
const message = messages[i];
|
|
2656
|
-
if (
|
|
2657
|
-
if (
|
|
2946
|
+
if (roleOf2(message) === "user") break;
|
|
2947
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2658
2948
|
}
|
|
2659
2949
|
}
|
|
2660
2950
|
const refs = [];
|
|
@@ -2694,7 +2984,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2694
2984
|
if (!messages || messages.length === 0) return [];
|
|
2695
2985
|
const calls = [];
|
|
2696
2986
|
for (const message of messages) {
|
|
2697
|
-
if (
|
|
2987
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2698
2988
|
for (const part of message.parts ?? []) {
|
|
2699
2989
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2700
2990
|
continue;
|
|
@@ -2724,7 +3014,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2724
3014
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2725
3015
|
const unattributed = [];
|
|
2726
3016
|
for (const message of messages ?? []) {
|
|
2727
|
-
if (
|
|
3017
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2728
3018
|
const created = finiteNumber(createdOf(message));
|
|
2729
3019
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2730
3020
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2746,8 +3036,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2746
3036
|
}
|
|
2747
3037
|
function sumAssistantUsage(messages) {
|
|
2748
3038
|
if (!messages || messages.length === 0) return null;
|
|
2749
|
-
const
|
|
2750
|
-
const
|
|
3039
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3040
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3041
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2751
3042
|
let sawAnyUsage = false;
|
|
2752
3043
|
let inputSum = 0;
|
|
2753
3044
|
let outputSum = 0;
|
|
@@ -2801,13 +3092,11 @@ function sumAssistantUsage(messages) {
|
|
|
2801
3092
|
function messageUsage(messages, userMessageId) {
|
|
2802
3093
|
if (!messages || messages.length === 0) return null;
|
|
2803
3094
|
const byParentAll = messages.filter(
|
|
2804
|
-
(m) =>
|
|
3095
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2805
3096
|
);
|
|
2806
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2807
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2808
3097
|
let correlated;
|
|
2809
|
-
if (
|
|
2810
|
-
correlated =
|
|
3098
|
+
if (byParentAll.length > 0) {
|
|
3099
|
+
correlated = byParentAll;
|
|
2811
3100
|
} else {
|
|
2812
3101
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2813
3102
|
correlated = reply ? [reply] : [];
|
|
@@ -2817,7 +3106,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2817
3106
|
}
|
|
2818
3107
|
function messageRunState(messages, userMessageId) {
|
|
2819
3108
|
if (!messages || messages.length === 0) return "unknown";
|
|
2820
|
-
const hasUser = messages.some((m) =>
|
|
3109
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2821
3110
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2822
3111
|
if (!hasUser) {
|
|
2823
3112
|
if (!reply) return "unknown";
|
|
@@ -2853,14 +3142,38 @@ function messageError(messages, userMessageId) {
|
|
|
2853
3142
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2854
3143
|
const error2 = errorOf(reply);
|
|
2855
3144
|
if (error2 == null) return null;
|
|
2856
|
-
|
|
2857
|
-
if (typeof error2 === "
|
|
3145
|
+
let rendered;
|
|
3146
|
+
if (typeof error2 === "string") {
|
|
3147
|
+
rendered = error2;
|
|
3148
|
+
} else if (typeof error2 === "object") {
|
|
2858
3149
|
const e = error2;
|
|
2859
3150
|
const dataMessage = e.data?.message;
|
|
2860
|
-
if (typeof dataMessage === "string")
|
|
2861
|
-
|
|
3151
|
+
if (typeof dataMessage === "string") {
|
|
3152
|
+
rendered = dataMessage;
|
|
3153
|
+
} else if (typeof e.message === "string") {
|
|
3154
|
+
rendered = e.message;
|
|
3155
|
+
} else {
|
|
3156
|
+
rendered = "The agent run failed.";
|
|
3157
|
+
}
|
|
3158
|
+
} else {
|
|
3159
|
+
rendered = "The agent run failed.";
|
|
2862
3160
|
}
|
|
2863
|
-
|
|
3161
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3162
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3163
|
+
) : void 0;
|
|
3164
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3165
|
+
}
|
|
3166
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3167
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3168
|
+
if (!reply) return null;
|
|
3169
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3170
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3171
|
+
return {
|
|
3172
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3173
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3174
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3175
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3176
|
+
};
|
|
2864
3177
|
}
|
|
2865
3178
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2866
3179
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2909,7 +3222,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2909
3222
|
if (!messages || messages.length === 0) return null;
|
|
2910
3223
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2911
3224
|
const message = messages[i];
|
|
2912
|
-
if (
|
|
3225
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2913
3226
|
const created = createdOf(message);
|
|
2914
3227
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2915
3228
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2941,25 +3254,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2941
3254
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2942
3255
|
if (!messages || messages.length === 0) return false;
|
|
2943
3256
|
return messages.some(
|
|
2944
|
-
(m) =>
|
|
3257
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2945
3258
|
);
|
|
2946
3259
|
}
|
|
2947
3260
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2948
3261
|
if (!messages || messages.length === 0) return false;
|
|
2949
|
-
const userIndex = messages.findIndex((message) =>
|
|
3262
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2950
3263
|
if (userIndex === -1) return false;
|
|
2951
3264
|
let hasLaterUser = false;
|
|
2952
3265
|
let hasStartedLaterUser = false;
|
|
2953
3266
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2954
3267
|
const message = messages[i];
|
|
2955
|
-
if (
|
|
3268
|
+
if (roleOf2(message) !== "user") continue;
|
|
2956
3269
|
hasLaterUser = true;
|
|
2957
|
-
const laterUserMessageId =
|
|
3270
|
+
const laterUserMessageId = idOf2(message);
|
|
2958
3271
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2959
3272
|
return false;
|
|
2960
3273
|
}
|
|
2961
3274
|
if (messages.some(
|
|
2962
|
-
(candidate) =>
|
|
3275
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2963
3276
|
)) {
|
|
2964
3277
|
hasStartedLaterUser = true;
|
|
2965
3278
|
}
|
|
@@ -4144,9 +4457,12 @@ function runSynchroniser(args, opts) {
|
|
|
4144
4457
|
|
|
4145
4458
|
// src/lib/opencode/session-db-boot.ts
|
|
4146
4459
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
4147
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4460
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
4148
4461
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
4149
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4462
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4463
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4464
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4465
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
4150
4466
|
function commandError(result) {
|
|
4151
4467
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
4152
4468
|
}
|
|
@@ -4457,20 +4773,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4457
4773
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4458
4774
|
env: {
|
|
4459
4775
|
...env,
|
|
4460
|
-
// The synchroniser reads
|
|
4461
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
4776
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4462
4777
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4463
4778
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
4779
|
+
),
|
|
4780
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
4781
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
4782
|
+
),
|
|
4783
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
4784
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
4785
|
+
),
|
|
4786
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
4787
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4464
4788
|
)
|
|
4465
4789
|
}
|
|
4466
4790
|
});
|
|
4467
4791
|
logSynchroniserDiagnostics(result, options);
|
|
4468
4792
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
4793
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
4794
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
4795
|
+
options.log(
|
|
4796
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
4797
|
+
"error"
|
|
4798
|
+
);
|
|
4799
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
4800
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
4801
|
+
}
|
|
4469
4802
|
options.log(
|
|
4470
4803
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4471
4804
|
"warn"
|
|
4472
4805
|
);
|
|
4473
|
-
return false;
|
|
4806
|
+
return { verifyFatal: false };
|
|
4474
4807
|
}
|
|
4475
4808
|
if (result.code === 34) {
|
|
4476
4809
|
reportRecord(
|
|
@@ -4480,14 +4813,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4480
4813
|
null,
|
|
4481
4814
|
options
|
|
4482
4815
|
);
|
|
4483
|
-
return true;
|
|
4816
|
+
return { verifyFatal: true };
|
|
4484
4817
|
}
|
|
4485
4818
|
if (result.code === 33) {
|
|
4486
4819
|
options.log(
|
|
4487
4820
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4488
4821
|
"warn"
|
|
4489
4822
|
);
|
|
4490
|
-
return false;
|
|
4823
|
+
return { verifyFatal: false };
|
|
4491
4824
|
}
|
|
4492
4825
|
if (result.code !== 0) {
|
|
4493
4826
|
options.log(
|
|
@@ -4495,13 +4828,13 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4495
4828
|
"warn"
|
|
4496
4829
|
);
|
|
4497
4830
|
}
|
|
4498
|
-
return false;
|
|
4831
|
+
return { verifyFatal: false };
|
|
4499
4832
|
}
|
|
4500
4833
|
options.log(
|
|
4501
4834
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4502
4835
|
"debug"
|
|
4503
4836
|
);
|
|
4504
|
-
return false;
|
|
4837
|
+
return { verifyFatal: false };
|
|
4505
4838
|
}
|
|
4506
4839
|
function fileExists(path) {
|
|
4507
4840
|
try {
|
|
@@ -4588,7 +4921,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4588
4921
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4589
4922
|
return { verifyFatal: false };
|
|
4590
4923
|
}
|
|
4591
|
-
return
|
|
4924
|
+
return verifySessionDb(options, configPath, env);
|
|
4592
4925
|
}
|
|
4593
4926
|
|
|
4594
4927
|
// src/lib/opencode/session-db-provenance.ts
|
|
@@ -6369,6 +6702,7 @@ function createSessionStoreHealth(options = {}) {
|
|
|
6369
6702
|
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6370
6703
|
let lastUnwritableAt = null;
|
|
6371
6704
|
let lastWritableAt = null;
|
|
6705
|
+
const causes = /* @__PURE__ */ new Map();
|
|
6372
6706
|
return {
|
|
6373
6707
|
observeUnwritable: () => {
|
|
6374
6708
|
lastUnwritableAt = now();
|
|
@@ -6380,6 +6714,23 @@ function createSessionStoreHealth(options = {}) {
|
|
|
6380
6714
|
if (lastUnwritableAt === null) return false;
|
|
6381
6715
|
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6382
6716
|
return now() - lastUnwritableAt < stalenessMs;
|
|
6717
|
+
},
|
|
6718
|
+
observeCause: (sessionId, cause) => {
|
|
6719
|
+
if (sessionId === null) return;
|
|
6720
|
+
causes.delete(sessionId);
|
|
6721
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
6722
|
+
while (causes.size > 256) {
|
|
6723
|
+
const oldest = causes.keys().next().value;
|
|
6724
|
+
if (typeof oldest !== "string") break;
|
|
6725
|
+
causes.delete(oldest);
|
|
6726
|
+
}
|
|
6727
|
+
},
|
|
6728
|
+
readCause: (sessionId) => {
|
|
6729
|
+
const observed = causes.get(sessionId);
|
|
6730
|
+
if (!observed) return null;
|
|
6731
|
+
causes.delete(sessionId);
|
|
6732
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
6733
|
+
return observed.cause;
|
|
6383
6734
|
}
|
|
6384
6735
|
};
|
|
6385
6736
|
}
|
|
@@ -6840,6 +7191,10 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6840
7191
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6841
7192
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6842
7193
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7194
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7195
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7196
|
+
var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
|
|
7197
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6843
7198
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6844
7199
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
6845
7200
|
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
@@ -6997,6 +7352,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6997
7352
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6998
7353
|
*/
|
|
6999
7354
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7355
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
7000
7356
|
/**
|
|
7001
7357
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
7002
7358
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -7029,10 +7385,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7029
7385
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
7030
7386
|
*/
|
|
7031
7387
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7388
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7389
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
7032
7390
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
7033
7391
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
7034
7392
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
7035
7393
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7394
|
+
/**
|
|
7395
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7396
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7397
|
+
* identical-text turn, so it never produces a correlation.
|
|
7398
|
+
*/
|
|
7399
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
7036
7400
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
7037
7401
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
7038
7402
|
/**
|
|
@@ -7118,21 +7482,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7118
7482
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
7119
7483
|
*/
|
|
7120
7484
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
7121
|
-
/**
|
|
7122
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
7123
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
7124
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
7125
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
7126
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
7127
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
7128
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
7129
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
7130
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
7131
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
7132
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
7133
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
7134
|
-
*/
|
|
7135
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
7136
7485
|
/**
|
|
7137
7486
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
7138
7487
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -7150,8 +7499,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7150
7499
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
7151
7500
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
7152
7501
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
7153
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
7154
|
-
*
|
|
7502
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7503
|
+
* the terminal-report fence and is never sent again).
|
|
7155
7504
|
*/
|
|
7156
7505
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
7157
7506
|
/**
|
|
@@ -7469,6 +7818,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7469
7818
|
}
|
|
7470
7819
|
return ids;
|
|
7471
7820
|
}
|
|
7821
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7822
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7823
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7824
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7825
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7826
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7827
|
+
}
|
|
7828
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7829
|
+
}
|
|
7830
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7831
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7832
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7833
|
+
this.awaitingReadopt.delete(messageId);
|
|
7834
|
+
this.sessions.delete(conversationId);
|
|
7835
|
+
this.supersede(conversationId, sessionId);
|
|
7836
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7837
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7838
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7839
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7840
|
+
this.log({
|
|
7841
|
+
level: "error",
|
|
7842
|
+
message: errorMessage3,
|
|
7843
|
+
conversation_id: conversationId,
|
|
7844
|
+
message_id: messageId
|
|
7845
|
+
});
|
|
7846
|
+
const reportError = await this.reportUntrackableAck(
|
|
7847
|
+
conversationId,
|
|
7848
|
+
messageId,
|
|
7849
|
+
null,
|
|
7850
|
+
errorMessage3
|
|
7851
|
+
);
|
|
7852
|
+
if (reportError !== null) {
|
|
7853
|
+
this.log({
|
|
7854
|
+
level: "warn",
|
|
7855
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7856
|
+
conversation_id: conversationId,
|
|
7857
|
+
message_id: messageId
|
|
7858
|
+
});
|
|
7859
|
+
}
|
|
7860
|
+
if (source === "readopt") {
|
|
7861
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7862
|
+
}
|
|
7863
|
+
}
|
|
7472
7864
|
/**
|
|
7473
7865
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7474
7866
|
*
|
|
@@ -7648,6 +8040,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7648
8040
|
}
|
|
7649
8041
|
break;
|
|
7650
8042
|
}
|
|
8043
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8044
|
+
if (correlation) {
|
|
8045
|
+
correlation.attempts += 1;
|
|
8046
|
+
let correlatedId = null;
|
|
8047
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8048
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8049
|
+
conv,
|
|
8050
|
+
message,
|
|
8051
|
+
correlation.sessionId
|
|
8052
|
+
);
|
|
8053
|
+
if (polled.ok)
|
|
8054
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8055
|
+
} else if (correlation.knownUserIds === null) {
|
|
8056
|
+
this.log({
|
|
8057
|
+
level: "debug",
|
|
8058
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8059
|
+
conversation_id: conv.id,
|
|
8060
|
+
message_id: message.id
|
|
8061
|
+
});
|
|
8062
|
+
}
|
|
8063
|
+
if (correlatedId !== null) {
|
|
8064
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8065
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8066
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8067
|
+
this.dispatched.add(message.id);
|
|
8068
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8069
|
+
dispatched += 1;
|
|
8070
|
+
this.log({
|
|
8071
|
+
level: "info",
|
|
8072
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8073
|
+
conversation_id: conv.id,
|
|
8074
|
+
message_id: message.id
|
|
8075
|
+
});
|
|
8076
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8077
|
+
opencode_message_id: correlatedId
|
|
8078
|
+
});
|
|
8079
|
+
continue;
|
|
8080
|
+
}
|
|
8081
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8082
|
+
await this.handoffUntrackableCorrelation(
|
|
8083
|
+
conv.id,
|
|
8084
|
+
message.id,
|
|
8085
|
+
correlation.sessionId,
|
|
8086
|
+
correlation.attempts,
|
|
8087
|
+
"fresh"
|
|
8088
|
+
);
|
|
8089
|
+
break;
|
|
8090
|
+
}
|
|
8091
|
+
break;
|
|
8092
|
+
}
|
|
7651
8093
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7652
8094
|
if (effectiveOpencodeMessageId) {
|
|
7653
8095
|
const outcome = await this.resolveRedrive(
|
|
@@ -7669,6 +8111,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7669
8111
|
model: message.opencode_model ?? void 0
|
|
7670
8112
|
};
|
|
7671
8113
|
let opencodeMessageId;
|
|
8114
|
+
let preDispatchUserIds = null;
|
|
7672
8115
|
try {
|
|
7673
8116
|
this.log({
|
|
7674
8117
|
level: "info",
|
|
@@ -7698,7 +8141,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7698
8141
|
message.content,
|
|
7699
8142
|
options,
|
|
7700
8143
|
sendAttachments,
|
|
7701
|
-
this.openCodeClient
|
|
8144
|
+
this.openCodeClient,
|
|
8145
|
+
{
|
|
8146
|
+
onKnownUserIds: (ids) => {
|
|
8147
|
+
preDispatchUserIds = ids;
|
|
8148
|
+
},
|
|
8149
|
+
onKnownUserIdsError: (error2) => {
|
|
8150
|
+
this.log({
|
|
8151
|
+
level: "warn",
|
|
8152
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8153
|
+
conversation_id: conv.id,
|
|
8154
|
+
message_id: message.id
|
|
8155
|
+
});
|
|
8156
|
+
}
|
|
8157
|
+
}
|
|
7702
8158
|
)
|
|
7703
8159
|
);
|
|
7704
8160
|
} catch (err) {
|
|
@@ -7706,6 +8162,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7706
8162
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7707
8163
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7708
8164
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8165
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7709
8166
|
this.log({
|
|
7710
8167
|
level: "error",
|
|
7711
8168
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7781,39 +8238,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7781
8238
|
if (this.isV2) {
|
|
7782
8239
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7783
8240
|
}
|
|
7784
|
-
|
|
7785
|
-
|
|
7786
|
-
|
|
7787
|
-
|
|
7788
|
-
|
|
7789
|
-
|
|
7790
|
-
message_id: message.id
|
|
7791
|
-
});
|
|
7792
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7793
|
-
continue;
|
|
7794
|
-
}
|
|
7795
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7796
|
-
this.sessions.delete(conv.id);
|
|
7797
|
-
this.supersede(conv.id, sessionId);
|
|
7798
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8241
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8242
|
+
sessionId,
|
|
8243
|
+
content: message.content,
|
|
8244
|
+
attempts: 0,
|
|
8245
|
+
knownUserIds: preDispatchUserIds
|
|
8246
|
+
});
|
|
7799
8247
|
this.log({
|
|
7800
|
-
level: "
|
|
7801
|
-
message:
|
|
8248
|
+
level: "warn",
|
|
8249
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7802
8250
|
conversation_id: conv.id,
|
|
7803
8251
|
message_id: message.id
|
|
7804
8252
|
});
|
|
7805
|
-
|
|
7806
|
-
this.log({
|
|
7807
|
-
level: "warn",
|
|
7808
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7809
|
-
conversation_id: conv.id,
|
|
7810
|
-
message_id: message.id
|
|
7811
|
-
});
|
|
7812
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7813
|
-
});
|
|
8253
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7814
8254
|
break;
|
|
7815
8255
|
}
|
|
7816
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7817
8256
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7818
8257
|
this.releasedOpencodeIds.delete(message.id);
|
|
7819
8258
|
this.dispatched.add(message.id);
|
|
@@ -8418,23 +8857,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8418
8857
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8419
8858
|
return 1;
|
|
8420
8859
|
}
|
|
8421
|
-
/**
|
|
8422
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8423
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8424
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8425
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8426
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8427
|
-
* count, since a new session is a genuinely different attempt.
|
|
8428
|
-
*/
|
|
8429
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8430
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8431
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8432
|
-
existing.count += 1;
|
|
8433
|
-
return existing.count;
|
|
8434
|
-
}
|
|
8435
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8436
|
-
return 1;
|
|
8437
|
-
}
|
|
8438
8860
|
/** Record one queued give-up toward the never-started bound. */
|
|
8439
8861
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8440
8862
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8746,6 +9168,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8746
9168
|
b2LastDescendantCheckMs: 0,
|
|
8747
9169
|
b2RootOngoingHeldLogged: false,
|
|
8748
9170
|
b2AbandonedSignalled: false,
|
|
9171
|
+
b2AbandonedWithoutAnswer: false,
|
|
8749
9172
|
ambiguousPinnedSinceMs: 0,
|
|
8750
9173
|
ambiguousResolved: false,
|
|
8751
9174
|
sessionStoreFailurePending: false,
|
|
@@ -8770,6 +9193,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8770
9193
|
this.bufferedSessionErrors.delete(oldest);
|
|
8771
9194
|
}
|
|
8772
9195
|
}
|
|
9196
|
+
retainSessionError(event) {
|
|
9197
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9198
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9199
|
+
reason: event.reason,
|
|
9200
|
+
observedAt: this.now()
|
|
9201
|
+
});
|
|
9202
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9203
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9204
|
+
if (typeof oldest !== "string") break;
|
|
9205
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9206
|
+
}
|
|
9207
|
+
}
|
|
9208
|
+
readRetainedSessionError(sessionId) {
|
|
9209
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9210
|
+
if (!retained) return null;
|
|
9211
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9212
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9213
|
+
return null;
|
|
9214
|
+
}
|
|
9215
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9216
|
+
return retained.reason;
|
|
9217
|
+
}
|
|
8773
9218
|
/**
|
|
8774
9219
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8775
9220
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8865,6 +9310,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8865
9310
|
b2LastDescendantCheckMs: 0,
|
|
8866
9311
|
b2RootOngoingHeldLogged: false,
|
|
8867
9312
|
b2AbandonedSignalled: false,
|
|
9313
|
+
b2AbandonedWithoutAnswer: false,
|
|
8868
9314
|
ambiguousPinnedSinceMs: 0,
|
|
8869
9315
|
ambiguousResolved: false,
|
|
8870
9316
|
sessionStoreFailurePending: false,
|
|
@@ -9095,9 +9541,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9095
9541
|
return;
|
|
9096
9542
|
}
|
|
9097
9543
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9544
|
+
this.retainSessionError(event);
|
|
9098
9545
|
this.log({
|
|
9099
|
-
level: "
|
|
9100
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9546
|
+
level: "warn",
|
|
9547
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
9101
9548
|
conversation_id: watcher.conv.id
|
|
9102
9549
|
});
|
|
9103
9550
|
return;
|
|
@@ -9127,9 +9574,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9127
9574
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
9128
9575
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
9129
9576
|
if (state !== "queued") {
|
|
9577
|
+
this.retainSessionError(event);
|
|
9130
9578
|
this.log({
|
|
9131
|
-
level: "
|
|
9132
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9579
|
+
level: "warn",
|
|
9580
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
9133
9581
|
conversation_id: watcher.conv.id,
|
|
9134
9582
|
message_id: inFlight.evidentMessageId
|
|
9135
9583
|
});
|
|
@@ -9358,7 +9806,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9358
9806
|
if (state === "failed") {
|
|
9359
9807
|
this.anchorDeliveryDeadline(inFlight);
|
|
9360
9808
|
if (!inFlight.done) {
|
|
9361
|
-
const
|
|
9809
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
9810
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
9811
|
+
const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
9812
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
9813
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
9814
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
9815
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
9816
|
+
(detail) => detail !== null
|
|
9817
|
+
);
|
|
9818
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9362
9819
|
this.log({
|
|
9363
9820
|
level: "error",
|
|
9364
9821
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9444,10 +9901,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9444
9901
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9445
9902
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9446
9903
|
inFlight.b2AbandonedSignalled = false;
|
|
9904
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9447
9905
|
}
|
|
9448
9906
|
} else {
|
|
9449
9907
|
if (inFlight.b2AbandonedSignalled) {
|
|
9450
|
-
|
|
9908
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9909
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9910
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9911
|
+
sessionId,
|
|
9912
|
+
watcher,
|
|
9913
|
+
inFlight,
|
|
9914
|
+
messages,
|
|
9915
|
+
reply?.info?.finish ?? reply?.finish
|
|
9916
|
+
);
|
|
9917
|
+
} else {
|
|
9918
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9919
|
+
}
|
|
9451
9920
|
return;
|
|
9452
9921
|
}
|
|
9453
9922
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9475,16 +9944,36 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9475
9944
|
return;
|
|
9476
9945
|
}
|
|
9477
9946
|
inFlight.b2AbandonedSignalled = true;
|
|
9947
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9948
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9949
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9950
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9951
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9952
|
+
watched_for_ms: pinnedForMs,
|
|
9953
|
+
run_state: "failed",
|
|
9954
|
+
finish,
|
|
9955
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9956
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9957
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9958
|
+
});
|
|
9959
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9960
|
+
sessionId,
|
|
9961
|
+
watcher,
|
|
9962
|
+
inFlight,
|
|
9963
|
+
messages,
|
|
9964
|
+
finish
|
|
9965
|
+
);
|
|
9966
|
+
return;
|
|
9967
|
+
}
|
|
9478
9968
|
this.log({
|
|
9479
9969
|
level: "warn",
|
|
9480
9970
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9481
9971
|
conversation_id: conv.id,
|
|
9482
9972
|
message_id: id
|
|
9483
9973
|
});
|
|
9484
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9485
9974
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9486
9975
|
watched_for_ms: pinnedForMs,
|
|
9487
|
-
finish
|
|
9976
|
+
finish,
|
|
9488
9977
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9489
9978
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9490
9979
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9766,6 +10255,76 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9766
10255
|
}
|
|
9767
10256
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9768
10257
|
}
|
|
10258
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10259
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10260
|
+
const conv = watcher.conv;
|
|
10261
|
+
const legacyError = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
10262
|
+
const replyShape = describeAbandonedReplyShape(messages, inFlight.opencodeMessageId);
|
|
10263
|
+
const error2 = replyShape === null ? legacyError : `${legacyError} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
10264
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10265
|
+
this.log({
|
|
10266
|
+
level: "warn",
|
|
10267
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10268
|
+
conversation_id: conv.id,
|
|
10269
|
+
message_id: inFlight.evidentMessageId
|
|
10270
|
+
});
|
|
10271
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10272
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10273
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10274
|
+
messages,
|
|
10275
|
+
inFlight.opencodeMessageId,
|
|
10276
|
+
inFlight.evidentMessageId
|
|
10277
|
+
);
|
|
10278
|
+
try {
|
|
10279
|
+
await this.markFailed(
|
|
10280
|
+
conv.id,
|
|
10281
|
+
inFlight.evidentMessageId,
|
|
10282
|
+
sessionId,
|
|
10283
|
+
error2,
|
|
10284
|
+
usage,
|
|
10285
|
+
void 0,
|
|
10286
|
+
usageAgentName,
|
|
10287
|
+
subagentInvocations
|
|
10288
|
+
);
|
|
10289
|
+
} catch (err) {
|
|
10290
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10291
|
+
if (err instanceof ChannelTerminalError) {
|
|
10292
|
+
this.log({
|
|
10293
|
+
level: "warn",
|
|
10294
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10295
|
+
conversation_id: conv.id,
|
|
10296
|
+
message_id: inFlight.evidentMessageId
|
|
10297
|
+
});
|
|
10298
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10299
|
+
return;
|
|
10300
|
+
}
|
|
10301
|
+
if (this.now() >= inFlight.deadline) {
|
|
10302
|
+
this.log({
|
|
10303
|
+
level: "warn",
|
|
10304
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10305
|
+
conversation_id: conv.id,
|
|
10306
|
+
message_id: inFlight.evidentMessageId
|
|
10307
|
+
});
|
|
10308
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10309
|
+
return;
|
|
10310
|
+
}
|
|
10311
|
+
this.log({
|
|
10312
|
+
level: "warn",
|
|
10313
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10314
|
+
conversation_id: conv.id,
|
|
10315
|
+
message_id: inFlight.evidentMessageId
|
|
10316
|
+
});
|
|
10317
|
+
return;
|
|
10318
|
+
}
|
|
10319
|
+
inFlight.done = true;
|
|
10320
|
+
await this.reportSubagentAuthFailures(
|
|
10321
|
+
watcher.conv.id,
|
|
10322
|
+
inFlight.opencodeMessageId,
|
|
10323
|
+
inFlight.evidentMessageId,
|
|
10324
|
+
messages
|
|
10325
|
+
);
|
|
10326
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10327
|
+
}
|
|
9769
10328
|
/**
|
|
9770
10329
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9771
10330
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9828,7 +10387,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9828
10387
|
*/
|
|
9829
10388
|
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
9830
10389
|
const conv = watcher.conv;
|
|
9831
|
-
const error2 =
|
|
10390
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
9832
10391
|
this.log({
|
|
9833
10392
|
level: "error",
|
|
9834
10393
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
@@ -9919,11 +10478,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9919
10478
|
}
|
|
9920
10479
|
}
|
|
9921
10480
|
}
|
|
9922
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10481
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9923
10482
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10483
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10484
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10485
|
+
this.awaitingCorrelation.delete(id);
|
|
10486
|
+
this.log({
|
|
10487
|
+
level: "debug",
|
|
10488
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10489
|
+
message_id: id
|
|
10490
|
+
});
|
|
10491
|
+
}
|
|
10492
|
+
}
|
|
9924
10493
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9925
10494
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9926
10495
|
this.untrackableAck.delete(id);
|
|
10496
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9927
10497
|
this.untrackableAckSignalled.delete(id);
|
|
9928
10498
|
this.untrackableAckWarned.delete(id);
|
|
9929
10499
|
this.log({
|
|
@@ -9963,7 +10533,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9963
10533
|
bySession.set(row.opencode_session_id, list);
|
|
9964
10534
|
}
|
|
9965
10535
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9966
|
-
let messages;
|
|
10536
|
+
let messages = null;
|
|
9967
10537
|
try {
|
|
9968
10538
|
if (this.isV2) {
|
|
9969
10539
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9972,9 +10542,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9972
10542
|
level: "warn",
|
|
9973
10543
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9974
10544
|
});
|
|
9975
|
-
|
|
10545
|
+
} else {
|
|
10546
|
+
messages = snapshot;
|
|
9976
10547
|
}
|
|
9977
|
-
messages = snapshot;
|
|
9978
10548
|
} else {
|
|
9979
10549
|
const polled = await pollSessionMessagesForRedrive(
|
|
9980
10550
|
this.port,
|
|
@@ -9987,15 +10557,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9987
10557
|
level: "warn",
|
|
9988
10558
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9989
10559
|
});
|
|
9990
|
-
|
|
10560
|
+
} else {
|
|
10561
|
+
messages = polled.messages;
|
|
9991
10562
|
}
|
|
9992
|
-
messages = polled.messages;
|
|
9993
10563
|
}
|
|
9994
10564
|
} catch (err) {
|
|
9995
10565
|
this.log({
|
|
9996
10566
|
level: "warn",
|
|
9997
10567
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9998
10568
|
});
|
|
10569
|
+
}
|
|
10570
|
+
if (messages === null) {
|
|
10571
|
+
for (const row of sessionRows) {
|
|
10572
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10573
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10574
|
+
}
|
|
10575
|
+
}
|
|
9999
10576
|
continue;
|
|
10000
10577
|
}
|
|
10001
10578
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -10005,6 +10582,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10005
10582
|
}
|
|
10006
10583
|
}
|
|
10007
10584
|
}
|
|
10585
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10586
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10587
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10588
|
+
if (!correlation) return false;
|
|
10589
|
+
correlation.attempts += 1;
|
|
10590
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10591
|
+
if (correlatedId !== null) {
|
|
10592
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10593
|
+
const conv = this.convForRow(sessionId, row);
|
|
10594
|
+
const message = this.queuedMessageForRow(row);
|
|
10595
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10596
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10597
|
+
this.dispatched.add(row.id);
|
|
10598
|
+
this.readopted.add(row.id);
|
|
10599
|
+
this.ensureWatcherRunning(sessionId);
|
|
10600
|
+
this.log({
|
|
10601
|
+
level: "info",
|
|
10602
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10603
|
+
conversation_id: row.conversation_id,
|
|
10604
|
+
message_id: row.id
|
|
10605
|
+
});
|
|
10606
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10607
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10608
|
+
opencode_message_id: correlatedId
|
|
10609
|
+
});
|
|
10610
|
+
return true;
|
|
10611
|
+
}
|
|
10612
|
+
if (correlation.knownUserIds === null) {
|
|
10613
|
+
this.log({
|
|
10614
|
+
level: "debug",
|
|
10615
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10616
|
+
conversation_id: row.conversation_id,
|
|
10617
|
+
message_id: row.id
|
|
10618
|
+
});
|
|
10619
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10620
|
+
this.log({
|
|
10621
|
+
level: "warn",
|
|
10622
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10623
|
+
conversation_id: row.conversation_id,
|
|
10624
|
+
message_id: row.id
|
|
10625
|
+
});
|
|
10626
|
+
}
|
|
10627
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10628
|
+
await this.handoffUntrackableCorrelation(
|
|
10629
|
+
row.conversation_id,
|
|
10630
|
+
row.id,
|
|
10631
|
+
correlation.sessionId,
|
|
10632
|
+
correlation.attempts,
|
|
10633
|
+
"readopt"
|
|
10634
|
+
);
|
|
10635
|
+
}
|
|
10636
|
+
return true;
|
|
10637
|
+
}
|
|
10008
10638
|
/**
|
|
10009
10639
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
10010
10640
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -10012,8 +10642,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10012
10642
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
10013
10643
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
10014
10644
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
10015
|
-
* before the restart)
|
|
10016
|
-
*
|
|
10645
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10646
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10647
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10648
|
+
* `forceReadoptRun`):
|
|
10017
10649
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
10018
10650
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
10019
10651
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -10022,7 +10654,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10022
10654
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
10023
10655
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
10024
10656
|
* tracking the stored id so the reply correlates by it;
|
|
10025
|
-
* - `unknown`/null id →
|
|
10657
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10658
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
10026
10659
|
*
|
|
10027
10660
|
* Only `ChannelAuthError` propagates.
|
|
10028
10661
|
*/
|
|
@@ -10036,6 +10669,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10036
10669
|
});
|
|
10037
10670
|
return;
|
|
10038
10671
|
}
|
|
10672
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
10039
10673
|
const ocId = row.opencode_message_id;
|
|
10040
10674
|
const state = messageRunState(messages, ocId ?? "");
|
|
10041
10675
|
if (state === "done") {
|
|
@@ -10102,12 +10736,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10102
10736
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
10103
10737
|
this.dontRedispatch.delete(row.id);
|
|
10104
10738
|
this.untrackableAck.delete(row.id);
|
|
10739
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
10105
10740
|
this.untrackableAckSignalled.delete(row.id);
|
|
10106
10741
|
this.untrackableAckWarned.delete(row.id);
|
|
10107
10742
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
10108
10743
|
return;
|
|
10109
10744
|
}
|
|
10110
|
-
if (this.
|
|
10745
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10746
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10747
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10748
|
+
this.untrackableAckWarned.add(row.id);
|
|
10749
|
+
this.log({
|
|
10750
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10751
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10752
|
+
conversation_id: row.conversation_id,
|
|
10753
|
+
message_id: row.id
|
|
10754
|
+
});
|
|
10755
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10756
|
+
const reportError = await this.reportUntrackableAck(
|
|
10757
|
+
row.conversation_id,
|
|
10758
|
+
row.id,
|
|
10759
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10760
|
+
errorMessage3
|
|
10761
|
+
);
|
|
10762
|
+
if (reportError !== null) {
|
|
10763
|
+
this.log({
|
|
10764
|
+
level: "warn",
|
|
10765
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10766
|
+
conversation_id: row.conversation_id,
|
|
10767
|
+
message_id: row.id
|
|
10768
|
+
});
|
|
10769
|
+
}
|
|
10770
|
+
} else {
|
|
10771
|
+
this.log({
|
|
10772
|
+
level: "debug",
|
|
10773
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10774
|
+
conversation_id: row.conversation_id,
|
|
10775
|
+
message_id: row.id
|
|
10776
|
+
});
|
|
10777
|
+
}
|
|
10778
|
+
return;
|
|
10779
|
+
}
|
|
10780
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
10111
10781
|
this.log({
|
|
10112
10782
|
level: "debug",
|
|
10113
10783
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -10285,6 +10955,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10285
10955
|
}
|
|
10286
10956
|
this.dontRedispatch.delete(row.id);
|
|
10287
10957
|
this.untrackableAck.delete(row.id);
|
|
10958
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
10288
10959
|
this.untrackableAckSignalled.delete(row.id);
|
|
10289
10960
|
this.untrackableAckWarned.delete(row.id);
|
|
10290
10961
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -10304,8 +10975,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10304
10975
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
10305
10976
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
10306
10977
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
10307
|
-
* re-entry)
|
|
10308
|
-
*
|
|
10978
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10979
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10980
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
10309
10981
|
*
|
|
10310
10982
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
10311
10983
|
* `processed_at` (Invariant 1).
|
|
@@ -10368,6 +11040,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10368
11040
|
);
|
|
10369
11041
|
}
|
|
10370
11042
|
let ocId;
|
|
11043
|
+
let preDispatchUserIds = null;
|
|
10371
11044
|
try {
|
|
10372
11045
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10373
11046
|
sessionId,
|
|
@@ -10377,7 +11050,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10377
11050
|
row.content,
|
|
10378
11051
|
options,
|
|
10379
11052
|
sendAttachments,
|
|
10380
|
-
this.openCodeClient
|
|
11053
|
+
this.openCodeClient,
|
|
11054
|
+
{
|
|
11055
|
+
onKnownUserIds: (ids) => {
|
|
11056
|
+
preDispatchUserIds = ids;
|
|
11057
|
+
},
|
|
11058
|
+
onKnownUserIdsError: (error2) => {
|
|
11059
|
+
this.log({
|
|
11060
|
+
level: "warn",
|
|
11061
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11062
|
+
conversation_id: row.conversation_id,
|
|
11063
|
+
message_id: row.id
|
|
11064
|
+
});
|
|
11065
|
+
}
|
|
11066
|
+
}
|
|
10381
11067
|
)
|
|
10382
11068
|
);
|
|
10383
11069
|
} catch (err) {
|
|
@@ -10410,6 +11096,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10410
11096
|
}
|
|
10411
11097
|
} else {
|
|
10412
11098
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11099
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10413
11100
|
}
|
|
10414
11101
|
this.log({
|
|
10415
11102
|
level: "error",
|
|
@@ -10458,39 +11145,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10458
11145
|
}
|
|
10459
11146
|
if (ocId === null) {
|
|
10460
11147
|
this.awaitingReadopt.delete(row.id);
|
|
10461
|
-
|
|
10462
|
-
|
|
10463
|
-
|
|
10464
|
-
|
|
10465
|
-
|
|
10466
|
-
|
|
10467
|
-
this.log({
|
|
10468
|
-
level: "error",
|
|
10469
|
-
message: errorMessage3,
|
|
10470
|
-
conversation_id: row.conversation_id,
|
|
10471
|
-
message_id: row.id
|
|
10472
|
-
});
|
|
10473
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10474
|
-
this.log({
|
|
10475
|
-
level: "warn",
|
|
10476
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10477
|
-
conversation_id: row.conversation_id,
|
|
10478
|
-
message_id: row.id
|
|
10479
|
-
});
|
|
10480
|
-
});
|
|
10481
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10482
|
-
return;
|
|
10483
|
-
}
|
|
11148
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11149
|
+
sessionId,
|
|
11150
|
+
content: row.content,
|
|
11151
|
+
attempts: 0,
|
|
11152
|
+
knownUserIds: preDispatchUserIds
|
|
11153
|
+
});
|
|
10484
11154
|
this.log({
|
|
10485
11155
|
level: "warn",
|
|
10486
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11156
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10487
11157
|
conversation_id: row.conversation_id,
|
|
10488
11158
|
message_id: row.id
|
|
10489
11159
|
});
|
|
10490
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10491
11160
|
return;
|
|
10492
11161
|
}
|
|
10493
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10494
11162
|
const processedAtMs = this.processedAtMs(row);
|
|
10495
11163
|
this.registerReadopted(
|
|
10496
11164
|
readoptConv,
|
|
@@ -13969,6 +14637,34 @@ async function run(options) {
|
|
|
13969
14637
|
} else {
|
|
13970
14638
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13971
14639
|
}
|
|
14640
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14641
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14642
|
+
log2(state, message, level);
|
|
14643
|
+
if (state.interactive && !state.json) {
|
|
14644
|
+
logActivity(state, { type: "info", level, message });
|
|
14645
|
+
}
|
|
14646
|
+
};
|
|
14647
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14648
|
+
if (vaultResolution.error) {
|
|
14649
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14650
|
+
logCredentialVaultResolution(
|
|
14651
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14652
|
+
"warn"
|
|
14653
|
+
);
|
|
14654
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14655
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14656
|
+
logCredentialVaultResolution(
|
|
14657
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14658
|
+
"info"
|
|
14659
|
+
);
|
|
14660
|
+
} else {
|
|
14661
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14662
|
+
logCredentialVaultResolution(
|
|
14663
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14664
|
+
"debug"
|
|
14665
|
+
);
|
|
14666
|
+
}
|
|
14667
|
+
}
|
|
13972
14668
|
if (options.restoreRunnerCredentials) {
|
|
13973
14669
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13974
14670
|
const credentialContext = {
|
|
@@ -13991,7 +14687,12 @@ async function run(options) {
|
|
|
13991
14687
|
agentId: state.agentId,
|
|
13992
14688
|
authHeader: state.authHeader
|
|
13993
14689
|
}),
|
|
13994
|
-
|
|
14690
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
14691
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
14692
|
+
if (cause !== null && sessionId !== null) {
|
|
14693
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
14694
|
+
}
|
|
14695
|
+
}
|
|
13995
14696
|
);
|
|
13996
14697
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13997
14698
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
@@ -14002,22 +14703,27 @@ async function run(options) {
|
|
|
14002
14703
|
openCodeLogTail.stop();
|
|
14003
14704
|
};
|
|
14004
14705
|
let sessionDbVerifyFatal = false;
|
|
14706
|
+
let sessionDbVerifyFatalReason;
|
|
14005
14707
|
if (!options.restoreSessionDb) {
|
|
14006
|
-
|
|
14708
|
+
logActivity(state, {
|
|
14709
|
+
type: "info",
|
|
14710
|
+
level: "warn",
|
|
14711
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
14712
|
+
});
|
|
14007
14713
|
} else {
|
|
14008
14714
|
const health = await checkOpenCodeHealth(state.port);
|
|
14009
14715
|
if (health.healthy) {
|
|
14010
|
-
|
|
14011
|
-
|
|
14012
|
-
|
|
14013
|
-
"
|
|
14014
|
-
);
|
|
14716
|
+
logActivity(state, {
|
|
14717
|
+
type: "info",
|
|
14718
|
+
level: "warn",
|
|
14719
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
14720
|
+
});
|
|
14015
14721
|
} else if (health.authFailed) {
|
|
14016
|
-
|
|
14017
|
-
|
|
14018
|
-
|
|
14019
|
-
"
|
|
14020
|
-
);
|
|
14722
|
+
logActivity(state, {
|
|
14723
|
+
type: "info",
|
|
14724
|
+
level: "warn",
|
|
14725
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
14726
|
+
});
|
|
14021
14727
|
} else {
|
|
14022
14728
|
const result = await restoreAndVerifySessionDb({
|
|
14023
14729
|
dbPath: sessionDbPath(),
|
|
@@ -14034,12 +14740,13 @@ async function run(options) {
|
|
|
14034
14740
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
14035
14741
|
});
|
|
14036
14742
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
14743
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
14037
14744
|
}
|
|
14038
14745
|
}
|
|
14039
14746
|
reportSessionDbRecovery(state);
|
|
14040
14747
|
if (sessionDbVerifyFatal) {
|
|
14041
14748
|
throw new Error(
|
|
14042
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
14749
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
14043
14750
|
);
|
|
14044
14751
|
}
|
|
14045
14752
|
applyRunnerOpenCodeConfig({
|
|
@@ -14366,14 +15073,14 @@ async function run(options) {
|
|
|
14366
15073
|
},
|
|
14367
15074
|
onDisconnected: (code, reason, transition) => {
|
|
14368
15075
|
state.connected = false;
|
|
14369
|
-
const
|
|
15076
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
14370
15077
|
logActivity(state, {
|
|
14371
15078
|
type: "info",
|
|
14372
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15079
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14373
15080
|
});
|
|
14374
15081
|
emitAgentDisconnected(state.agentId, {
|
|
14375
15082
|
code,
|
|
14376
|
-
reason:
|
|
15083
|
+
reason: safeReason2,
|
|
14377
15084
|
connection_generation: transition.generation
|
|
14378
15085
|
});
|
|
14379
15086
|
if (state.interactive) displayStatus(state);
|
|
@@ -14536,6 +15243,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14536
15243
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14537
15244
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14538
15245
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15246
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14539
15247
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14540
15248
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14541
15249
|
"-a, --agent [id]",
|