@evident-ai/cli 3.5.2-dev.6d7d122 → 3.5.2-dev.7261603
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/index.js +2556 -631
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -316,14 +315,14 @@ function blank() {
|
|
|
316
315
|
console.log();
|
|
317
316
|
}
|
|
318
317
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
319
|
-
return new Promise((
|
|
318
|
+
return new Promise((resolve5) => {
|
|
320
319
|
process.stdout.write(chalk.dim(prompt));
|
|
321
320
|
const handler = () => {
|
|
322
321
|
process.stdin.removeListener("data", handler);
|
|
323
322
|
process.stdin.setRawMode?.(false);
|
|
324
323
|
process.stdin.pause();
|
|
325
324
|
console.log();
|
|
326
|
-
|
|
325
|
+
resolve5();
|
|
327
326
|
};
|
|
328
327
|
if (process.stdin.isTTY) {
|
|
329
328
|
process.stdin.setRawMode?.(true);
|
|
@@ -333,7 +332,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
333
332
|
});
|
|
334
333
|
}
|
|
335
334
|
function sleep(ms) {
|
|
336
|
-
return new Promise((
|
|
335
|
+
return new Promise((resolve5) => setTimeout(resolve5, ms));
|
|
337
336
|
}
|
|
338
337
|
|
|
339
338
|
// src/commands/login.ts
|
|
@@ -411,19 +410,19 @@ async function tokenLogin() {
|
|
|
411
410
|
);
|
|
412
411
|
blank();
|
|
413
412
|
process.stdout.write("Paste token: ");
|
|
414
|
-
const token = await new Promise((
|
|
413
|
+
const token = await new Promise((resolve5) => {
|
|
415
414
|
let data = "";
|
|
416
415
|
process.stdin.setEncoding("utf8");
|
|
417
416
|
process.stdin.on("data", (chunk) => {
|
|
418
417
|
data += chunk;
|
|
419
418
|
});
|
|
420
419
|
process.stdin.on("end", () => {
|
|
421
|
-
|
|
420
|
+
resolve5(data.trim());
|
|
422
421
|
});
|
|
423
422
|
if (process.stdin.isTTY) {
|
|
424
423
|
process.stdin.once("data", (chunk) => {
|
|
425
424
|
process.stdin.pause();
|
|
426
|
-
|
|
425
|
+
resolve5(chunk.toString().trim());
|
|
427
426
|
});
|
|
428
427
|
process.stdin.resume();
|
|
429
428
|
}
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1064,12 +1158,82 @@ async function claudeUsage() {
|
|
|
1064
1158
|
// src/commands/run.ts
|
|
1065
1159
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
1160
|
import { homedir as homedir7 } from "node:os";
|
|
1067
|
-
import { isAbsolute as
|
|
1161
|
+
import { isAbsolute as isAbsolute4, join as join12, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1162
|
import chalk7 from "chalk";
|
|
1069
1163
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1129,7 +1293,11 @@ var FORWARD_FAILURE_REASON_HEADER = "X-Evident-Failure-Reason";
|
|
|
1129
1293
|
function log(level, event, fields) {
|
|
1130
1294
|
const method = level === "debug" ? "log" : level;
|
|
1131
1295
|
try {
|
|
1132
|
-
|
|
1296
|
+
const payload = Object.fromEntries(
|
|
1297
|
+
Object.entries({ level, event, ...fields }).filter(([, value]) => value !== void 0)
|
|
1298
|
+
);
|
|
1299
|
+
const secondArgument = globalThis.navigator?.userAgent === "Cloudflare-Workers" ? payload : JSON.stringify(payload);
|
|
1300
|
+
console[method]("[evident]", secondArgument);
|
|
1133
1301
|
} catch (err) {
|
|
1134
1302
|
console.error(
|
|
1135
1303
|
"[evident] log_serialize_failed",
|
|
@@ -1158,7 +1326,7 @@ import ora4 from "ora";
|
|
|
1158
1326
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1327
|
|
|
1160
1328
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1329
|
+
var CLI_VERSION = (true ? "3.5.2-dev.7261603" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1330
|
function getCliVersion() {
|
|
1163
1331
|
return CLI_VERSION;
|
|
1164
1332
|
}
|
|
@@ -1319,8 +1487,591 @@ var EventTypes = {
|
|
|
1319
1487
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1320
1488
|
};
|
|
1321
1489
|
|
|
1322
|
-
// src/lib/
|
|
1490
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1491
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1492
|
+
import { dirname as dirname2 } from "node:path";
|
|
1493
|
+
|
|
1494
|
+
// src/lib/diagnostic-redaction.ts
|
|
1323
1495
|
import { homedir as homedir2 } from "node:os";
|
|
1496
|
+
function redactHomeDirectory(message) {
|
|
1497
|
+
const home = homedir2();
|
|
1498
|
+
if (!home || home === "/") {
|
|
1499
|
+
return message;
|
|
1500
|
+
}
|
|
1501
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1502
|
+
let result = "";
|
|
1503
|
+
let searchFrom = 0;
|
|
1504
|
+
let homeIndex;
|
|
1505
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1506
|
+
const homeEnd = homeIndex + home.length;
|
|
1507
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1508
|
+
const nextCharacter = message[homeEnd];
|
|
1509
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1510
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1511
|
+
result += atPathBoundary ? "~" : home;
|
|
1512
|
+
searchFrom = homeEnd;
|
|
1513
|
+
}
|
|
1514
|
+
return result + message.slice(searchFrom);
|
|
1515
|
+
}
|
|
1516
|
+
function redactDiagnosticText(message) {
|
|
1517
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1518
|
+
return redactHomeDirectory(redacted);
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1522
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1523
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1524
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1525
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1526
|
+
function errorMessage(error2) {
|
|
1527
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1528
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1529
|
+
}
|
|
1530
|
+
function errorMessages(error2) {
|
|
1531
|
+
const messages = [];
|
|
1532
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1533
|
+
let current = error2;
|
|
1534
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1535
|
+
visited.add(current);
|
|
1536
|
+
if (typeof current === "string") {
|
|
1537
|
+
messages.push(current);
|
|
1538
|
+
break;
|
|
1539
|
+
}
|
|
1540
|
+
if (current instanceof Error) {
|
|
1541
|
+
messages.push(current.name, current.message);
|
|
1542
|
+
current = current.cause;
|
|
1543
|
+
continue;
|
|
1544
|
+
}
|
|
1545
|
+
if (typeof current === "object") {
|
|
1546
|
+
const record = current;
|
|
1547
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1548
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1549
|
+
current = record.cause;
|
|
1550
|
+
continue;
|
|
1551
|
+
}
|
|
1552
|
+
break;
|
|
1553
|
+
}
|
|
1554
|
+
return messages;
|
|
1555
|
+
}
|
|
1556
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1557
|
+
return errorMessages(error2).some(
|
|
1558
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1559
|
+
message
|
|
1560
|
+
)
|
|
1561
|
+
);
|
|
1562
|
+
}
|
|
1563
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1564
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1565
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1566
|
+
function isCauseBearingText(text) {
|
|
1567
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1568
|
+
}
|
|
1569
|
+
function structuredLogFields(line) {
|
|
1570
|
+
const fields = [];
|
|
1571
|
+
for (let index = 0; index < line.length; index++) {
|
|
1572
|
+
if (line[index] === '"') {
|
|
1573
|
+
index++;
|
|
1574
|
+
while (index < line.length) {
|
|
1575
|
+
if (line[index] === "\\") {
|
|
1576
|
+
index++;
|
|
1577
|
+
} else if (line[index] === '"') {
|
|
1578
|
+
break;
|
|
1579
|
+
}
|
|
1580
|
+
index++;
|
|
1581
|
+
}
|
|
1582
|
+
continue;
|
|
1583
|
+
}
|
|
1584
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1585
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1586
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1587
|
+
const valueStart = index + key.length;
|
|
1588
|
+
if (line[valueStart] !== '"') {
|
|
1589
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1590
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1591
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1592
|
+
continue;
|
|
1593
|
+
}
|
|
1594
|
+
let valueEnd = valueStart + 1;
|
|
1595
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1596
|
+
if (line[valueEnd] === "\\") {
|
|
1597
|
+
valueEnd++;
|
|
1598
|
+
} else if (line[valueEnd] === '"') {
|
|
1599
|
+
break;
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1603
|
+
index = valueEnd;
|
|
1604
|
+
}
|
|
1605
|
+
return fields;
|
|
1606
|
+
}
|
|
1607
|
+
function structuredCauseMessages(text) {
|
|
1608
|
+
const fields = structuredLogFields(text);
|
|
1609
|
+
if (fields.length > 0) return fields;
|
|
1610
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1611
|
+
}
|
|
1612
|
+
function causeTokenFromText(text) {
|
|
1613
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1614
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1615
|
+
const match = pattern.exec(text);
|
|
1616
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1617
|
+
return match[1];
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
return null;
|
|
1621
|
+
}
|
|
1622
|
+
function sessionStoreCauseToken(value) {
|
|
1623
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1624
|
+
const token = causeTokenFromText(message);
|
|
1625
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1626
|
+
}
|
|
1627
|
+
return null;
|
|
1628
|
+
}
|
|
1629
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1630
|
+
try {
|
|
1631
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1632
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1633
|
+
if (availableBytes < requiredBytes) {
|
|
1634
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1635
|
+
}
|
|
1636
|
+
return null;
|
|
1637
|
+
} catch (err) {
|
|
1638
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
function readLogicalBytes(db) {
|
|
1642
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1643
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1644
|
+
return pageCount * pageSize;
|
|
1645
|
+
}
|
|
1646
|
+
function readFreelistCount(db) {
|
|
1647
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1648
|
+
}
|
|
1649
|
+
function readCheckpointResult(db) {
|
|
1650
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1651
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1652
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1653
|
+
}
|
|
1654
|
+
async function probeReclaimAvailability(input) {
|
|
1655
|
+
const { dbPath, requiredBytes } = input;
|
|
1656
|
+
let sqlite;
|
|
1657
|
+
try {
|
|
1658
|
+
sqlite = await import("node:sqlite");
|
|
1659
|
+
} catch (err) {
|
|
1660
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1661
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1662
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1663
|
+
}
|
|
1664
|
+
let autoVacuum = null;
|
|
1665
|
+
try {
|
|
1666
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1667
|
+
try {
|
|
1668
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1669
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1670
|
+
} finally {
|
|
1671
|
+
db.close();
|
|
1672
|
+
}
|
|
1673
|
+
} catch (err) {
|
|
1674
|
+
console.warn(
|
|
1675
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1676
|
+
);
|
|
1677
|
+
}
|
|
1678
|
+
if (autoVacuum !== 0) return null;
|
|
1679
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1680
|
+
}
|
|
1681
|
+
async function reclaimSessionDbSpace(input) {
|
|
1682
|
+
const {
|
|
1683
|
+
dbPath,
|
|
1684
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1685
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1686
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1687
|
+
now = () => performance.now(),
|
|
1688
|
+
hasLiveTurn = false,
|
|
1689
|
+
hasProvenanceAnomaly = false
|
|
1690
|
+
} = input;
|
|
1691
|
+
let sqlite;
|
|
1692
|
+
try {
|
|
1693
|
+
sqlite = await import("node:sqlite");
|
|
1694
|
+
} catch (err) {
|
|
1695
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1696
|
+
console.warn(
|
|
1697
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1698
|
+
);
|
|
1699
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1700
|
+
}
|
|
1701
|
+
const { DatabaseSync } = sqlite;
|
|
1702
|
+
let db;
|
|
1703
|
+
try {
|
|
1704
|
+
db = new DatabaseSync(dbPath);
|
|
1705
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1706
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1707
|
+
if (autoVacuum === 0) {
|
|
1708
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1709
|
+
const detail = [
|
|
1710
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1711
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1712
|
+
].join(" and ");
|
|
1713
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1714
|
+
return {
|
|
1715
|
+
ok: false,
|
|
1716
|
+
skipped: "full-vacuum-blocked",
|
|
1717
|
+
detail
|
|
1718
|
+
};
|
|
1719
|
+
}
|
|
1720
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1721
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1722
|
+
if (skipReason !== null) {
|
|
1723
|
+
console.warn(
|
|
1724
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1725
|
+
);
|
|
1726
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1727
|
+
}
|
|
1728
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1729
|
+
const freelistBefore = readFreelistCount(db);
|
|
1730
|
+
const startedAt = now();
|
|
1731
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1732
|
+
db.exec("VACUUM");
|
|
1733
|
+
const afterBytes = readLogicalBytes(db);
|
|
1734
|
+
const freelistAfter = readFreelistCount(db);
|
|
1735
|
+
const checkpointStartedAt = now();
|
|
1736
|
+
const checkpoint = readCheckpointResult(db);
|
|
1737
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1738
|
+
return {
|
|
1739
|
+
ok: true,
|
|
1740
|
+
mode: "convert",
|
|
1741
|
+
beforeBytes,
|
|
1742
|
+
afterBytes,
|
|
1743
|
+
freelistBefore,
|
|
1744
|
+
freelistAfter,
|
|
1745
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1746
|
+
stoppedBy: "freelist-drained",
|
|
1747
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1748
|
+
checkpointElapsedMs,
|
|
1749
|
+
checkpoint
|
|
1750
|
+
};
|
|
1751
|
+
}
|
|
1752
|
+
if (autoVacuum === 2) {
|
|
1753
|
+
if (hasLiveTurn) {
|
|
1754
|
+
const detail = "a session turn is live";
|
|
1755
|
+
console.warn(
|
|
1756
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1757
|
+
);
|
|
1758
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1759
|
+
}
|
|
1760
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1761
|
+
const freelistBefore = readFreelistCount(db);
|
|
1762
|
+
const startedAt = now();
|
|
1763
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1764
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1765
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1766
|
+
let pagesAttempted = 0;
|
|
1767
|
+
let freelistAfter = freelistBefore;
|
|
1768
|
+
let stoppedBy = "freelist-drained";
|
|
1769
|
+
while (freelistAfter > 0) {
|
|
1770
|
+
if (pagesAttempted >= pageLimit) {
|
|
1771
|
+
stoppedBy = "page-ceiling";
|
|
1772
|
+
break;
|
|
1773
|
+
}
|
|
1774
|
+
if (now() - startedAt >= budget) {
|
|
1775
|
+
stoppedBy = "time-budget";
|
|
1776
|
+
break;
|
|
1777
|
+
}
|
|
1778
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1779
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1780
|
+
pagesAttempted += pages;
|
|
1781
|
+
freelistAfter = readFreelistCount(db);
|
|
1782
|
+
}
|
|
1783
|
+
const afterBytes = readLogicalBytes(db);
|
|
1784
|
+
const checkpointStartedAt = now();
|
|
1785
|
+
const checkpoint = readCheckpointResult(db);
|
|
1786
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1787
|
+
return {
|
|
1788
|
+
ok: true,
|
|
1789
|
+
mode: "incremental",
|
|
1790
|
+
beforeBytes,
|
|
1791
|
+
afterBytes,
|
|
1792
|
+
freelistBefore,
|
|
1793
|
+
freelistAfter,
|
|
1794
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1795
|
+
stoppedBy,
|
|
1796
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1797
|
+
checkpointElapsedMs,
|
|
1798
|
+
checkpoint
|
|
1799
|
+
};
|
|
1800
|
+
}
|
|
1801
|
+
console.warn(
|
|
1802
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1803
|
+
);
|
|
1804
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1805
|
+
} catch (err) {
|
|
1806
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1807
|
+
return {
|
|
1808
|
+
ok: false,
|
|
1809
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1810
|
+
detail: errorMessage(err)
|
|
1811
|
+
};
|
|
1812
|
+
} finally {
|
|
1813
|
+
db?.close();
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1817
|
+
// src/lib/opencode/dependency-install-diagnostic.ts
|
|
1818
|
+
import { lstatSync, realpathSync } from "node:fs";
|
|
1819
|
+
import { basename, dirname as dirname3, isAbsolute, join as join2, relative, resolve, sep } from "node:path";
|
|
1820
|
+
var DEPENDENCY_INSTALL_FAILURE_MESSAGE = "background dependency install failed";
|
|
1821
|
+
var MAX_DISTINCT_DIAGNOSTICS = 32;
|
|
1822
|
+
var EFFECT = "OpenCode could not install background dependencies for this scope; its plugins and tools may be unavailable";
|
|
1823
|
+
function readOpenCodeLogField(line, key) {
|
|
1824
|
+
const match = line.match(new RegExp(`(?:^|\\s)${key}=(?:"((?:\\\\.|[^"])*)"|(\\S+))`));
|
|
1825
|
+
if (!match) return void 0;
|
|
1826
|
+
if (match[1] === void 0) return match[2];
|
|
1827
|
+
return match[1].replace(/\\(["\\n])/g, (_match, escaped) => {
|
|
1828
|
+
return escaped === "n" ? "\n" : escaped;
|
|
1829
|
+
});
|
|
1830
|
+
}
|
|
1831
|
+
function parseTrigger(line) {
|
|
1832
|
+
if (readOpenCodeLogField(line, "message") !== DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
1833
|
+
return null;
|
|
1834
|
+
}
|
|
1835
|
+
const error2 = readOpenCodeLogField(line, "error");
|
|
1836
|
+
if (!error2 || !/\bEACCES\b/.test(error2)) return null;
|
|
1837
|
+
const errnoMatch = error2.match(/\bEACCES:\s*permission denied,\s*([a-z_]+)\s+'([^']*)'/);
|
|
1838
|
+
const capturedPath = errnoMatch?.[2];
|
|
1839
|
+
return {
|
|
1840
|
+
dir: readOpenCodeLogField(line, "dir"),
|
|
1841
|
+
op: errnoMatch?.[1],
|
|
1842
|
+
path: capturedPath && isAbsolute(capturedPath) ? capturedPath : void 0
|
|
1843
|
+
};
|
|
1844
|
+
}
|
|
1845
|
+
function getErrorCode(error2) {
|
|
1846
|
+
if (typeof error2 === "object" && error2 !== null && "code" in error2) {
|
|
1847
|
+
const code = error2.code;
|
|
1848
|
+
if (typeof code === "string") return code;
|
|
1849
|
+
}
|
|
1850
|
+
return "UNKNOWN";
|
|
1851
|
+
}
|
|
1852
|
+
function isWithin(path, root) {
|
|
1853
|
+
const candidate = resolve(path);
|
|
1854
|
+
const resolvedRoot = resolve(root);
|
|
1855
|
+
return candidate === resolvedRoot || candidate.startsWith(`${resolvedRoot}${sep}`);
|
|
1856
|
+
}
|
|
1857
|
+
function rootSpecs(context) {
|
|
1858
|
+
return [
|
|
1859
|
+
{ token: "~", configured: resolve(context.homeDir) },
|
|
1860
|
+
{ token: "$WORKSPACE", configured: resolve(context.cwd) }
|
|
1861
|
+
];
|
|
1862
|
+
}
|
|
1863
|
+
function resolveRoots(context) {
|
|
1864
|
+
const resolved = [];
|
|
1865
|
+
const unresolved = [];
|
|
1866
|
+
for (const spec of rootSpecs(context)) {
|
|
1867
|
+
try {
|
|
1868
|
+
resolved.push({ ...spec, real: realpathSync(spec.configured) });
|
|
1869
|
+
} catch (_error) {
|
|
1870
|
+
unresolved.push(spec);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
return { resolved, unresolved };
|
|
1874
|
+
}
|
|
1875
|
+
function resolveNearestAncestor(failingPath, remaining, error2) {
|
|
1876
|
+
let ancestor = dirname3(failingPath);
|
|
1877
|
+
const suffix = [basename(failingPath), ...remaining];
|
|
1878
|
+
while (ancestor) {
|
|
1879
|
+
try {
|
|
1880
|
+
return {
|
|
1881
|
+
path: resolve(realpathSync(ancestor), ...suffix),
|
|
1882
|
+
failure: { path: resolve(failingPath), error: error2 }
|
|
1883
|
+
};
|
|
1884
|
+
} catch (ancestorError) {
|
|
1885
|
+
if (ancestor === dirname3(ancestor) || !getErrorCode(ancestorError).startsWith("E")) {
|
|
1886
|
+
return null;
|
|
1887
|
+
}
|
|
1888
|
+
suffix.unshift(basename(ancestor));
|
|
1889
|
+
ancestor = dirname3(ancestor);
|
|
1890
|
+
}
|
|
1891
|
+
}
|
|
1892
|
+
return null;
|
|
1893
|
+
}
|
|
1894
|
+
function resolveExistingPath(absolutePath) {
|
|
1895
|
+
if (!isAbsolute(absolutePath)) return null;
|
|
1896
|
+
const candidate = resolve(absolutePath);
|
|
1897
|
+
const remaining = [];
|
|
1898
|
+
let current = candidate;
|
|
1899
|
+
while (current) {
|
|
1900
|
+
try {
|
|
1901
|
+
const stat2 = lstatSync(current);
|
|
1902
|
+
if (stat2.isSymbolicLink()) {
|
|
1903
|
+
if (remaining.length === 0) {
|
|
1904
|
+
return { path: join2(realpathSync(dirname3(current)), basename(current)) };
|
|
1905
|
+
}
|
|
1906
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1907
|
+
}
|
|
1908
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1909
|
+
} catch (error2) {
|
|
1910
|
+
const code = getErrorCode(error2);
|
|
1911
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
1912
|
+
return resolveNearestAncestor(current, remaining, error2);
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
if (current === dirname3(current)) return null;
|
|
1916
|
+
remaining.unshift(basename(current));
|
|
1917
|
+
current = dirname3(current);
|
|
1918
|
+
}
|
|
1919
|
+
return null;
|
|
1920
|
+
}
|
|
1921
|
+
function tokenise(absolutePath, roots) {
|
|
1922
|
+
const pathResolution = resolveExistingPath(absolutePath);
|
|
1923
|
+
if (!pathResolution) return null;
|
|
1924
|
+
const root = roots.reduce((selected, candidate) => {
|
|
1925
|
+
if (!isWithin(pathResolution.path, candidate.real)) return selected;
|
|
1926
|
+
if (!selected || candidate.real.length > selected.real.length || candidate.real.length === selected.real.length && candidate.token === "$WORKSPACE") {
|
|
1927
|
+
return candidate;
|
|
1928
|
+
}
|
|
1929
|
+
return selected;
|
|
1930
|
+
}, null);
|
|
1931
|
+
if (!root) return null;
|
|
1932
|
+
const suffix = relative(root.real, pathResolution.path);
|
|
1933
|
+
return {
|
|
1934
|
+
value: suffix ? `${root.token}/${suffix.split(sep).join("/")}` : root.token,
|
|
1935
|
+
root,
|
|
1936
|
+
resolutionFailure: pathResolution.failure
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
function classifyScope(dir, roots) {
|
|
1940
|
+
if (!dir) return "other";
|
|
1941
|
+
const tokenisedDir = tokenise(dir, roots.resolved);
|
|
1942
|
+
if (tokenisedDir) return tokenisedDir.root.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1943
|
+
for (const spec of roots.unresolved) {
|
|
1944
|
+
if (isWithin(dir, spec.configured)) {
|
|
1945
|
+
return spec.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
return "other";
|
|
1949
|
+
}
|
|
1950
|
+
function unresolvedRootForPath(path, unresolved) {
|
|
1951
|
+
return unresolved.find((spec) => isWithin(path, spec.configured));
|
|
1952
|
+
}
|
|
1953
|
+
function statType(stat2) {
|
|
1954
|
+
if (stat2.isDirectory()) return "d";
|
|
1955
|
+
if (stat2.isFile()) return "f";
|
|
1956
|
+
if (stat2.isSymbolicLink()) return "l";
|
|
1957
|
+
return "o";
|
|
1958
|
+
}
|
|
1959
|
+
function renderStat(name, stat2) {
|
|
1960
|
+
const mode = (stat2.mode & 4095).toString(8).padStart(4, "0");
|
|
1961
|
+
return `${name}:${statType(stat2)}:${stat2.uid}:${stat2.gid}:${mode}`;
|
|
1962
|
+
}
|
|
1963
|
+
function renderStatError(name, error2) {
|
|
1964
|
+
const code = getErrorCode(error2);
|
|
1965
|
+
return `${name}:${renderErrorCode(code)}`;
|
|
1966
|
+
}
|
|
1967
|
+
function renderErrorCode(code) {
|
|
1968
|
+
if (code === "ENOENT") return "missing";
|
|
1969
|
+
return code.startsWith("E") ? code : `E${code}`;
|
|
1970
|
+
}
|
|
1971
|
+
function inspectChain(rootReal, rootToken, absolutePath, resolutionFailure) {
|
|
1972
|
+
const target = resolve(absolutePath);
|
|
1973
|
+
const relativePath = relative(rootReal, target);
|
|
1974
|
+
const components = relativePath ? relativePath.split(sep) : [];
|
|
1975
|
+
const entries = [];
|
|
1976
|
+
let current = rootReal;
|
|
1977
|
+
let status2 = "complete";
|
|
1978
|
+
let finalStat;
|
|
1979
|
+
for (const name of [rootToken, ...components]) {
|
|
1980
|
+
if (name !== rootToken) current = join2(current, name);
|
|
1981
|
+
if (resolutionFailure && resolve(current) === resolve(resolutionFailure.path)) {
|
|
1982
|
+
entries.push(renderStatError(name, resolutionFailure.error));
|
|
1983
|
+
status2 = "stat_failed";
|
|
1984
|
+
break;
|
|
1985
|
+
}
|
|
1986
|
+
try {
|
|
1987
|
+
const stat2 = lstatSync(current);
|
|
1988
|
+
entries.push(renderStat(name, stat2));
|
|
1989
|
+
finalStat = stat2;
|
|
1990
|
+
} catch (error2) {
|
|
1991
|
+
entries.push(renderStatError(name, error2));
|
|
1992
|
+
if (getErrorCode(error2) !== "ENOENT") status2 = "stat_failed";
|
|
1993
|
+
break;
|
|
1994
|
+
}
|
|
1995
|
+
}
|
|
1996
|
+
let symlinkTarget;
|
|
1997
|
+
let symlinkMeta;
|
|
1998
|
+
if (finalStat?.isSymbolicLink() && entries.length === components.length + 1) {
|
|
1999
|
+
try {
|
|
2000
|
+
symlinkTarget = realpathSync(current);
|
|
2001
|
+
try {
|
|
2002
|
+
symlinkMeta = renderStat("", lstatSync(symlinkTarget)).slice(1);
|
|
2003
|
+
} catch (error2) {
|
|
2004
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2005
|
+
status2 = "stat_failed";
|
|
2006
|
+
}
|
|
2007
|
+
} catch (error2) {
|
|
2008
|
+
symlinkTarget = "<unresolved>";
|
|
2009
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2010
|
+
status2 = "stat_failed";
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
return { status: status2, chain: entries.join(","), symlinkTarget, symlinkMeta };
|
|
2014
|
+
}
|
|
2015
|
+
function buildDiagnostic(line, context) {
|
|
2016
|
+
const trigger = parseTrigger(line);
|
|
2017
|
+
if (!trigger) return null;
|
|
2018
|
+
const roots = resolveRoots(context);
|
|
2019
|
+
const scope = classifyScope(trigger.dir, roots);
|
|
2020
|
+
const pathSource = trigger.path ? "error" : trigger.dir ? "dir" : "none";
|
|
2021
|
+
const inspectedPath = trigger.path ?? trigger.dir;
|
|
2022
|
+
const microvmId = context.microvmId?.trim() || "unknown";
|
|
2023
|
+
const euid = context.euid === void 0 ? "unavailable" : String(context.euid);
|
|
2024
|
+
const egid = context.egid === void 0 ? "unavailable" : String(context.egid);
|
|
2025
|
+
const op = trigger.op ?? "unknown";
|
|
2026
|
+
if (!inspectedPath) {
|
|
2027
|
+
return {
|
|
2028
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=no_path path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2029
|
+
localDetail: null
|
|
2030
|
+
};
|
|
2031
|
+
}
|
|
2032
|
+
const tokenisedPath = tokenise(inspectedPath, roots.resolved);
|
|
2033
|
+
if (!tokenisedPath) {
|
|
2034
|
+
const unresolvedRoot = unresolvedRootForPath(inspectedPath, roots.unresolved);
|
|
2035
|
+
if (unresolvedRoot) {
|
|
2036
|
+
return {
|
|
2037
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=root_unresolved path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2038
|
+
localDetail: null
|
|
2039
|
+
};
|
|
2040
|
+
}
|
|
2041
|
+
return {
|
|
2042
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=outside_roots path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=<outside-approved-roots> cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2043
|
+
localDetail: `OPENCODE-DEPENDENCY-INSTALL-EACCES (local only): outside approved roots: ${inspectedPath}`
|
|
2044
|
+
};
|
|
2045
|
+
}
|
|
2046
|
+
const inspection = inspectChain(
|
|
2047
|
+
tokenisedPath.root.real,
|
|
2048
|
+
tokenisedPath.root.token,
|
|
2049
|
+
inspectedPath,
|
|
2050
|
+
tokenisedPath.resolutionFailure
|
|
2051
|
+
);
|
|
2052
|
+
const optionalSymlinkFields = inspection.symlinkTarget ? ` symlink_target=${tokenise(inspection.symlinkTarget, roots.resolved)?.value ?? "<outside-approved-roots>"}` + (inspection.symlinkMeta ? ` symlink_meta=${inspection.symlinkMeta}` : "") : "";
|
|
2053
|
+
return {
|
|
2054
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=${inspection.status} path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=${tokenisedPath.value} cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=${inspection.chain}${optionalSymlinkFields}`,
|
|
2055
|
+
localDetail: null
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
function createDependencyInstallDiagnosticReporter({
|
|
2059
|
+
context,
|
|
2060
|
+
emitEvent: emitEvent2,
|
|
2061
|
+
emitLocal
|
|
2062
|
+
}) {
|
|
2063
|
+
const emittedEvents = /* @__PURE__ */ new Set();
|
|
2064
|
+
return (line) => {
|
|
2065
|
+
const diagnostic = buildDiagnostic(line, context);
|
|
2066
|
+
if (!diagnostic || emittedEvents.has(diagnostic.event)) return;
|
|
2067
|
+
if (emittedEvents.size >= MAX_DISTINCT_DIAGNOSTICS) return;
|
|
2068
|
+
emittedEvents.add(diagnostic.event);
|
|
2069
|
+
emitEvent2(diagnostic.event);
|
|
2070
|
+
if (diagnostic.localDetail) emitLocal(diagnostic.localDetail);
|
|
2071
|
+
};
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1324
2075
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1325
2076
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1326
2077
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1337,11 +2088,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1337
2088
|
if (!levelMatch) return null;
|
|
1338
2089
|
const level = levelMatch[1].toUpperCase();
|
|
1339
2090
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1340
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
2091
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
2092
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1341
2093
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1342
2094
|
const parsed = {
|
|
1343
2095
|
level: level === "WARN" ? "warn" : "error",
|
|
1344
|
-
sessionID: sessionMatch?.[1]
|
|
2096
|
+
sessionID: sessionMatch?.[1],
|
|
2097
|
+
messageID: messageIDMatch?.[1]
|
|
1345
2098
|
};
|
|
1346
2099
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1347
2100
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1349,18 +2102,29 @@ function parseOpenCodeLogLine(line) {
|
|
|
1349
2102
|
}
|
|
1350
2103
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1351
2104
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1352
|
-
function
|
|
2105
|
+
function sanitiseDependencyInstallWarning(line) {
|
|
2106
|
+
const withoutDirectory = line.replace(
|
|
2107
|
+
/((?:^|\s)dir=)(?:"(?:\\.|[^"])*"|\/[^\s)"']+|\S+)/g,
|
|
2108
|
+
"$1<path>"
|
|
2109
|
+
);
|
|
2110
|
+
return withoutDirectory.replace(
|
|
2111
|
+
/((?:^|\s)error=)"((?:\\.|[^"])*)"/,
|
|
2112
|
+
(_match, prefix, error2) => `${prefix}"${error2.replace(/'\/[^']*'/g, "'<path>'")}"`
|
|
2113
|
+
);
|
|
2114
|
+
}
|
|
2115
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure, onDependencyInstallFailure) {
|
|
1353
2116
|
let buffer = Buffer.alloc(0);
|
|
1354
2117
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1355
2118
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1356
2119
|
const metadata = {
|
|
1357
2120
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
2121
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1358
2122
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1359
2123
|
};
|
|
1360
2124
|
forwardRunnerActivity(
|
|
1361
2125
|
{
|
|
1362
2126
|
level: parsed.level,
|
|
1363
|
-
error: line,
|
|
2127
|
+
error: parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE ? sanitiseDependencyInstallWarning(line) : line,
|
|
1364
2128
|
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1365
2129
|
source: "opencode"
|
|
1366
2130
|
},
|
|
@@ -1378,11 +2142,9 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1378
2142
|
repeatStates.delete(key);
|
|
1379
2143
|
}
|
|
1380
2144
|
};
|
|
1381
|
-
const
|
|
1382
|
-
const parsed = parseOpenCodeLogLine(line);
|
|
1383
|
-
if (!parsed) return;
|
|
2145
|
+
const forwardWithRepeatSuppression = (line, parsed) => {
|
|
1384
2146
|
const now = Date.now();
|
|
1385
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
2147
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1386
2148
|
expireRepeatStates(now, key);
|
|
1387
2149
|
if (key === void 0) {
|
|
1388
2150
|
forwardLine(line, parsed);
|
|
@@ -1403,6 +2165,35 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1403
2165
|
previous.parsed = parsed;
|
|
1404
2166
|
previous.repeatsSuppressed++;
|
|
1405
2167
|
};
|
|
2168
|
+
const flushLine = (line) => {
|
|
2169
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
2170
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
2171
|
+
const cause = sessionStoreCauseToken(line);
|
|
2172
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
2173
|
+
try {
|
|
2174
|
+
onSessionStoreWriteFailure({
|
|
2175
|
+
isDiskFailure,
|
|
2176
|
+
sessionId: parsed?.sessionID ?? null,
|
|
2177
|
+
cause
|
|
2178
|
+
});
|
|
2179
|
+
} catch (err) {
|
|
2180
|
+
reportTelemetryDiagnostic(
|
|
2181
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
2182
|
+
);
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
if (!parsed) return;
|
|
2186
|
+
forwardWithRepeatSuppression(line, parsed);
|
|
2187
|
+
if (onDependencyInstallFailure && parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
2188
|
+
try {
|
|
2189
|
+
onDependencyInstallFailure(line);
|
|
2190
|
+
} catch (err) {
|
|
2191
|
+
reportTelemetryDiagnostic(
|
|
2192
|
+
`Runner activity telemetry could not record dependency-install failure diagnostic: ${err instanceof Error ? err.message : String(err)}`
|
|
2193
|
+
);
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
};
|
|
1406
2197
|
const forward = ((chunk) => {
|
|
1407
2198
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1408
2199
|
let newlineIndex;
|
|
@@ -1428,34 +2219,38 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1428
2219
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1429
2220
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1430
2221
|
var MAX_METADATA_ENTRIES = 20;
|
|
1431
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1432
|
-
function redactHomeDirectory(message) {
|
|
1433
|
-
const home = homedir2();
|
|
1434
|
-
if (!home || home === "/") {
|
|
1435
|
-
return message;
|
|
1436
|
-
}
|
|
1437
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1438
|
-
let result = "";
|
|
1439
|
-
let searchFrom = 0;
|
|
1440
|
-
let homeIndex;
|
|
1441
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1442
|
-
const homeEnd = homeIndex + home.length;
|
|
1443
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1444
|
-
const nextCharacter = message[homeEnd];
|
|
1445
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1446
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1447
|
-
result += atPathBoundary ? "~" : home;
|
|
1448
|
-
searchFrom = homeEnd;
|
|
1449
|
-
}
|
|
1450
|
-
return result + message.slice(searchFrom);
|
|
1451
|
-
}
|
|
1452
|
-
function redactDiagnosticText(message) {
|
|
1453
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1454
|
-
return redactHomeDirectory(redacted);
|
|
1455
|
-
}
|
|
1456
2222
|
function truncate(message) {
|
|
1457
2223
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1458
|
-
|
|
2224
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
2225
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2226
|
+
let previousDropped = -1;
|
|
2227
|
+
while (dropped !== previousDropped) {
|
|
2228
|
+
previousDropped = dropped;
|
|
2229
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
2230
|
+
dropped = nextDropped;
|
|
2231
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2232
|
+
}
|
|
2233
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
2234
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
2235
|
+
const tailLength = remaining - headLength;
|
|
2236
|
+
const effectMatch = message.startsWith("OPENCODE-DEPENDENCY-INSTALL-EACCES:") ? message.match(/ effect="[^"]*"/) : null;
|
|
2237
|
+
if (effectMatch?.index !== void 0) {
|
|
2238
|
+
const effectEnd = effectMatch.index + effectMatch[0].length;
|
|
2239
|
+
if (effectEnd > headLength && effectEnd < message.length) {
|
|
2240
|
+
const effectLength = effectMatch[0].length;
|
|
2241
|
+
const nonEffectLength = remaining - effectLength;
|
|
2242
|
+
const preservedTailLength = Math.min(
|
|
2243
|
+
message.length - effectEnd,
|
|
2244
|
+
nonEffectLength - Math.min(effectMatch.index, Math.ceil(nonEffectLength * 0.6))
|
|
2245
|
+
);
|
|
2246
|
+
const preservedHeadLength = Math.min(
|
|
2247
|
+
effectMatch.index,
|
|
2248
|
+
nonEffectLength - preservedTailLength
|
|
2249
|
+
);
|
|
2250
|
+
return message.slice(0, preservedHeadLength) + marker + message.slice(effectMatch.index, effectEnd) + message.slice(message.length - preservedTailLength);
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1459
2254
|
}
|
|
1460
2255
|
function sanitiseMetadata(metadata) {
|
|
1461
2256
|
if (!metadata) return {};
|
|
@@ -1532,10 +2327,10 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1532
2327
|
|
|
1533
2328
|
// src/lib/opencode/session-db-recovery-report.ts
|
|
1534
2329
|
import { readFileSync as readFileSync2, unlinkSync } from "node:fs";
|
|
1535
|
-
import { join as
|
|
2330
|
+
import { join as join3 } from "node:path";
|
|
1536
2331
|
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1537
2332
|
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1538
|
-
return override ||
|
|
2333
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1539
2334
|
}
|
|
1540
2335
|
function drainSessionDbRecoveryReport({
|
|
1541
2336
|
homeDir,
|
|
@@ -1665,12 +2460,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1665
2460
|
metadata: withoutContractFields(record),
|
|
1666
2461
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1667
2462
|
};
|
|
1668
|
-
case "session_db_boot_refused":
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
2463
|
+
case "session_db_boot_refused": {
|
|
2464
|
+
const message = (() => {
|
|
2465
|
+
switch (record.reason) {
|
|
2466
|
+
case "local_discard_failed":
|
|
2467
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2468
|
+
case "replica_separation_unproven":
|
|
2469
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2470
|
+
case "verification_timeout_before_separation":
|
|
2471
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2472
|
+
case "verification_timeout_during_separation":
|
|
2473
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2474
|
+
default:
|
|
2475
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2476
|
+
}
|
|
2477
|
+
})();
|
|
2478
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2479
|
+
}
|
|
1674
2480
|
case "schema_provenance_mismatch":
|
|
1675
2481
|
return {
|
|
1676
2482
|
level,
|
|
@@ -1797,7 +2603,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1797
2603
|
if (health.healthy || health.authFailed) {
|
|
1798
2604
|
return health;
|
|
1799
2605
|
}
|
|
1800
|
-
await new Promise((
|
|
2606
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1801
2607
|
}
|
|
1802
2608
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1803
2609
|
}
|
|
@@ -1808,7 +2614,7 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1808
2614
|
if (health.healthy || health.authFailed) {
|
|
1809
2615
|
return health;
|
|
1810
2616
|
}
|
|
1811
|
-
await new Promise((
|
|
2617
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1812
2618
|
}
|
|
1813
2619
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1814
2620
|
}
|
|
@@ -1920,7 +2726,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1920
2726
|
return null;
|
|
1921
2727
|
}
|
|
1922
2728
|
}
|
|
1923
|
-
function
|
|
2729
|
+
function roleOf2(m) {
|
|
1924
2730
|
if (!m || typeof m !== "object") return void 0;
|
|
1925
2731
|
if (typeof m.role === "string") return m.role;
|
|
1926
2732
|
const infoRole = m.info?.role;
|
|
@@ -1934,13 +2740,13 @@ function createdOf(m) {
|
|
|
1934
2740
|
if (!m || typeof m !== "object") return void 0;
|
|
1935
2741
|
return m.info?.time?.created ?? m.time?.created;
|
|
1936
2742
|
}
|
|
1937
|
-
function
|
|
2743
|
+
function idOf2(m) {
|
|
1938
2744
|
if (!m || typeof m !== "object") return void 0;
|
|
1939
2745
|
if (typeof m.id === "string") return m.id;
|
|
1940
2746
|
const infoId = m.info?.id;
|
|
1941
2747
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1942
2748
|
}
|
|
1943
|
-
function
|
|
2749
|
+
function parentIdOf2(m) {
|
|
1944
2750
|
if (!m || typeof m !== "object") return void 0;
|
|
1945
2751
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1946
2752
|
const infoParent = m.info?.parentID;
|
|
@@ -1956,8 +2762,12 @@ function errorOf(m) {
|
|
|
1956
2762
|
if (!m || typeof m !== "object") return void 0;
|
|
1957
2763
|
return m.info?.error ?? m.error;
|
|
1958
2764
|
}
|
|
2765
|
+
function isTerminalStopAnswer(m) {
|
|
2766
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2767
|
+
}
|
|
1959
2768
|
function isAssistantInFlight(m) {
|
|
1960
2769
|
if (completedOf(m) == null) return true;
|
|
2770
|
+
if (errorOf(m) != null) return false;
|
|
1961
2771
|
return finishOf(m) === "tool-calls";
|
|
1962
2772
|
}
|
|
1963
2773
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -2005,9 +2815,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
2005
2815
|
function isSessionActivelyGenerating(messages) {
|
|
2006
2816
|
if (!messages || messages.length === 0) return false;
|
|
2007
2817
|
const last = messages[messages.length - 1];
|
|
2008
|
-
if (
|
|
2818
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
2009
2819
|
return completedOf(last) == null;
|
|
2010
2820
|
}
|
|
2821
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
2011
2822
|
function sessionLastActivityMs(session) {
|
|
2012
2823
|
const candidates = [
|
|
2013
2824
|
session.time?.updated,
|
|
@@ -2022,30 +2833,71 @@ function sessionLastActivityMs(session) {
|
|
|
2022
2833
|
}
|
|
2023
2834
|
return null;
|
|
2024
2835
|
}
|
|
2025
|
-
|
|
2026
|
-
|
|
2836
|
+
function sessionListPath(options) {
|
|
2837
|
+
const params = new URLSearchParams();
|
|
2838
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2839
|
+
const query = params.toString();
|
|
2840
|
+
return query ? `/session?${query}` : "/session";
|
|
2841
|
+
}
|
|
2842
|
+
async function listSessionPage(port, client, options) {
|
|
2843
|
+
const path = sessionListPath(options);
|
|
2027
2844
|
try {
|
|
2028
|
-
const res = await requestWithClient(port, client,
|
|
2845
|
+
const res = await requestWithClient(port, client, path);
|
|
2029
2846
|
if (!res.ok) return null;
|
|
2030
2847
|
const body = await res.json();
|
|
2031
2848
|
return Array.isArray(body) ? body : null;
|
|
2032
2849
|
} catch (error2) {
|
|
2033
2850
|
console.error(
|
|
2034
|
-
`[
|
|
2851
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2035
2852
|
);
|
|
2036
2853
|
return null;
|
|
2037
2854
|
}
|
|
2038
2855
|
}
|
|
2856
|
+
async function listSessions(port, client, options) {
|
|
2857
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2858
|
+
return listSessionPage(port, client, options);
|
|
2859
|
+
}
|
|
2860
|
+
async function listAllSessions(port, client) {
|
|
2861
|
+
if (client?.version === "v2") {
|
|
2862
|
+
const sessions2 = await listV2Sessions(client);
|
|
2863
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2864
|
+
}
|
|
2865
|
+
const sessions = await listSessionPage(port, client, {
|
|
2866
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2867
|
+
});
|
|
2868
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2869
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2870
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2871
|
+
}
|
|
2872
|
+
return { sessions };
|
|
2873
|
+
}
|
|
2039
2874
|
async function deleteSession(port, id, client) {
|
|
2040
2875
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2041
2876
|
try {
|
|
2042
|
-
const res = await requestWithClient(
|
|
2043
|
-
|
|
2877
|
+
const res = await requestWithClient(
|
|
2878
|
+
port,
|
|
2879
|
+
client,
|
|
2880
|
+
`/session/${id}`,
|
|
2881
|
+
{ method: "DELETE" },
|
|
2882
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2883
|
+
);
|
|
2884
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2885
|
+
return { ok: true, status: res.status };
|
|
2886
|
+
}
|
|
2887
|
+
try {
|
|
2888
|
+
const body = await res.text();
|
|
2889
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2890
|
+
} catch (error2) {
|
|
2891
|
+
console.error(
|
|
2892
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2893
|
+
);
|
|
2894
|
+
return { ok: false, status: res.status };
|
|
2895
|
+
}
|
|
2044
2896
|
} catch (error2) {
|
|
2045
2897
|
console.error(
|
|
2046
2898
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2047
2899
|
);
|
|
2048
|
-
return false;
|
|
2900
|
+
return { ok: false, status: "threw" };
|
|
2049
2901
|
}
|
|
2050
2902
|
}
|
|
2051
2903
|
async function sessionExists(port, id, client) {
|
|
@@ -2283,11 +3135,31 @@ function messageText(m) {
|
|
|
2283
3135
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2284
3136
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2285
3137
|
}
|
|
2286
|
-
|
|
3138
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
3139
|
+
if (!messages || messages.length === 0) return null;
|
|
3140
|
+
let best = null;
|
|
3141
|
+
for (const message of messages) {
|
|
3142
|
+
if (roleOf2(message) !== "user") continue;
|
|
3143
|
+
const id = idOf2(message);
|
|
3144
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
3145
|
+
if (messageText(message) !== content) continue;
|
|
3146
|
+
const created = createdOf(message) ?? 0;
|
|
3147
|
+
if (best === null || created > best.created) best = { id, created };
|
|
3148
|
+
}
|
|
3149
|
+
return best?.id ?? null;
|
|
3150
|
+
}
|
|
3151
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2287
3152
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2288
3153
|
const knownUserIds = new Set(
|
|
2289
|
-
(before ?? []).filter((m) =>
|
|
3154
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2290
3155
|
);
|
|
3156
|
+
if (callbacks) {
|
|
3157
|
+
try {
|
|
3158
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
3159
|
+
} catch (error2) {
|
|
3160
|
+
callbacks.onKnownUserIdsError(error2);
|
|
3161
|
+
}
|
|
3162
|
+
}
|
|
2291
3163
|
const parts = [{ type: "text", text: content }];
|
|
2292
3164
|
let pendingOutcomes = null;
|
|
2293
3165
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2326,25 +3198,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2326
3198
|
const READ_BACK_DELAY_MS = 150;
|
|
2327
3199
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2328
3200
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
const id = idOf(m);
|
|
2334
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2335
|
-
if (messageText(m) !== content) continue;
|
|
2336
|
-
const created = createdOf(m) ?? 0;
|
|
2337
|
-
if (best === null || created > best.created) {
|
|
2338
|
-
best = { id, created };
|
|
2339
|
-
}
|
|
2340
|
-
}
|
|
2341
|
-
if (best) {
|
|
2342
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2343
|
-
return best.id;
|
|
2344
|
-
}
|
|
3201
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
3202
|
+
if (dispatchedUserMessageId !== null) {
|
|
3203
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
3204
|
+
return dispatchedUserMessageId;
|
|
2345
3205
|
}
|
|
2346
3206
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2347
|
-
await new Promise((
|
|
3207
|
+
await new Promise((resolve5) => setTimeout(resolve5, READ_BACK_DELAY_MS));
|
|
2348
3208
|
}
|
|
2349
3209
|
}
|
|
2350
3210
|
return null;
|
|
@@ -2352,57 +3212,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2352
3212
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2353
3213
|
if (!messages || messages.length === 0) return null;
|
|
2354
3214
|
const byParent = messages.find(
|
|
2355
|
-
(m) =>
|
|
3215
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2356
3216
|
);
|
|
2357
3217
|
if (byParent) return byParent;
|
|
2358
|
-
const userIndex = messages.findIndex((m) =>
|
|
3218
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2359
3219
|
if (userIndex === -1) return null;
|
|
2360
3220
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2361
|
-
if (
|
|
3221
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2362
3222
|
}
|
|
2363
3223
|
return null;
|
|
2364
3224
|
}
|
|
2365
3225
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2366
3226
|
if (!messages || messages.length === 0) return null;
|
|
2367
3227
|
let lastCorrelated = null;
|
|
2368
|
-
let lastNonErrored = null;
|
|
2369
3228
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2370
3229
|
const m = messages[i];
|
|
2371
|
-
if (
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
3230
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
3231
|
+
lastCorrelated = m;
|
|
3232
|
+
break;
|
|
3233
|
+
}
|
|
3234
|
+
if (lastCorrelated) {
|
|
3235
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
3236
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3237
|
+
const m = messages[i];
|
|
3238
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
3239
|
+
return m;
|
|
3240
|
+
}
|
|
2376
3241
|
}
|
|
3242
|
+
return lastCorrelated;
|
|
2377
3243
|
}
|
|
2378
|
-
|
|
2379
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
3244
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2380
3245
|
if (userIndex === -1) return null;
|
|
2381
3246
|
let last = null;
|
|
2382
|
-
let
|
|
3247
|
+
let lastAssistantIndex = -1;
|
|
2383
3248
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2384
|
-
const role =
|
|
3249
|
+
const role = roleOf2(messages[i]);
|
|
2385
3250
|
if (role === "user") break;
|
|
2386
3251
|
if (role === "assistant") {
|
|
2387
3252
|
last = messages[i];
|
|
2388
|
-
|
|
3253
|
+
lastAssistantIndex = i;
|
|
2389
3254
|
}
|
|
2390
3255
|
}
|
|
2391
|
-
|
|
3256
|
+
if (last === null || errorOf(last) == null) return last;
|
|
3257
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
3258
|
+
const m = messages[i];
|
|
3259
|
+
if (roleOf2(m) === "user") break;
|
|
3260
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
3261
|
+
}
|
|
3262
|
+
return last;
|
|
2392
3263
|
}
|
|
2393
3264
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2394
3265
|
if (!messages || messages.length === 0) return [];
|
|
2395
3266
|
const byParent = messages.filter(
|
|
2396
|
-
(message) =>
|
|
3267
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2397
3268
|
);
|
|
2398
3269
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2399
3270
|
if (assistants.length === 0) {
|
|
2400
|
-
const userIndex = messages.findIndex((message) =>
|
|
3271
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2401
3272
|
if (userIndex === -1) return [];
|
|
2402
3273
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2403
3274
|
const message = messages[i];
|
|
2404
|
-
if (
|
|
2405
|
-
if (
|
|
3275
|
+
if (roleOf2(message) === "user") break;
|
|
3276
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2406
3277
|
}
|
|
2407
3278
|
}
|
|
2408
3279
|
const refs = [];
|
|
@@ -2442,7 +3313,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2442
3313
|
if (!messages || messages.length === 0) return [];
|
|
2443
3314
|
const calls = [];
|
|
2444
3315
|
for (const message of messages) {
|
|
2445
|
-
if (
|
|
3316
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2446
3317
|
for (const part of message.parts ?? []) {
|
|
2447
3318
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2448
3319
|
continue;
|
|
@@ -2472,7 +3343,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2472
3343
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2473
3344
|
const unattributed = [];
|
|
2474
3345
|
for (const message of messages ?? []) {
|
|
2475
|
-
if (
|
|
3346
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2476
3347
|
const created = finiteNumber(createdOf(message));
|
|
2477
3348
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2478
3349
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2494,8 +3365,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2494
3365
|
}
|
|
2495
3366
|
function sumAssistantUsage(messages) {
|
|
2496
3367
|
if (!messages || messages.length === 0) return null;
|
|
2497
|
-
const
|
|
2498
|
-
const
|
|
3368
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3369
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3370
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2499
3371
|
let sawAnyUsage = false;
|
|
2500
3372
|
let inputSum = 0;
|
|
2501
3373
|
let outputSum = 0;
|
|
@@ -2504,9 +3376,11 @@ function sumAssistantUsage(messages) {
|
|
|
2504
3376
|
let cacheWriteSum = 0;
|
|
2505
3377
|
let costSum = 0;
|
|
2506
3378
|
let sawCost = false;
|
|
3379
|
+
let toolCallCount = 0;
|
|
2507
3380
|
let modelId = null;
|
|
2508
3381
|
let providerId = null;
|
|
2509
3382
|
for (const m of selected) {
|
|
3383
|
+
toolCallCount += m.parts?.filter((part) => part.type === "tool").length ?? 0;
|
|
2510
3384
|
const info = m.info;
|
|
2511
3385
|
if (!info) continue;
|
|
2512
3386
|
const tokens = info.tokens;
|
|
@@ -2543,19 +3417,18 @@ function sumAssistantUsage(messages) {
|
|
|
2543
3417
|
usage_tokens_cache_write: cacheWriteSum,
|
|
2544
3418
|
// NULL means OpenCode never reported a cost; it is distinct from a genuine
|
|
2545
3419
|
// zero-cost message, which sets `sawCost` with `costSum === 0`.
|
|
2546
|
-
usage_cost_usd: sawCost ? costSum : null
|
|
3420
|
+
usage_cost_usd: sawCost ? costSum : null,
|
|
3421
|
+
turn_tool_calls: toolCallCount
|
|
2547
3422
|
};
|
|
2548
3423
|
}
|
|
2549
3424
|
function messageUsage(messages, userMessageId) {
|
|
2550
3425
|
if (!messages || messages.length === 0) return null;
|
|
2551
3426
|
const byParentAll = messages.filter(
|
|
2552
|
-
(m) =>
|
|
3427
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2553
3428
|
);
|
|
2554
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2555
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2556
3429
|
let correlated;
|
|
2557
|
-
if (
|
|
2558
|
-
correlated =
|
|
3430
|
+
if (byParentAll.length > 0) {
|
|
3431
|
+
correlated = byParentAll;
|
|
2559
3432
|
} else {
|
|
2560
3433
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2561
3434
|
correlated = reply ? [reply] : [];
|
|
@@ -2565,7 +3438,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2565
3438
|
}
|
|
2566
3439
|
function messageRunState(messages, userMessageId) {
|
|
2567
3440
|
if (!messages || messages.length === 0) return "unknown";
|
|
2568
|
-
const hasUser = messages.some((m) =>
|
|
3441
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2569
3442
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2570
3443
|
if (!hasUser) {
|
|
2571
3444
|
if (!reply) return "unknown";
|
|
@@ -2601,14 +3474,38 @@ function messageError(messages, userMessageId) {
|
|
|
2601
3474
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2602
3475
|
const error2 = errorOf(reply);
|
|
2603
3476
|
if (error2 == null) return null;
|
|
2604
|
-
|
|
2605
|
-
if (typeof error2 === "
|
|
3477
|
+
let rendered;
|
|
3478
|
+
if (typeof error2 === "string") {
|
|
3479
|
+
rendered = error2;
|
|
3480
|
+
} else if (typeof error2 === "object") {
|
|
2606
3481
|
const e = error2;
|
|
2607
3482
|
const dataMessage = e.data?.message;
|
|
2608
|
-
if (typeof dataMessage === "string")
|
|
2609
|
-
|
|
3483
|
+
if (typeof dataMessage === "string") {
|
|
3484
|
+
rendered = dataMessage;
|
|
3485
|
+
} else if (typeof e.message === "string") {
|
|
3486
|
+
rendered = e.message;
|
|
3487
|
+
} else {
|
|
3488
|
+
rendered = "The agent run failed.";
|
|
3489
|
+
}
|
|
3490
|
+
} else {
|
|
3491
|
+
rendered = "The agent run failed.";
|
|
2610
3492
|
}
|
|
2611
|
-
|
|
3493
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3494
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3495
|
+
) : void 0;
|
|
3496
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3497
|
+
}
|
|
3498
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3499
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3500
|
+
if (!reply) return null;
|
|
3501
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3502
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3503
|
+
return {
|
|
3504
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3505
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3506
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3507
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3508
|
+
};
|
|
2612
3509
|
}
|
|
2613
3510
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2614
3511
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2625,19 +3522,32 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2625
3522
|
}
|
|
2626
3523
|
return false;
|
|
2627
3524
|
}
|
|
3525
|
+
var CLAUDE_AUTH_EXPIRY_MESSAGES = Object.freeze([
|
|
3526
|
+
"Claude Code credentials are unavailable or expired. Run `claude` to refresh them."
|
|
3527
|
+
]);
|
|
3528
|
+
function extractReplyErrorShape(reply) {
|
|
3529
|
+
const error2 = errorOf(reply);
|
|
3530
|
+
if (error2 == null || typeof error2 !== "object") {
|
|
3531
|
+
return { name: null, statusCode: null };
|
|
3532
|
+
}
|
|
3533
|
+
const e = error2;
|
|
3534
|
+
const name = typeof e.name === "string" ? e.name.slice(0, 100) : null;
|
|
3535
|
+
const data = e.data;
|
|
3536
|
+
const statusCode = typeof data?.statusCode === "number" && Number.isFinite(data.statusCode) ? data.statusCode : null;
|
|
3537
|
+
return { name, statusCode };
|
|
3538
|
+
}
|
|
2628
3539
|
function classifyReplyAuthError(reply) {
|
|
2629
3540
|
const error2 = errorOf(reply);
|
|
2630
3541
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2631
3542
|
const e = error2;
|
|
2632
3543
|
const replyProviderId = reply?.info?.providerID ?? null;
|
|
2633
3544
|
const replyModelId = reply?.info?.modelID ?? null;
|
|
3545
|
+
const data = e.data;
|
|
2634
3546
|
if (e.name === "ProviderAuthError") {
|
|
2635
|
-
const data = e.data;
|
|
2636
3547
|
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
2637
3548
|
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
2638
3549
|
}
|
|
2639
3550
|
if (e.name === "APIError") {
|
|
2640
|
-
const data = e.data;
|
|
2641
3551
|
const statusCode = data?.statusCode;
|
|
2642
3552
|
if (statusCode === 401 || statusCode === 403) {
|
|
2643
3553
|
return {
|
|
@@ -2648,16 +3558,28 @@ function classifyReplyAuthError(reply) {
|
|
|
2648
3558
|
};
|
|
2649
3559
|
}
|
|
2650
3560
|
}
|
|
3561
|
+
const errorMessage3 = typeof data?.message === "string" ? data.message : typeof e.message === "string" ? e.message : null;
|
|
3562
|
+
if (errorMessage3 !== null && CLAUDE_AUTH_EXPIRY_MESSAGES.some((knownMessage) => errorMessage3.includes(knownMessage))) {
|
|
3563
|
+
return {
|
|
3564
|
+
kind: "model_auth",
|
|
3565
|
+
providerId: typeof data?.providerID === "string" && data.providerID || replyProviderId,
|
|
3566
|
+
modelId: replyModelId,
|
|
3567
|
+
reason: "rejected"
|
|
3568
|
+
};
|
|
3569
|
+
}
|
|
2651
3570
|
return null;
|
|
2652
3571
|
}
|
|
2653
3572
|
function messageFailure(messages, userMessageId) {
|
|
2654
3573
|
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
2655
3574
|
}
|
|
3575
|
+
function isProviderModelNotFoundError(reason) {
|
|
3576
|
+
return reason.includes("Model not found:");
|
|
3577
|
+
}
|
|
2656
3578
|
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
2657
3579
|
if (!messages || messages.length === 0) return null;
|
|
2658
3580
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2659
3581
|
const message = messages[i];
|
|
2660
|
-
if (
|
|
3582
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2661
3583
|
const created = createdOf(message);
|
|
2662
3584
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2663
3585
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2689,25 +3611,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2689
3611
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2690
3612
|
if (!messages || messages.length === 0) return false;
|
|
2691
3613
|
return messages.some(
|
|
2692
|
-
(m) =>
|
|
3614
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2693
3615
|
);
|
|
2694
3616
|
}
|
|
2695
3617
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2696
3618
|
if (!messages || messages.length === 0) return false;
|
|
2697
|
-
const userIndex = messages.findIndex((message) =>
|
|
3619
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2698
3620
|
if (userIndex === -1) return false;
|
|
2699
3621
|
let hasLaterUser = false;
|
|
2700
3622
|
let hasStartedLaterUser = false;
|
|
2701
3623
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2702
3624
|
const message = messages[i];
|
|
2703
|
-
if (
|
|
3625
|
+
if (roleOf2(message) !== "user") continue;
|
|
2704
3626
|
hasLaterUser = true;
|
|
2705
|
-
const laterUserMessageId =
|
|
3627
|
+
const laterUserMessageId = idOf2(message);
|
|
2706
3628
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2707
3629
|
return false;
|
|
2708
3630
|
}
|
|
2709
3631
|
if (messages.some(
|
|
2710
|
-
(candidate) =>
|
|
3632
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2711
3633
|
)) {
|
|
2712
3634
|
hasStartedLaterUser = true;
|
|
2713
3635
|
}
|
|
@@ -2861,23 +3783,49 @@ async function readSessionErrorStream(port, options, client) {
|
|
|
2861
3783
|
if (reader) void reader.cancel().catch(() => void 0);
|
|
2862
3784
|
}
|
|
2863
3785
|
}
|
|
2864
|
-
async function
|
|
2865
|
-
if (client?.version === "v2") return;
|
|
3786
|
+
async function disposeAndReloadProviderConfig(port, client) {
|
|
3787
|
+
if (client?.version === "v2") return { outcome: "skipped_v2" };
|
|
2866
3788
|
try {
|
|
2867
|
-
const res = await requestWithClient(
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
3789
|
+
const res = await requestWithClient(
|
|
3790
|
+
port,
|
|
3791
|
+
client,
|
|
3792
|
+
"/config",
|
|
3793
|
+
{
|
|
3794
|
+
method: "PATCH",
|
|
3795
|
+
headers: { "Content-Type": "application/json" },
|
|
3796
|
+
body: JSON.stringify({})
|
|
3797
|
+
},
|
|
3798
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
3799
|
+
);
|
|
2872
3800
|
if (!res.ok) {
|
|
2873
3801
|
console.error(
|
|
2874
|
-
`[
|
|
3802
|
+
`[disposeAndReloadProviderConfig] PATCH /config returned HTTP ${res.status} (port ${port})`
|
|
2875
3803
|
);
|
|
3804
|
+
return { outcome: "failed", status: res.status, detail: null };
|
|
2876
3805
|
}
|
|
3806
|
+
return { outcome: "reloaded" };
|
|
2877
3807
|
} catch (err) {
|
|
2878
3808
|
console.error(
|
|
2879
|
-
`[
|
|
3809
|
+
`[disposeAndReloadProviderConfig] PATCH /config failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
2880
3810
|
);
|
|
3811
|
+
return {
|
|
3812
|
+
outcome: "failed",
|
|
3813
|
+
status: null,
|
|
3814
|
+
detail: err instanceof Error ? err.message : String(err)
|
|
3815
|
+
};
|
|
3816
|
+
}
|
|
3817
|
+
}
|
|
3818
|
+
async function probeOpenCodeRequestReadiness(port, client) {
|
|
3819
|
+
try {
|
|
3820
|
+
const res = await requestWithClient(port, client, "/config", void 0, {
|
|
3821
|
+
allowStatuses: ALL_HTTP_STATUSES
|
|
3822
|
+
});
|
|
3823
|
+
const body = await res.text();
|
|
3824
|
+
return { ok: res.ok, status: res.status, bodyEmpty: body.trim().length === 0, detail: null };
|
|
3825
|
+
} catch (err) {
|
|
3826
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
3827
|
+
console.error(`[probeOpenCodeRequestReadiness] GET /config failed (port ${port}): ${detail}`);
|
|
3828
|
+
return { ok: false, status: null, bodyEmpty: null, detail };
|
|
2881
3829
|
}
|
|
2882
3830
|
}
|
|
2883
3831
|
|
|
@@ -3178,13 +4126,28 @@ async function listV2Sessions(client) {
|
|
|
3178
4126
|
}
|
|
3179
4127
|
async function deleteV2Session(client, sessionId) {
|
|
3180
4128
|
try {
|
|
3181
|
-
await client.request(
|
|
3182
|
-
|
|
4129
|
+
const response = await client.request(
|
|
4130
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
4131
|
+
{ method: "DELETE" },
|
|
4132
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
4133
|
+
);
|
|
4134
|
+
if (response.status >= 200 && response.status < 300) {
|
|
4135
|
+
return { ok: true, status: response.status };
|
|
4136
|
+
}
|
|
4137
|
+
try {
|
|
4138
|
+
const body = await response.text();
|
|
4139
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
4140
|
+
} catch (error2) {
|
|
4141
|
+
console.error(
|
|
4142
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4143
|
+
);
|
|
4144
|
+
return { ok: false, status: response.status };
|
|
4145
|
+
}
|
|
3183
4146
|
} catch (error2) {
|
|
3184
4147
|
console.error(
|
|
3185
4148
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3186
4149
|
);
|
|
3187
|
-
return false;
|
|
4150
|
+
return { ok: false, status: "threw" };
|
|
3188
4151
|
}
|
|
3189
4152
|
}
|
|
3190
4153
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3355,12 +4318,12 @@ async function readV2SessionErrorStream(client, options) {
|
|
|
3355
4318
|
|
|
3356
4319
|
// src/lib/opencode/credential-import-v2.ts
|
|
3357
4320
|
import { readFileSync as readFileSync4 } from "node:fs";
|
|
3358
|
-
import { join as
|
|
4321
|
+
import { join as join5 } from "node:path";
|
|
3359
4322
|
|
|
3360
4323
|
// src/lib/openai-usage.ts
|
|
3361
4324
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3362
4325
|
import { homedir as homedir3 } from "node:os";
|
|
3363
|
-
import { join as
|
|
4326
|
+
import { join as join4 } from "node:path";
|
|
3364
4327
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3365
4328
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3366
4329
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3374,7 +4337,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3374
4337
|
}
|
|
3375
4338
|
function readOpenCodeChatGptCredentials() {
|
|
3376
4339
|
try {
|
|
3377
|
-
const raw = readFileSync3(
|
|
4340
|
+
const raw = readFileSync3(join4(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3378
4341
|
let parsed;
|
|
3379
4342
|
try {
|
|
3380
4343
|
parsed = JSON.parse(raw);
|
|
@@ -3600,13 +4563,13 @@ var INTEGRATION_LIST_RETRY_DELAY_MS = 1e3;
|
|
|
3600
4563
|
function isRecord3(value) {
|
|
3601
4564
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3602
4565
|
}
|
|
3603
|
-
var sleep2 = (milliseconds) => new Promise((
|
|
4566
|
+
var sleep2 = (milliseconds) => new Promise((resolve5) => setTimeout(resolve5, milliseconds));
|
|
3604
4567
|
function emptyResult(unsupported = [], outcome = "completed") {
|
|
3605
4568
|
return { outcome, imported: [], alreadyConnected: [], unsupported, failed: [] };
|
|
3606
4569
|
}
|
|
3607
4570
|
function readOpenCodeCredentials(homeDir, log3) {
|
|
3608
4571
|
try {
|
|
3609
|
-
const raw = readFileSync4(
|
|
4572
|
+
const raw = readFileSync4(join5(homeDir, ...OPENCODE_AUTH_SEGMENTS), "utf8");
|
|
3610
4573
|
let parsed;
|
|
3611
4574
|
try {
|
|
3612
4575
|
parsed = JSON.parse(raw);
|
|
@@ -3806,9 +4769,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3806
4769
|
|
|
3807
4770
|
// src/lib/opencode/session-db-boot.ts
|
|
3808
4771
|
import { spawn as spawn2 } from "node:child_process";
|
|
3809
|
-
import { mkdirSync, statSync as
|
|
4772
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3810
4773
|
import { homedir as homedir4 } from "node:os";
|
|
3811
|
-
import { dirname as
|
|
4774
|
+
import { dirname as dirname4, resolve as resolvePath } from "node:path";
|
|
3812
4775
|
|
|
3813
4776
|
// src/lib/runner-synchroniser.ts
|
|
3814
4777
|
import { spawn } from "node:child_process";
|
|
@@ -3818,7 +4781,7 @@ function appendError(stderr, error2) {
|
|
|
3818
4781
|
${message}`;
|
|
3819
4782
|
}
|
|
3820
4783
|
function runSynchroniser(args, opts) {
|
|
3821
|
-
return new Promise((
|
|
4784
|
+
return new Promise((resolve5) => {
|
|
3822
4785
|
let child;
|
|
3823
4786
|
let stdout = "";
|
|
3824
4787
|
let stderr = "";
|
|
@@ -3832,7 +4795,7 @@ function runSynchroniser(args, opts) {
|
|
|
3832
4795
|
if (timer.handle) clearTimeout(timer.handle);
|
|
3833
4796
|
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
3834
4797
|
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
3835
|
-
|
|
4798
|
+
resolve5(result);
|
|
3836
4799
|
};
|
|
3837
4800
|
try {
|
|
3838
4801
|
child = spawn("runner-synchroniser", args, {
|
|
@@ -3888,9 +4851,12 @@ function runSynchroniser(args, opts) {
|
|
|
3888
4851
|
|
|
3889
4852
|
// src/lib/opencode/session-db-boot.ts
|
|
3890
4853
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3891
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4854
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3892
4855
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3893
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4856
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4857
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4858
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4859
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3894
4860
|
function commandError(result) {
|
|
3895
4861
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3896
4862
|
}
|
|
@@ -3933,7 +4899,7 @@ function clearMarker(options) {
|
|
|
3933
4899
|
function markNoReplicate(options, message) {
|
|
3934
4900
|
if (options.noReplicateMarker) {
|
|
3935
4901
|
try {
|
|
3936
|
-
mkdirSync(
|
|
4902
|
+
mkdirSync(dirname4(options.noReplicateMarker), { recursive: true });
|
|
3937
4903
|
writeFileSync(options.noReplicateMarker, "");
|
|
3938
4904
|
} catch (error2) {
|
|
3939
4905
|
options.log(
|
|
@@ -3992,7 +4958,7 @@ function parseSynchroniserEnv(stdout) {
|
|
|
3992
4958
|
return values;
|
|
3993
4959
|
}
|
|
3994
4960
|
function runCommand(command, args, options) {
|
|
3995
|
-
return new Promise((
|
|
4961
|
+
return new Promise((resolve5) => {
|
|
3996
4962
|
let child;
|
|
3997
4963
|
let stdout = "";
|
|
3998
4964
|
let stderr = "";
|
|
@@ -4001,7 +4967,7 @@ function runCommand(command, args, options) {
|
|
|
4001
4967
|
if (settled) return;
|
|
4002
4968
|
settled = true;
|
|
4003
4969
|
if (timer) clearTimeout(timer);
|
|
4004
|
-
|
|
4970
|
+
resolve5(result);
|
|
4005
4971
|
};
|
|
4006
4972
|
try {
|
|
4007
4973
|
child = spawn2(command, args, {
|
|
@@ -4009,7 +4975,7 @@ function runCommand(command, args, options) {
|
|
|
4009
4975
|
stdio: ["ignore", "pipe", "pipe"]
|
|
4010
4976
|
});
|
|
4011
4977
|
} catch (error2) {
|
|
4012
|
-
|
|
4978
|
+
resolve5({
|
|
4013
4979
|
code: null,
|
|
4014
4980
|
stdout,
|
|
4015
4981
|
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
@@ -4058,7 +5024,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4058
5024
|
return null;
|
|
4059
5025
|
}
|
|
4060
5026
|
try {
|
|
4061
|
-
if (
|
|
5027
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4062
5028
|
} catch (error2) {
|
|
4063
5029
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4064
5030
|
options.log(
|
|
@@ -4088,7 +5054,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4088
5054
|
return null;
|
|
4089
5055
|
}
|
|
4090
5056
|
try {
|
|
4091
|
-
mkdirSync(
|
|
5057
|
+
mkdirSync(dirname4(configPath), { recursive: true });
|
|
4092
5058
|
writeFileSync(configPath, rendered.stdout);
|
|
4093
5059
|
} catch (error2) {
|
|
4094
5060
|
options.log(
|
|
@@ -4201,20 +5167,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4201
5167
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4202
5168
|
env: {
|
|
4203
5169
|
...env,
|
|
4204
|
-
// The synchroniser reads
|
|
4205
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
5170
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4206
5171
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4207
5172
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
5173
|
+
),
|
|
5174
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
5175
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
5176
|
+
),
|
|
5177
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
5178
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
5179
|
+
),
|
|
5180
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
5181
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4208
5182
|
)
|
|
4209
5183
|
}
|
|
4210
5184
|
});
|
|
4211
5185
|
logSynchroniserDiagnostics(result, options);
|
|
4212
5186
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
5187
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
5188
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
5189
|
+
options.log(
|
|
5190
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
5191
|
+
"error"
|
|
5192
|
+
);
|
|
5193
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
5194
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
5195
|
+
}
|
|
4213
5196
|
options.log(
|
|
4214
5197
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4215
5198
|
"warn"
|
|
4216
5199
|
);
|
|
4217
|
-
return false;
|
|
5200
|
+
return { verifyFatal: false };
|
|
4218
5201
|
}
|
|
4219
5202
|
if (result.code === 34) {
|
|
4220
5203
|
reportRecord(
|
|
@@ -4224,14 +5207,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4224
5207
|
null,
|
|
4225
5208
|
options
|
|
4226
5209
|
);
|
|
4227
|
-
return true;
|
|
5210
|
+
return { verifyFatal: true };
|
|
4228
5211
|
}
|
|
4229
5212
|
if (result.code === 33) {
|
|
4230
5213
|
options.log(
|
|
4231
5214
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4232
5215
|
"warn"
|
|
4233
5216
|
);
|
|
4234
|
-
return false;
|
|
5217
|
+
return { verifyFatal: false };
|
|
4235
5218
|
}
|
|
4236
5219
|
if (result.code !== 0) {
|
|
4237
5220
|
options.log(
|
|
@@ -4239,17 +5222,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4239
5222
|
"warn"
|
|
4240
5223
|
);
|
|
4241
5224
|
}
|
|
4242
|
-
return false;
|
|
5225
|
+
return { verifyFatal: false };
|
|
4243
5226
|
}
|
|
4244
5227
|
options.log(
|
|
4245
5228
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4246
5229
|
"debug"
|
|
4247
5230
|
);
|
|
4248
|
-
return false;
|
|
5231
|
+
return { verifyFatal: false };
|
|
4249
5232
|
}
|
|
4250
5233
|
function fileExists(path) {
|
|
4251
5234
|
try {
|
|
4252
|
-
|
|
5235
|
+
statSync3(path);
|
|
4253
5236
|
return true;
|
|
4254
5237
|
} catch (error2) {
|
|
4255
5238
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4332,13 +5315,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4332
5315
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4333
5316
|
return { verifyFatal: false };
|
|
4334
5317
|
}
|
|
4335
|
-
return
|
|
5318
|
+
return verifySessionDb(options, configPath, env);
|
|
4336
5319
|
}
|
|
4337
5320
|
|
|
4338
5321
|
// src/lib/opencode/session-db-provenance.ts
|
|
4339
5322
|
import { createRequire } from "node:module";
|
|
4340
5323
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4341
|
-
import { dirname as
|
|
5324
|
+
import { dirname as dirname5, join as join6 } from "node:path";
|
|
4342
5325
|
var require2 = createRequire(import.meta.url);
|
|
4343
5326
|
function readSessionDbMigrationIds(dbPath) {
|
|
4344
5327
|
let db;
|
|
@@ -4375,7 +5358,7 @@ function readSessionDbMigrationIds(dbPath) {
|
|
|
4375
5358
|
}
|
|
4376
5359
|
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
4377
5360
|
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
4378
|
-
return override ||
|
|
5361
|
+
return override || join6(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
4379
5362
|
}
|
|
4380
5363
|
function loadSessionDbProvenanceState(path) {
|
|
4381
5364
|
let value;
|
|
@@ -4404,7 +5387,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4404
5387
|
}
|
|
4405
5388
|
function saveSessionDbProvenanceState(path, state) {
|
|
4406
5389
|
try {
|
|
4407
|
-
mkdirSync2(
|
|
5390
|
+
mkdirSync2(dirname5(path), { recursive: true });
|
|
4408
5391
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4409
5392
|
`, "utf8");
|
|
4410
5393
|
} catch (error2) {
|
|
@@ -4495,7 +5478,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4495
5478
|
if (child.exitCode !== null || child.signalCode !== null) {
|
|
4496
5479
|
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
4497
5480
|
}
|
|
4498
|
-
return new Promise((
|
|
5481
|
+
return new Promise((resolve5, reject) => {
|
|
4499
5482
|
let forced = false;
|
|
4500
5483
|
let settled = false;
|
|
4501
5484
|
const timer = setTimeout(() => {
|
|
@@ -4515,7 +5498,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4515
5498
|
settled = true;
|
|
4516
5499
|
clearTimeout(timer);
|
|
4517
5500
|
child.removeListener("exit", onExit);
|
|
4518
|
-
|
|
5501
|
+
resolve5(result);
|
|
4519
5502
|
};
|
|
4520
5503
|
const fail = (error2) => {
|
|
4521
5504
|
if (settled) return;
|
|
@@ -4985,10 +5968,13 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4985
5968
|
toDelete.push(s.id);
|
|
4986
5969
|
}
|
|
4987
5970
|
}
|
|
4988
|
-
|
|
5971
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5972
|
+
return toDelete.sort(
|
|
5973
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5974
|
+
);
|
|
4989
5975
|
}
|
|
4990
5976
|
var DEFAULT_INTERVAL = "1h";
|
|
4991
|
-
function
|
|
5977
|
+
function resolve2(flag, envValue, fallback) {
|
|
4992
5978
|
return flag ?? envValue ?? fallback;
|
|
4993
5979
|
}
|
|
4994
5980
|
function parseMaxCount(input) {
|
|
@@ -5004,9 +5990,9 @@ function parseMaxCount(input) {
|
|
|
5004
5990
|
}
|
|
5005
5991
|
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
5006
5992
|
const warnings = [];
|
|
5007
|
-
const maxAgeRaw =
|
|
5008
|
-
const maxCountRaw =
|
|
5009
|
-
const intervalRaw =
|
|
5993
|
+
const maxAgeRaw = resolve2(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
5994
|
+
const maxCountRaw = resolve2(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
5995
|
+
const intervalRaw = resolve2(
|
|
5010
5996
|
flags.interval,
|
|
5011
5997
|
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
5012
5998
|
DEFAULT_INTERVAL
|
|
@@ -5045,13 +6031,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5045
6031
|
}
|
|
5046
6032
|
|
|
5047
6033
|
// src/lib/opencode/session-db-size.ts
|
|
5048
|
-
import { statSync as
|
|
5049
|
-
import { join as
|
|
6034
|
+
import { statSync as statSync4 } from "node:fs";
|
|
6035
|
+
import { join as join7 } from "node:path";
|
|
5050
6036
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5051
6037
|
function statSessionDbBytes(homeDir) {
|
|
5052
|
-
const dbPath =
|
|
6038
|
+
const dbPath = join7(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5053
6039
|
try {
|
|
5054
|
-
return
|
|
6040
|
+
return statSync4(dbPath).size;
|
|
5055
6041
|
} catch (err) {
|
|
5056
6042
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5057
6043
|
if (!isMissingFile) {
|
|
@@ -5077,14 +6063,14 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5077
6063
|
}
|
|
5078
6064
|
|
|
5079
6065
|
// src/lib/opencode/log-tail.ts
|
|
5080
|
-
import { statSync as
|
|
6066
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5081
6067
|
import { homedir as homedir5 } from "node:os";
|
|
5082
|
-
import { join as
|
|
6068
|
+
import { join as join8 } from "node:path";
|
|
5083
6069
|
import { open as open2, stat } from "node:fs/promises";
|
|
5084
6070
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
5085
6071
|
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
5086
|
-
const dataDir = env.XDG_DATA_HOME ||
|
|
5087
|
-
return
|
|
6072
|
+
const dataDir = env.XDG_DATA_HOME || join8(homeDir, ".local", "share");
|
|
6073
|
+
return join8(dataDir, "opencode", "log", "opencode.log");
|
|
5088
6074
|
}
|
|
5089
6075
|
function isEnoent(error2) {
|
|
5090
6076
|
return error2?.code === "ENOENT";
|
|
@@ -5100,7 +6086,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5100
6086
|
let inode = null;
|
|
5101
6087
|
let baselineReady = true;
|
|
5102
6088
|
try {
|
|
5103
|
-
const initial =
|
|
6089
|
+
const initial = statSync5(logPath);
|
|
5104
6090
|
offset = initial.size;
|
|
5105
6091
|
inode = initial.ino;
|
|
5106
6092
|
} catch (error2) {
|
|
@@ -5159,129 +6145,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5159
6145
|
};
|
|
5160
6146
|
}
|
|
5161
6147
|
|
|
5162
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5163
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5164
|
-
import { dirname as dirname4 } from "node:path";
|
|
5165
|
-
function errorMessage(error2) {
|
|
5166
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5167
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5168
|
-
}
|
|
5169
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5170
|
-
try {
|
|
5171
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5172
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5173
|
-
if (availableBytes < requiredBytes) {
|
|
5174
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5175
|
-
}
|
|
5176
|
-
return null;
|
|
5177
|
-
} catch (err) {
|
|
5178
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5179
|
-
}
|
|
5180
|
-
}
|
|
5181
|
-
function readLogicalBytes(db) {
|
|
5182
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5183
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5184
|
-
return pageCount * pageSize;
|
|
5185
|
-
}
|
|
5186
|
-
function readCheckpointResult(db) {
|
|
5187
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5188
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5189
|
-
}
|
|
5190
|
-
async function probeReclaimAvailability(input) {
|
|
5191
|
-
const { dbPath, requiredBytes } = input;
|
|
5192
|
-
let sqlite;
|
|
5193
|
-
try {
|
|
5194
|
-
sqlite = await import("node:sqlite");
|
|
5195
|
-
} catch (err) {
|
|
5196
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5197
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5198
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5199
|
-
}
|
|
5200
|
-
let autoVacuum = null;
|
|
5201
|
-
try {
|
|
5202
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5203
|
-
try {
|
|
5204
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5205
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5206
|
-
} finally {
|
|
5207
|
-
db.close();
|
|
5208
|
-
}
|
|
5209
|
-
} catch (err) {
|
|
5210
|
-
console.warn(
|
|
5211
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5212
|
-
);
|
|
5213
|
-
}
|
|
5214
|
-
if (autoVacuum !== 0) return null;
|
|
5215
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5216
|
-
}
|
|
5217
|
-
async function reclaimSessionDbSpace(input) {
|
|
5218
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5219
|
-
let sqlite;
|
|
5220
|
-
try {
|
|
5221
|
-
sqlite = await import("node:sqlite");
|
|
5222
|
-
} catch (err) {
|
|
5223
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5224
|
-
console.warn(
|
|
5225
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5226
|
-
);
|
|
5227
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5228
|
-
}
|
|
5229
|
-
const { DatabaseSync } = sqlite;
|
|
5230
|
-
let db;
|
|
5231
|
-
try {
|
|
5232
|
-
db = new DatabaseSync(dbPath);
|
|
5233
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5234
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5235
|
-
if (autoVacuum === 0) {
|
|
5236
|
-
if (!allowFullVacuum) {
|
|
5237
|
-
console.warn(
|
|
5238
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5239
|
-
);
|
|
5240
|
-
return {
|
|
5241
|
-
ok: false,
|
|
5242
|
-
skipped: "full-vacuum-blocked",
|
|
5243
|
-
detail: "a session turn is live"
|
|
5244
|
-
};
|
|
5245
|
-
}
|
|
5246
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5247
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5248
|
-
if (skipReason !== null) {
|
|
5249
|
-
console.warn(
|
|
5250
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5251
|
-
);
|
|
5252
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5253
|
-
}
|
|
5254
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5255
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5256
|
-
db.exec("VACUUM");
|
|
5257
|
-
const afterBytes = readLogicalBytes(db);
|
|
5258
|
-
const checkpoint = readCheckpointResult(db);
|
|
5259
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5260
|
-
}
|
|
5261
|
-
if (autoVacuum === 2) {
|
|
5262
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5263
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5264
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5265
|
-
const afterBytes = readLogicalBytes(db);
|
|
5266
|
-
const checkpoint = readCheckpointResult(db);
|
|
5267
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5268
|
-
}
|
|
5269
|
-
console.warn(
|
|
5270
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5271
|
-
);
|
|
5272
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5273
|
-
} catch (err) {
|
|
5274
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5275
|
-
return {
|
|
5276
|
-
ok: false,
|
|
5277
|
-
skipped: "reclaim-error",
|
|
5278
|
-
detail: errorMessage(err)
|
|
5279
|
-
};
|
|
5280
|
-
} finally {
|
|
5281
|
-
db?.close();
|
|
5282
|
-
}
|
|
5283
|
-
}
|
|
5284
|
-
|
|
5285
6148
|
// src/lib/tunnel/connection.ts
|
|
5286
6149
|
import WebSocket2 from "ws";
|
|
5287
6150
|
|
|
@@ -5362,8 +6225,8 @@ var StreamForwarder = class {
|
|
|
5362
6225
|
pauseStream(sid, stream) {
|
|
5363
6226
|
if (!stream.paused) {
|
|
5364
6227
|
stream.paused = true;
|
|
5365
|
-
stream.pausePromise = new Promise((
|
|
5366
|
-
stream.resume =
|
|
6228
|
+
stream.pausePromise = new Promise((resolve5) => {
|
|
6229
|
+
stream.resume = resolve5;
|
|
5367
6230
|
});
|
|
5368
6231
|
}
|
|
5369
6232
|
if (stream.pauseTimer !== null) clearTimeout(stream.pauseTimer);
|
|
@@ -5430,12 +6293,12 @@ var StreamForwarder = class {
|
|
|
5430
6293
|
let endBody;
|
|
5431
6294
|
if (has_body) {
|
|
5432
6295
|
const chunks = [];
|
|
5433
|
-
bodyPromise = new Promise((
|
|
6296
|
+
bodyPromise = new Promise((resolve5) => {
|
|
5434
6297
|
pushBody = (buf) => {
|
|
5435
6298
|
chunks.push(buf);
|
|
5436
6299
|
};
|
|
5437
6300
|
endBody = () => {
|
|
5438
|
-
|
|
6301
|
+
resolve5(Buffer.concat(chunks));
|
|
5439
6302
|
};
|
|
5440
6303
|
});
|
|
5441
6304
|
}
|
|
@@ -5511,6 +6374,8 @@ var StreamForwarder = class {
|
|
|
5511
6374
|
if (done) break;
|
|
5512
6375
|
const chunk = Buffer.from(value);
|
|
5513
6376
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
6377
|
+
await stream.pausePromise;
|
|
6378
|
+
if (ac.signal.aborted) break;
|
|
5514
6379
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5515
6380
|
if (this.binaryFramesSupported) {
|
|
5516
6381
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5553,6 +6418,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5553
6418
|
}
|
|
5554
6419
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5555
6420
|
var BASE_RECONNECT_DELAY = 500;
|
|
6421
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5556
6422
|
function getReconnectDelay(attempt) {
|
|
5557
6423
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5558
6424
|
const jitter = Math.random() * 1e3;
|
|
@@ -5603,7 +6469,7 @@ function connectTunnel(options) {
|
|
|
5603
6469
|
} = options;
|
|
5604
6470
|
const tunnelUrl = getTunnelUrlConfig();
|
|
5605
6471
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
5606
|
-
return new Promise((
|
|
6472
|
+
return new Promise((resolve5, reject) => {
|
|
5607
6473
|
const ws = new WebSocket2(url, {
|
|
5608
6474
|
headers: {
|
|
5609
6475
|
Authorization: authHeader,
|
|
@@ -5624,10 +6490,27 @@ function connectTunnel(options) {
|
|
|
5624
6490
|
ws.close();
|
|
5625
6491
|
reject(new Error("Connection timeout"));
|
|
5626
6492
|
}, 3e4);
|
|
6493
|
+
let relaySilenceTimeout;
|
|
6494
|
+
let relaySilenceTimedOut = false;
|
|
6495
|
+
const clearRelaySilenceTimeout = () => {
|
|
6496
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6497
|
+
clearTimeout(relaySilenceTimeout);
|
|
6498
|
+
relaySilenceTimeout = void 0;
|
|
6499
|
+
};
|
|
6500
|
+
const armRelaySilenceTimeout = () => {
|
|
6501
|
+
clearRelaySilenceTimeout();
|
|
6502
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6503
|
+
relaySilenceTimeout = void 0;
|
|
6504
|
+
relaySilenceTimedOut = true;
|
|
6505
|
+
ws.terminate();
|
|
6506
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6507
|
+
relaySilenceTimeout.unref?.();
|
|
6508
|
+
};
|
|
5627
6509
|
let upgradeRejection = null;
|
|
5628
6510
|
let upgradeRejectionReason = null;
|
|
5629
6511
|
ws.on("unexpected-response", (_req, res) => {
|
|
5630
6512
|
clearTimeout(connectionTimeout);
|
|
6513
|
+
clearRelaySilenceTimeout();
|
|
5631
6514
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5632
6515
|
upgradeRejectionReason = reason;
|
|
5633
6516
|
const chunks = [];
|
|
@@ -5657,6 +6540,7 @@ function connectTunnel(options) {
|
|
|
5657
6540
|
onInfo?.("WebSocket connection established");
|
|
5658
6541
|
});
|
|
5659
6542
|
ws.on("message", (data, isBinary) => {
|
|
6543
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5660
6544
|
if (isBinary) {
|
|
5661
6545
|
try {
|
|
5662
6546
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5690,10 +6574,11 @@ function connectTunnel(options) {
|
|
|
5690
6574
|
switch (message.type) {
|
|
5691
6575
|
case "connected": {
|
|
5692
6576
|
clearTimeout(connectionTimeout);
|
|
6577
|
+
armRelaySilenceTimeout();
|
|
5693
6578
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5694
6579
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5695
6580
|
onConnected?.(connectedAgentId);
|
|
5696
|
-
|
|
6581
|
+
resolve5({
|
|
5697
6582
|
ws,
|
|
5698
6583
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
5699
6584
|
});
|
|
@@ -5714,6 +6599,7 @@ function connectTunnel(options) {
|
|
|
5714
6599
|
});
|
|
5715
6600
|
ws.on("error", (error2) => {
|
|
5716
6601
|
clearTimeout(connectionTimeout);
|
|
6602
|
+
clearRelaySilenceTimeout();
|
|
5717
6603
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5718
6604
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5719
6605
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5725,7 +6611,8 @@ function connectTunnel(options) {
|
|
|
5725
6611
|
);
|
|
5726
6612
|
});
|
|
5727
6613
|
ws.on("close", (code, reason) => {
|
|
5728
|
-
|
|
6614
|
+
clearRelaySilenceTimeout();
|
|
6615
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5729
6616
|
forwarder.abortAll();
|
|
5730
6617
|
onDisconnected?.(code, reasonStr);
|
|
5731
6618
|
});
|
|
@@ -5733,7 +6620,7 @@ function connectTunnel(options) {
|
|
|
5733
6620
|
}
|
|
5734
6621
|
|
|
5735
6622
|
// src/lib/tunnel/runner-connection.ts
|
|
5736
|
-
var RunnerConnection = class {
|
|
6623
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5737
6624
|
opts;
|
|
5738
6625
|
sleep;
|
|
5739
6626
|
now;
|
|
@@ -5747,6 +6634,10 @@ var RunnerConnection = class {
|
|
|
5747
6634
|
reconnectAttempt = 0;
|
|
5748
6635
|
generation = 0;
|
|
5749
6636
|
disconnectedAtMs = null;
|
|
6637
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6638
|
+
"No pong received",
|
|
6639
|
+
"Orphaned socket"
|
|
6640
|
+
]);
|
|
5750
6641
|
constructor(opts) {
|
|
5751
6642
|
this.opts = opts;
|
|
5752
6643
|
this.resolvedAgentId = opts.agentId;
|
|
@@ -5803,7 +6694,8 @@ var RunnerConnection = class {
|
|
|
5803
6694
|
onDisconnected: (code, reason) => {
|
|
5804
6695
|
this.disconnectedAtMs = this.now();
|
|
5805
6696
|
events.onDisconnected(code, reason, { generation: this.generation });
|
|
5806
|
-
|
|
6697
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6698
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5807
6699
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5808
6700
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5809
6701
|
});
|
|
@@ -6142,16 +7034,111 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6142
7034
|
};
|
|
6143
7035
|
}
|
|
6144
7036
|
|
|
7037
|
+
// src/lib/resource-usage-low-disk.ts
|
|
7038
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
7039
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
7040
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
7041
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
7042
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
7043
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
7044
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
7045
|
+
var BAND_RANK = {
|
|
7046
|
+
low: 1,
|
|
7047
|
+
very_low: 2,
|
|
7048
|
+
critical: 3
|
|
7049
|
+
};
|
|
7050
|
+
function bandFor(freeBytes) {
|
|
7051
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
7052
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
7053
|
+
return "low";
|
|
7054
|
+
}
|
|
7055
|
+
function formatBytes(bytes) {
|
|
7056
|
+
if (bytes === null) return "unavailable";
|
|
7057
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
7058
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
7059
|
+
const value = bytes / 1024 ** unitIndex;
|
|
7060
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
7061
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
7062
|
+
}
|
|
7063
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
7064
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
7065
|
+
}
|
|
7066
|
+
function evaluateLowDiskWarning(input) {
|
|
7067
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
7068
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
7069
|
+
return { warning: null, state };
|
|
7070
|
+
}
|
|
7071
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
7072
|
+
if (recovered) {
|
|
7073
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
7074
|
+
}
|
|
7075
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
7076
|
+
if (!low) return { warning: null, state };
|
|
7077
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
7078
|
+
const currentRank = BAND_RANK[currentBand];
|
|
7079
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
7080
|
+
const worse = currentRank > previousRank;
|
|
7081
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
7082
|
+
const nextState = {
|
|
7083
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
7084
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
7085
|
+
};
|
|
7086
|
+
return {
|
|
7087
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
7088
|
+
state: nextState
|
|
7089
|
+
};
|
|
7090
|
+
}
|
|
7091
|
+
|
|
7092
|
+
// src/lib/opencode/session-store-health.ts
|
|
7093
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
7094
|
+
function createSessionStoreHealth(options = {}) {
|
|
7095
|
+
const now = options.now ?? (() => Date.now());
|
|
7096
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
7097
|
+
let lastUnwritableAt = null;
|
|
7098
|
+
let lastWritableAt = null;
|
|
7099
|
+
const causes = /* @__PURE__ */ new Map();
|
|
7100
|
+
return {
|
|
7101
|
+
observeUnwritable: () => {
|
|
7102
|
+
lastUnwritableAt = now();
|
|
7103
|
+
},
|
|
7104
|
+
observeWritable: () => {
|
|
7105
|
+
lastWritableAt = now();
|
|
7106
|
+
},
|
|
7107
|
+
isUnwritable: () => {
|
|
7108
|
+
if (lastUnwritableAt === null) return false;
|
|
7109
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
7110
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
7111
|
+
},
|
|
7112
|
+
observeCause: (sessionId, cause) => {
|
|
7113
|
+
if (sessionId === null) return;
|
|
7114
|
+
causes.delete(sessionId);
|
|
7115
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
7116
|
+
while (causes.size > 256) {
|
|
7117
|
+
const oldest = causes.keys().next().value;
|
|
7118
|
+
if (typeof oldest !== "string") break;
|
|
7119
|
+
causes.delete(oldest);
|
|
7120
|
+
}
|
|
7121
|
+
},
|
|
7122
|
+
readCause: (sessionId) => {
|
|
7123
|
+
const observed = causes.get(sessionId);
|
|
7124
|
+
if (!observed) return null;
|
|
7125
|
+
causes.delete(sessionId);
|
|
7126
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
7127
|
+
return observed.cause;
|
|
7128
|
+
}
|
|
7129
|
+
};
|
|
7130
|
+
}
|
|
7131
|
+
|
|
6145
7132
|
// src/lib/channels/driver.ts
|
|
6146
7133
|
import { homedir as homedir6 } from "node:os";
|
|
6147
7134
|
|
|
6148
7135
|
// src/lib/runner-file-sync.ts
|
|
6149
|
-
import { join as
|
|
7136
|
+
import { join as join10 } from "node:path";
|
|
6150
7137
|
|
|
6151
7138
|
// src/lib/file-push.ts
|
|
6152
7139
|
import { randomUUID } from "node:crypto";
|
|
6153
7140
|
import { chmod, mkdir, open as open3, realpath, rename, unlink } from "node:fs/promises";
|
|
6154
|
-
import { basename, dirname as
|
|
7141
|
+
import { basename as basename2, dirname as dirname6, isAbsolute as isAbsolute2, join as join9, relative as relative2, resolve as resolve3, sep as sep2 } from "node:path";
|
|
6155
7142
|
var FILE_MODE = 384;
|
|
6156
7143
|
var DIRECTORY_MODE = 448;
|
|
6157
7144
|
async function writePushedFile(request2) {
|
|
@@ -6182,9 +7169,9 @@ async function writePushedFile(request2) {
|
|
|
6182
7169
|
}
|
|
6183
7170
|
try {
|
|
6184
7171
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
6185
|
-
|
|
7172
|
+
dirname6(candidate)
|
|
6186
7173
|
);
|
|
6187
|
-
const realTarget =
|
|
7174
|
+
const realTarget = join9(existingAncestor, ...missingSegments, basename2(candidate));
|
|
6188
7175
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
6189
7176
|
if (allowedDirectory === null) {
|
|
6190
7177
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -6194,8 +7181,8 @@ async function writePushedFile(request2) {
|
|
|
6194
7181
|
}
|
|
6195
7182
|
if (missingSegments.length > 0) {
|
|
6196
7183
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
6197
|
-
const realParent = await realpath(
|
|
6198
|
-
if (realParent !==
|
|
7184
|
+
const realParent = await realpath(dirname6(realTarget));
|
|
7185
|
+
if (realParent !== dirname6(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
6199
7186
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
6200
7187
|
path: realTarget,
|
|
6201
7188
|
bytes,
|
|
@@ -6220,15 +7207,15 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
6220
7207
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
6221
7208
|
return null;
|
|
6222
7209
|
}
|
|
6223
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
7210
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join9(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
6224
7211
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
6225
7212
|
return null;
|
|
6226
7213
|
}
|
|
6227
|
-
if (!
|
|
7214
|
+
if (!isAbsolute2(expanded)) {
|
|
6228
7215
|
return null;
|
|
6229
7216
|
}
|
|
6230
|
-
const candidate =
|
|
6231
|
-
const name =
|
|
7217
|
+
const candidate = resolve3(expanded);
|
|
7218
|
+
const name = basename2(candidate);
|
|
6232
7219
|
return name === "" || name === "." || name === ".." ? null : candidate;
|
|
6233
7220
|
}
|
|
6234
7221
|
async function resolveNearestExistingAncestor(directory) {
|
|
@@ -6238,18 +7225,18 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
6238
7225
|
try {
|
|
6239
7226
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
6240
7227
|
} catch (err) {
|
|
6241
|
-
const parent =
|
|
7228
|
+
const parent = dirname6(current);
|
|
6242
7229
|
if (err.code !== "ENOENT" || parent === current) {
|
|
6243
7230
|
throw err;
|
|
6244
7231
|
}
|
|
6245
|
-
missingSegments.unshift(
|
|
7232
|
+
missingSegments.unshift(basename2(current));
|
|
6246
7233
|
current = parent;
|
|
6247
7234
|
}
|
|
6248
7235
|
}
|
|
6249
7236
|
}
|
|
6250
7237
|
async function findContainingAllowedDirectory(allowedDirectories, realTarget) {
|
|
6251
7238
|
for (const directory of allowedDirectories) {
|
|
6252
|
-
if (!
|
|
7239
|
+
if (!isAbsolute2(directory)) {
|
|
6253
7240
|
log("warn", "file_push_allowed_directory_skipped", { directory, reason: "not_absolute" });
|
|
6254
7241
|
continue;
|
|
6255
7242
|
}
|
|
@@ -6287,19 +7274,19 @@ async function realpathCreatingIfMissing(directory) {
|
|
|
6287
7274
|
}
|
|
6288
7275
|
}
|
|
6289
7276
|
function contains(realDirectory, realTarget) {
|
|
6290
|
-
const rel =
|
|
6291
|
-
return rel !== "" && rel !== ".." && !rel.startsWith(`..${
|
|
7277
|
+
const rel = relative2(realDirectory, realTarget);
|
|
7278
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep2}`) && !isAbsolute2(rel);
|
|
6292
7279
|
}
|
|
6293
7280
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
6294
7281
|
let current = existingAncestor;
|
|
6295
7282
|
for (const segment of missingSegments) {
|
|
6296
|
-
current =
|
|
7283
|
+
current = join9(current, segment);
|
|
6297
7284
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
6298
7285
|
await chmod(current, DIRECTORY_MODE);
|
|
6299
7286
|
}
|
|
6300
7287
|
}
|
|
6301
7288
|
async function writeAtomically(realTarget, content) {
|
|
6302
|
-
const temporaryPath =
|
|
7289
|
+
const temporaryPath = join9(dirname6(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
6303
7290
|
let handle;
|
|
6304
7291
|
try {
|
|
6305
7292
|
handle = await open3(temporaryPath, "wx", FILE_MODE);
|
|
@@ -6429,12 +7416,12 @@ var NOT_APPLIED = {
|
|
|
6429
7416
|
opencodeAuthApplied: false
|
|
6430
7417
|
};
|
|
6431
7418
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
6432
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6433
|
-
return expanded ===
|
|
7419
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7420
|
+
return expanded === join10(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
6434
7421
|
}
|
|
6435
7422
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
6436
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6437
|
-
return expanded ===
|
|
7423
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7424
|
+
return expanded === join10(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
6438
7425
|
}
|
|
6439
7426
|
async function applyOne(options, file) {
|
|
6440
7427
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -6578,6 +7565,11 @@ function messageIdOf(m) {
|
|
|
6578
7565
|
const infoId = m.info?.id;
|
|
6579
7566
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6580
7567
|
}
|
|
7568
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7569
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7570
|
+
if (replyShape === null) return base;
|
|
7571
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7572
|
+
}
|
|
6581
7573
|
function cleanImageMime(contentType) {
|
|
6582
7574
|
if (!contentType) return null;
|
|
6583
7575
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6598,8 +7590,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6598
7590
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6599
7591
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6600
7592
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7593
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7594
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7595
|
+
var SESSION_ERROR_RETAINED_DETAIL = "Runner-observed OpenCode session error: ";
|
|
7596
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6601
7597
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6602
7598
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7599
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6603
7600
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6604
7601
|
var HEARTBEAT_MS = 6e4;
|
|
6605
7602
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6661,6 +7658,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6661
7658
|
maxActiveSessions;
|
|
6662
7659
|
watcherStallMs;
|
|
6663
7660
|
wedgeWarningIntervalMs;
|
|
7661
|
+
sessionStoreHealth;
|
|
6664
7662
|
openCodeClient;
|
|
6665
7663
|
/** Cache of conversationId → opencode sessionId. */
|
|
6666
7664
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6718,12 +7716,15 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6718
7716
|
/**
|
|
6719
7717
|
* Per-conversation throttle state for the #183 recurrence warning (#1618
|
|
6720
7718
|
* WI-4) — see `reportWedgedConversation`'s doc comment for why this exists.
|
|
6721
|
-
* `firstWedgedAt` anchors `stuck_for_ms`; `
|
|
6722
|
-
*
|
|
6723
|
-
*
|
|
6724
|
-
*
|
|
6725
|
-
* conversation dispatches anything
|
|
6726
|
-
*
|
|
7719
|
+
* `firstWedgedAt` anchors `stuck_for_ms`; `warnAnchorAt` is the throttle
|
|
7720
|
+
* anchor from first detection through the first warning, then from each
|
|
7721
|
+
* warning onward; `consecutiveTicks` is reported in the log text so the
|
|
7722
|
+
* operator sees magnitude, not repetition. Cleared the moment the
|
|
7723
|
+
* conversation dispatches anything, or when it drops off the pending list
|
|
7724
|
+
* (`runDrain`'s prune, the dominant real-world path — a wedge normally ends
|
|
7725
|
+
* with its message completing, not a fresh dispatch on the same
|
|
7726
|
+
* conversation) — either way a fresh wedge, if it recurs, is a new
|
|
7727
|
+
* incident. Bounded FIFO, mirroring `supersededSessions`
|
|
6727
7728
|
* (`MAX_WEDGED_CONVERSATIONS`).
|
|
6728
7729
|
*/
|
|
6729
7730
|
wedgeWarnings = /* @__PURE__ */ new Map();
|
|
@@ -6753,6 +7754,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6753
7754
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6754
7755
|
*/
|
|
6755
7756
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7757
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6756
7758
|
/**
|
|
6757
7759
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6758
7760
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6785,10 +7787,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6785
7787
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6786
7788
|
*/
|
|
6787
7789
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7790
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7791
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6788
7792
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6789
7793
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6790
7794
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6791
7795
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7796
|
+
/**
|
|
7797
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7798
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7799
|
+
* identical-text turn, so it never produces a correlation.
|
|
7800
|
+
*/
|
|
7801
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6792
7802
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6793
7803
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6794
7804
|
/**
|
|
@@ -6874,21 +7884,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6874
7884
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6875
7885
|
*/
|
|
6876
7886
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6877
|
-
/**
|
|
6878
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6879
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6880
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6881
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6882
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6883
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6884
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6885
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6886
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6887
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6888
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6889
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6890
|
-
*/
|
|
6891
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6892
7887
|
/**
|
|
6893
7888
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6894
7889
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6906,8 +7901,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6906
7901
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6907
7902
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6908
7903
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6909
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6910
|
-
*
|
|
7904
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7905
|
+
* the terminal-report fence and is never sent again).
|
|
6911
7906
|
*/
|
|
6912
7907
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6913
7908
|
/**
|
|
@@ -6959,6 +7954,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6959
7954
|
subagentInvocationPrefetches = /* @__PURE__ */ new Map();
|
|
6960
7955
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
6961
7956
|
draining = false;
|
|
7957
|
+
pendingProviderRegistryRepair = null;
|
|
6962
7958
|
/**
|
|
6963
7959
|
* Serialises runner-file syncs (#559) so the ~2s poll tick and a concurrent
|
|
6964
7960
|
* drain ping don't download, write and ack the same file twice.
|
|
@@ -7030,6 +8026,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7030
8026
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
7031
8027
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
7032
8028
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
8029
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
7033
8030
|
}
|
|
7034
8031
|
get isV2() {
|
|
7035
8032
|
return this.openCodeClient.version === "v2";
|
|
@@ -7045,7 +8042,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7045
8042
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
7046
8043
|
}
|
|
7047
8044
|
async listSessions() {
|
|
7048
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
8045
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
8046
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
8047
|
+
});
|
|
7049
8048
|
}
|
|
7050
8049
|
async sessionExists(sessionId) {
|
|
7051
8050
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7141,8 +8140,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7141
8140
|
async runDrain() {
|
|
7142
8141
|
let dispatched = 0;
|
|
7143
8142
|
try {
|
|
8143
|
+
const pendingRepair = this.pendingProviderRegistryRepair;
|
|
8144
|
+
if (pendingRepair && this.protectedSessionIds().size === 0) {
|
|
8145
|
+
try {
|
|
8146
|
+
await this.issueProviderRegistryReload(
|
|
8147
|
+
pendingRepair.conversationId,
|
|
8148
|
+
pendingRepair.messageId
|
|
8149
|
+
);
|
|
8150
|
+
} catch (err) {
|
|
8151
|
+
this.log({
|
|
8152
|
+
level: "error",
|
|
8153
|
+
message: `Provider registry repair adoption failed for message ${pendingRepair.messageId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
8154
|
+
conversation_id: pendingRepair.conversationId,
|
|
8155
|
+
message_id: pendingRepair.messageId
|
|
8156
|
+
});
|
|
8157
|
+
}
|
|
8158
|
+
return 0;
|
|
8159
|
+
}
|
|
7144
8160
|
this.pendingMessageIds.clear();
|
|
7145
8161
|
const conversations = await this.getPendingConversations();
|
|
8162
|
+
if (this.wedgeWarnings.size > 0) {
|
|
8163
|
+
const pendingConversationIds = new Set(conversations.map(({ id }) => id));
|
|
8164
|
+
for (const conversationId of this.wedgeWarnings.keys()) {
|
|
8165
|
+
if (!pendingConversationIds.has(conversationId)) {
|
|
8166
|
+
this.wedgeWarnings.delete(conversationId);
|
|
8167
|
+
}
|
|
8168
|
+
}
|
|
8169
|
+
}
|
|
7146
8170
|
if (this.recycleRequestedFlag) {
|
|
7147
8171
|
this.stop();
|
|
7148
8172
|
}
|
|
@@ -7222,6 +8246,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7222
8246
|
}
|
|
7223
8247
|
return ids;
|
|
7224
8248
|
}
|
|
8249
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
8250
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
8251
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
8252
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
8253
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
8254
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
8255
|
+
}
|
|
8256
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
8257
|
+
}
|
|
8258
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
8259
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
8260
|
+
this.awaitingCorrelation.delete(messageId);
|
|
8261
|
+
this.awaitingReadopt.delete(messageId);
|
|
8262
|
+
this.sessions.delete(conversationId);
|
|
8263
|
+
this.supersede(conversationId, sessionId);
|
|
8264
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
8265
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
8266
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
8267
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
8268
|
+
this.log({
|
|
8269
|
+
level: "error",
|
|
8270
|
+
message: errorMessage3,
|
|
8271
|
+
conversation_id: conversationId,
|
|
8272
|
+
message_id: messageId
|
|
8273
|
+
});
|
|
8274
|
+
const reportError = await this.reportUntrackableAck(
|
|
8275
|
+
conversationId,
|
|
8276
|
+
messageId,
|
|
8277
|
+
null,
|
|
8278
|
+
errorMessage3
|
|
8279
|
+
);
|
|
8280
|
+
if (reportError !== null) {
|
|
8281
|
+
this.log({
|
|
8282
|
+
level: "warn",
|
|
8283
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
8284
|
+
conversation_id: conversationId,
|
|
8285
|
+
message_id: messageId
|
|
8286
|
+
});
|
|
8287
|
+
}
|
|
8288
|
+
if (source === "readopt") {
|
|
8289
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
8290
|
+
}
|
|
8291
|
+
}
|
|
7225
8292
|
/**
|
|
7226
8293
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7227
8294
|
*
|
|
@@ -7401,6 +8468,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7401
8468
|
}
|
|
7402
8469
|
break;
|
|
7403
8470
|
}
|
|
8471
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8472
|
+
if (correlation) {
|
|
8473
|
+
correlation.attempts += 1;
|
|
8474
|
+
let correlatedId = null;
|
|
8475
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8476
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8477
|
+
conv,
|
|
8478
|
+
message,
|
|
8479
|
+
correlation.sessionId
|
|
8480
|
+
);
|
|
8481
|
+
if (polled.ok)
|
|
8482
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8483
|
+
} else if (correlation.knownUserIds === null) {
|
|
8484
|
+
this.log({
|
|
8485
|
+
level: "debug",
|
|
8486
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8487
|
+
conversation_id: conv.id,
|
|
8488
|
+
message_id: message.id
|
|
8489
|
+
});
|
|
8490
|
+
}
|
|
8491
|
+
if (correlatedId !== null) {
|
|
8492
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8493
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8494
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8495
|
+
this.dispatched.add(message.id);
|
|
8496
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8497
|
+
dispatched += 1;
|
|
8498
|
+
this.log({
|
|
8499
|
+
level: "info",
|
|
8500
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8501
|
+
conversation_id: conv.id,
|
|
8502
|
+
message_id: message.id
|
|
8503
|
+
});
|
|
8504
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8505
|
+
opencode_message_id: correlatedId
|
|
8506
|
+
});
|
|
8507
|
+
continue;
|
|
8508
|
+
}
|
|
8509
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8510
|
+
await this.handoffUntrackableCorrelation(
|
|
8511
|
+
conv.id,
|
|
8512
|
+
message.id,
|
|
8513
|
+
correlation.sessionId,
|
|
8514
|
+
correlation.attempts,
|
|
8515
|
+
"fresh"
|
|
8516
|
+
);
|
|
8517
|
+
break;
|
|
8518
|
+
}
|
|
8519
|
+
break;
|
|
8520
|
+
}
|
|
7404
8521
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7405
8522
|
if (effectiveOpencodeMessageId) {
|
|
7406
8523
|
const outcome = await this.resolveRedrive(
|
|
@@ -7422,6 +8539,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7422
8539
|
model: message.opencode_model ?? void 0
|
|
7423
8540
|
};
|
|
7424
8541
|
let opencodeMessageId;
|
|
8542
|
+
let preDispatchUserIds = null;
|
|
7425
8543
|
try {
|
|
7426
8544
|
this.log({
|
|
7427
8545
|
level: "info",
|
|
@@ -7451,7 +8569,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7451
8569
|
message.content,
|
|
7452
8570
|
options,
|
|
7453
8571
|
sendAttachments,
|
|
7454
|
-
this.openCodeClient
|
|
8572
|
+
this.openCodeClient,
|
|
8573
|
+
{
|
|
8574
|
+
onKnownUserIds: (ids) => {
|
|
8575
|
+
preDispatchUserIds = ids;
|
|
8576
|
+
},
|
|
8577
|
+
onKnownUserIdsError: (error2) => {
|
|
8578
|
+
this.log({
|
|
8579
|
+
level: "warn",
|
|
8580
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8581
|
+
conversation_id: conv.id,
|
|
8582
|
+
message_id: message.id
|
|
8583
|
+
});
|
|
8584
|
+
}
|
|
8585
|
+
}
|
|
7455
8586
|
)
|
|
7456
8587
|
);
|
|
7457
8588
|
} catch (err) {
|
|
@@ -7459,6 +8590,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7459
8590
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7460
8591
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7461
8592
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8593
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7462
8594
|
this.log({
|
|
7463
8595
|
level: "error",
|
|
7464
8596
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7529,44 +8661,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7529
8661
|
message_id: message.id
|
|
7530
8662
|
});
|
|
7531
8663
|
break;
|
|
7532
|
-
}
|
|
7533
|
-
if (opencodeMessageId === null) {
|
|
7534
|
-
if (this.isV2) {
|
|
7535
|
-
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7536
|
-
}
|
|
7537
|
-
const streak = this.recordUnconfirmedDispatch(message.id, sessionId);
|
|
7538
|
-
if (streak < MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7539
|
-
this.log({
|
|
7540
|
-
level: "warn",
|
|
7541
|
-
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back (${streak}/${MAX_IDENTICAL_REDRIVE_POLL_FAILURES}) \u2014 leaving un-tracked to retry next tick`,
|
|
7542
|
-
conversation_id: conv.id,
|
|
7543
|
-
message_id: message.id
|
|
7544
|
-
});
|
|
7545
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7546
|
-
continue;
|
|
8664
|
+
}
|
|
8665
|
+
if (opencodeMessageId === null) {
|
|
8666
|
+
if (this.isV2) {
|
|
8667
|
+
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7547
8668
|
}
|
|
7548
|
-
this.
|
|
7549
|
-
|
|
7550
|
-
|
|
7551
|
-
|
|
8669
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8670
|
+
sessionId,
|
|
8671
|
+
content: message.content,
|
|
8672
|
+
attempts: 0,
|
|
8673
|
+
knownUserIds: preDispatchUserIds
|
|
8674
|
+
});
|
|
7552
8675
|
this.log({
|
|
7553
|
-
level: "
|
|
7554
|
-
message:
|
|
8676
|
+
level: "warn",
|
|
8677
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7555
8678
|
conversation_id: conv.id,
|
|
7556
8679
|
message_id: message.id
|
|
7557
8680
|
});
|
|
7558
|
-
|
|
7559
|
-
this.log({
|
|
7560
|
-
level: "warn",
|
|
7561
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7562
|
-
conversation_id: conv.id,
|
|
7563
|
-
message_id: message.id
|
|
7564
|
-
});
|
|
7565
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7566
|
-
});
|
|
8681
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7567
8682
|
break;
|
|
7568
8683
|
}
|
|
7569
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7570
8684
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7571
8685
|
this.releasedOpencodeIds.delete(message.id);
|
|
7572
8686
|
this.dispatched.add(message.id);
|
|
@@ -7589,14 +8703,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7589
8703
|
* skip-because-already-`dispatched`, the exact signature of a message stuck
|
|
7590
8704
|
* acknowledged-but-never-worked.
|
|
7591
8705
|
*
|
|
7592
|
-
*
|
|
7593
|
-
*
|
|
7594
|
-
*
|
|
7595
|
-
*
|
|
7596
|
-
*
|
|
7597
|
-
*
|
|
7598
|
-
*
|
|
7599
|
-
* than repetition.
|
|
8706
|
+
* The first detection starts the clock and emits a local `debug` entry. The
|
|
8707
|
+
* `warn` log and paired `dispatch_wedged` signal fire once the wedge has
|
|
8708
|
+
* lasted `wedgeWarningIntervalMs`, then at most once per interval, naming the
|
|
8709
|
+
* consecutive-tick count so the operator sees magnitude rather than repetition.
|
|
8710
|
+
* A watcher stall that the loop-liveness watchdog recovers is expected and is
|
|
8711
|
+
* already reported by its `watcher_recovered` signal, so it does not reach an
|
|
8712
|
+
* operator through this warning.
|
|
7600
8713
|
*
|
|
7601
8714
|
* Deliberately does NOT trigger a release: WI-1's `reconcileWatchers` runs
|
|
7602
8715
|
* unconditionally on this same tick and is already recovering anything it
|
|
@@ -7608,36 +8721,41 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7608
8721
|
const existing = this.wedgeWarnings.get(conv.id);
|
|
7609
8722
|
const firstWedgedAt = existing?.firstWedgedAt ?? now;
|
|
7610
8723
|
const consecutiveTicks = (existing?.consecutiveTicks ?? 0) + 1;
|
|
7611
|
-
const
|
|
7612
|
-
|
|
7613
|
-
|
|
7614
|
-
this.
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
8724
|
+
const warnAnchorAt = existing?.warnAnchorAt ?? now;
|
|
8725
|
+
const dueForWarn = now - warnAnchorAt >= this.wedgeWarningIntervalMs;
|
|
8726
|
+
if (!existing) {
|
|
8727
|
+
this.log({
|
|
8728
|
+
level: "debug",
|
|
8729
|
+
message: `Detected a possible wedge for conversation ${conv.id.slice(0, 8)} with ${messages.length} pending message(s); it will be reported only if it outlasts the watchdog's recovery.`,
|
|
8730
|
+
conversation_id: conv.id
|
|
8731
|
+
});
|
|
8732
|
+
}
|
|
8733
|
+
let untracked = 0;
|
|
8734
|
+
if (dueForWarn) {
|
|
8735
|
+
const stuckForMs = now - firstWedgedAt;
|
|
8736
|
+
untracked = this.countUntrackedIds(messages);
|
|
8737
|
+
this.log({
|
|
8738
|
+
level: "warn",
|
|
8739
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${Math.round(stuckForMs / 1e3)}s stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
8740
|
+
conversation_id: conv.id
|
|
7618
8741
|
});
|
|
7619
|
-
return;
|
|
7620
8742
|
}
|
|
7621
|
-
const stuckForMs = now - firstWedgedAt;
|
|
7622
|
-
const untracked = this.countUntrackedIds(messages);
|
|
7623
|
-
this.log({
|
|
7624
|
-
level: "warn",
|
|
7625
|
-
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode for ${consecutiveTicks} consecutive tick(s) now (${stuckForMs}ms stuck). ` + (untracked > 0 ? `${untracked} of these id(s) are tracked by NO watcher \u2014 the dispatched/in-flight pairing invariant is violated for this conversation, which will NOT self-heal and needs a runner restart.` : `A watcher is tracking this work; the loop-liveness watchdog is already recovering it.`),
|
|
7626
|
-
conversation_id: conv.id
|
|
7627
|
-
});
|
|
7628
8743
|
this.wedgeWarnings.delete(conv.id);
|
|
7629
|
-
this.wedgeWarnings.set(conv.id, {
|
|
8744
|
+
this.wedgeWarnings.set(conv.id, {
|
|
8745
|
+
firstWedgedAt,
|
|
8746
|
+
warnAnchorAt: dueForWarn ? now : warnAnchorAt,
|
|
8747
|
+
consecutiveTicks
|
|
8748
|
+
});
|
|
7630
8749
|
while (this.wedgeWarnings.size > MAX_WEDGED_CONVERSATIONS) {
|
|
7631
8750
|
const oldest = this.wedgeWarnings.keys().next().value;
|
|
7632
8751
|
if (oldest === void 0) break;
|
|
7633
8752
|
this.wedgeWarnings.delete(oldest);
|
|
7634
8753
|
}
|
|
7635
|
-
if (
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
}
|
|
8754
|
+
if (!dueForWarn) return;
|
|
8755
|
+
void this.postSignal(conv.id, messages[0].id, "dispatch_wedged", {
|
|
8756
|
+
stuck_for_ms: now - firstWedgedAt,
|
|
8757
|
+
untracked
|
|
8758
|
+
});
|
|
7641
8759
|
}
|
|
7642
8760
|
/**
|
|
7643
8761
|
* How many of `messages`' ids are tracked by NO watcher's `inFlight` (#1618
|
|
@@ -8171,23 +9289,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8171
9289
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8172
9290
|
return 1;
|
|
8173
9291
|
}
|
|
8174
|
-
/**
|
|
8175
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8176
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8177
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8178
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8179
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8180
|
-
* count, since a new session is a genuinely different attempt.
|
|
8181
|
-
*/
|
|
8182
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8183
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8184
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8185
|
-
existing.count += 1;
|
|
8186
|
-
return existing.count;
|
|
8187
|
-
}
|
|
8188
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8189
|
-
return 1;
|
|
8190
|
-
}
|
|
8191
9292
|
/** Record one queued give-up toward the never-started bound. */
|
|
8192
9293
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8193
9294
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8499,8 +9600,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8499
9600
|
b2LastDescendantCheckMs: 0,
|
|
8500
9601
|
b2RootOngoingHeldLogged: false,
|
|
8501
9602
|
b2AbandonedSignalled: false,
|
|
9603
|
+
b2AbandonedWithoutAnswer: false,
|
|
8502
9604
|
ambiguousPinnedSinceMs: 0,
|
|
8503
9605
|
ambiguousResolved: false,
|
|
9606
|
+
sessionStoreFailurePending: false,
|
|
8504
9607
|
neverStartedResolved: false
|
|
8505
9608
|
});
|
|
8506
9609
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8522,6 +9625,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8522
9625
|
this.bufferedSessionErrors.delete(oldest);
|
|
8523
9626
|
}
|
|
8524
9627
|
}
|
|
9628
|
+
retainSessionError(event) {
|
|
9629
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9630
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9631
|
+
reason: event.reason,
|
|
9632
|
+
observedAt: this.now()
|
|
9633
|
+
});
|
|
9634
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9635
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9636
|
+
if (typeof oldest !== "string") break;
|
|
9637
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9638
|
+
}
|
|
9639
|
+
}
|
|
9640
|
+
readRetainedSessionError(sessionId) {
|
|
9641
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9642
|
+
if (!retained) return null;
|
|
9643
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9644
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9645
|
+
return null;
|
|
9646
|
+
}
|
|
9647
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9648
|
+
return retained.reason;
|
|
9649
|
+
}
|
|
8525
9650
|
/**
|
|
8526
9651
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8527
9652
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8617,8 +9742,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8617
9742
|
b2LastDescendantCheckMs: 0,
|
|
8618
9743
|
b2RootOngoingHeldLogged: false,
|
|
8619
9744
|
b2AbandonedSignalled: false,
|
|
9745
|
+
b2AbandonedWithoutAnswer: false,
|
|
8620
9746
|
ambiguousPinnedSinceMs: 0,
|
|
8621
9747
|
ambiguousResolved: false,
|
|
9748
|
+
sessionStoreFailurePending: false,
|
|
8622
9749
|
neverStartedResolved: false
|
|
8623
9750
|
});
|
|
8624
9751
|
}
|
|
@@ -8846,9 +9973,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8846
9973
|
return;
|
|
8847
9974
|
}
|
|
8848
9975
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9976
|
+
this.retainSessionError(event);
|
|
8849
9977
|
this.log({
|
|
8850
|
-
level: "
|
|
8851
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9978
|
+
level: "warn",
|
|
9979
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8852
9980
|
conversation_id: watcher.conv.id
|
|
8853
9981
|
});
|
|
8854
9982
|
return;
|
|
@@ -8878,9 +10006,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8878
10006
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8879
10007
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8880
10008
|
if (state !== "queued") {
|
|
10009
|
+
this.retainSessionError(event);
|
|
8881
10010
|
this.log({
|
|
8882
|
-
level: "
|
|
8883
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
10011
|
+
level: "warn",
|
|
10012
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8884
10013
|
conversation_id: watcher.conv.id,
|
|
8885
10014
|
message_id: inFlight.evidentMessageId
|
|
8886
10015
|
});
|
|
@@ -8892,14 +10021,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8892
10021
|
conversation_id: watcher.conv.id,
|
|
8893
10022
|
message_id: inFlight.evidentMessageId
|
|
8894
10023
|
});
|
|
8895
|
-
|
|
8896
|
-
|
|
8897
|
-
inFlight.
|
|
8898
|
-
|
|
8899
|
-
|
|
8900
|
-
|
|
10024
|
+
const isMissingProvider = isProviderModelNotFoundError(event.reason);
|
|
10025
|
+
if (isMissingProvider) {
|
|
10026
|
+
const dispatchedModel = inFlight.message.opencode_model;
|
|
10027
|
+
const separator = dispatchedModel?.indexOf("/") ?? -1;
|
|
10028
|
+
const hasProviderAndModel = dispatchedModel !== null && separator > 0 && separator < dispatchedModel.length - 1;
|
|
10029
|
+
await this.markFailed(
|
|
10030
|
+
watcher.conv.id,
|
|
10031
|
+
inFlight.evidentMessageId,
|
|
10032
|
+
event.sessionId,
|
|
10033
|
+
`OpenCode could not run this turn: ${event.reason}`,
|
|
10034
|
+
void 0,
|
|
10035
|
+
{
|
|
10036
|
+
classification: {
|
|
10037
|
+
kind: "model_auth",
|
|
10038
|
+
providerId: hasProviderAndModel ? dispatchedModel.slice(0, separator) : null,
|
|
10039
|
+
modelId: hasProviderAndModel ? dispatchedModel.slice(separator + 1) : null,
|
|
10040
|
+
reason: "missing"
|
|
10041
|
+
},
|
|
10042
|
+
errorShape: { name: null, statusCode: null }
|
|
10043
|
+
}
|
|
10044
|
+
);
|
|
10045
|
+
} else {
|
|
10046
|
+
await this.markFailed(
|
|
10047
|
+
watcher.conv.id,
|
|
10048
|
+
inFlight.evidentMessageId,
|
|
10049
|
+
event.sessionId,
|
|
10050
|
+
`OpenCode could not run this turn: ${event.reason}`
|
|
10051
|
+
);
|
|
10052
|
+
}
|
|
8901
10053
|
inFlight.done = true;
|
|
8902
10054
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10055
|
+
if (isMissingProvider) {
|
|
10056
|
+
await this.repairProviderRegistry(watcher.conv.id, inFlight.evidentMessageId);
|
|
10057
|
+
}
|
|
8903
10058
|
} catch (err) {
|
|
8904
10059
|
if (err instanceof ChannelAuthError) {
|
|
8905
10060
|
this.log({
|
|
@@ -8920,6 +10075,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8920
10075
|
this.sessionErrorHandled.delete(inFlight.evidentMessageId);
|
|
8921
10076
|
}
|
|
8922
10077
|
}
|
|
10078
|
+
async repairProviderRegistry(conversationId, messageId) {
|
|
10079
|
+
if (this.isV2) {
|
|
10080
|
+
void this.postSignal(conversationId, messageId, "provider_registry_reload_unsupported");
|
|
10081
|
+
return;
|
|
10082
|
+
}
|
|
10083
|
+
const activeSessions = this.protectedSessionIds().size;
|
|
10084
|
+
if (activeSessions > 0 || this.draining) {
|
|
10085
|
+
this.pendingProviderRegistryRepair = { conversationId, messageId };
|
|
10086
|
+
void this.postSignal(conversationId, messageId, "provider_registry_repair_deferred", {
|
|
10087
|
+
active_sessions: activeSessions
|
|
10088
|
+
});
|
|
10089
|
+
this.log({
|
|
10090
|
+
level: "warn",
|
|
10091
|
+
message: `Provider registry repair deferred until a lifecycle-safe point; ${activeSessions} active session(s) protected`,
|
|
10092
|
+
conversation_id: conversationId,
|
|
10093
|
+
message_id: messageId
|
|
10094
|
+
});
|
|
10095
|
+
return;
|
|
10096
|
+
}
|
|
10097
|
+
await this.issueProviderRegistryReload(conversationId, messageId);
|
|
10098
|
+
}
|
|
10099
|
+
async issueProviderRegistryReload(conversationId, messageId) {
|
|
10100
|
+
this.pendingProviderRegistryRepair = null;
|
|
10101
|
+
const result = await disposeAndReloadProviderConfig(this.port, this.openCodeClient);
|
|
10102
|
+
if (result.outcome === "failed") {
|
|
10103
|
+
const failure = result.status === null ? `transport failure${result.detail ? `: ${result.detail}` : ""}` : `HTTP ${result.status}`;
|
|
10104
|
+
this.log({
|
|
10105
|
+
level: "warn",
|
|
10106
|
+
message: `Provider registry reload failed (${failure})`,
|
|
10107
|
+
conversation_id: conversationId,
|
|
10108
|
+
message_id: messageId
|
|
10109
|
+
});
|
|
10110
|
+
}
|
|
10111
|
+
const repairOutcome = result.outcome === "reloaded" ? "reloaded" : "reload_failed";
|
|
10112
|
+
void this.postSignal(conversationId, messageId, "provider_registry_reload_issued", {
|
|
10113
|
+
repair_outcome: repairOutcome
|
|
10114
|
+
});
|
|
10115
|
+
}
|
|
8923
10116
|
/**
|
|
8924
10117
|
* The per-session polling loop (WI-3). Once per tick it:
|
|
8925
10118
|
* 1. polls `GET /session/:id/message` once and, per in-flight message,
|
|
@@ -9032,6 +10225,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9032
10225
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9033
10226
|
this.neverStartedAttempts.delete(id);
|
|
9034
10227
|
}
|
|
10228
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
10229
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10230
|
+
return;
|
|
10231
|
+
}
|
|
9035
10232
|
if (inFlight.neverStartedResolved) {
|
|
9036
10233
|
if (state === "running" || state === "done" || state === "failed") {
|
|
9037
10234
|
inFlight.neverStartedResolved = false;
|
|
@@ -9105,7 +10302,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9105
10302
|
if (state === "failed") {
|
|
9106
10303
|
this.anchorDeliveryDeadline(inFlight);
|
|
9107
10304
|
if (!inFlight.done) {
|
|
9108
|
-
const
|
|
10305
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
10306
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
10307
|
+
const retainedError = retainedReason === null || messageErrorText !== null && messageErrorText.includes(retainedReason) ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
10308
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
10309
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
10310
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
10311
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
10312
|
+
(detail) => detail !== null
|
|
10313
|
+
);
|
|
10314
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9109
10315
|
this.log({
|
|
9110
10316
|
level: "error",
|
|
9111
10317
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9191,10 +10397,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9191
10397
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9192
10398
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9193
10399
|
inFlight.b2AbandonedSignalled = false;
|
|
10400
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9194
10401
|
}
|
|
9195
10402
|
} else {
|
|
9196
10403
|
if (inFlight.b2AbandonedSignalled) {
|
|
9197
|
-
|
|
10404
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10405
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10406
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
10407
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10408
|
+
sessionId,
|
|
10409
|
+
watcher,
|
|
10410
|
+
inFlight,
|
|
10411
|
+
messages,
|
|
10412
|
+
finish
|
|
10413
|
+
);
|
|
10414
|
+
} else {
|
|
10415
|
+
await this.settleMessageDone(
|
|
10416
|
+
sessionId,
|
|
10417
|
+
watcher,
|
|
10418
|
+
inFlight,
|
|
10419
|
+
messages,
|
|
10420
|
+
composeAbandonmentNote(
|
|
10421
|
+
finish,
|
|
10422
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10423
|
+
"with-answer"
|
|
10424
|
+
)
|
|
10425
|
+
);
|
|
10426
|
+
}
|
|
9198
10427
|
return;
|
|
9199
10428
|
}
|
|
9200
10429
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9211,22 +10440,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9211
10440
|
descendantOngoing,
|
|
9212
10441
|
rootOngoing
|
|
9213
10442
|
})) {
|
|
10443
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10444
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10445
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10446
|
+
watched_for_ms: pinnedForMs,
|
|
10447
|
+
run_state: "failed",
|
|
10448
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10449
|
+
});
|
|
10450
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10451
|
+
return;
|
|
10452
|
+
}
|
|
9214
10453
|
inFlight.b2AbandonedSignalled = true;
|
|
10454
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10455
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10456
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
10457
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
10458
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10459
|
+
watched_for_ms: pinnedForMs,
|
|
10460
|
+
run_state: "failed",
|
|
10461
|
+
finish,
|
|
10462
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
10463
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
10464
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10465
|
+
});
|
|
10466
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10467
|
+
sessionId,
|
|
10468
|
+
watcher,
|
|
10469
|
+
inFlight,
|
|
10470
|
+
messages,
|
|
10471
|
+
finish
|
|
10472
|
+
);
|
|
10473
|
+
return;
|
|
10474
|
+
}
|
|
9215
10475
|
this.log({
|
|
9216
10476
|
level: "warn",
|
|
9217
10477
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9218
10478
|
conversation_id: conv.id,
|
|
9219
10479
|
message_id: id
|
|
9220
10480
|
});
|
|
9221
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9222
10481
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9223
10482
|
watched_for_ms: pinnedForMs,
|
|
9224
|
-
finish
|
|
10483
|
+
finish,
|
|
9225
10484
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9226
10485
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9227
10486
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9228
10487
|
});
|
|
9229
|
-
await this.settleMessageDone(
|
|
10488
|
+
await this.settleMessageDone(
|
|
10489
|
+
sessionId,
|
|
10490
|
+
watcher,
|
|
10491
|
+
inFlight,
|
|
10492
|
+
messages,
|
|
10493
|
+
composeAbandonmentNote(
|
|
10494
|
+
finish,
|
|
10495
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10496
|
+
"with-answer"
|
|
10497
|
+
)
|
|
10498
|
+
);
|
|
9230
10499
|
return;
|
|
9231
10500
|
}
|
|
9232
10501
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9269,6 +10538,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9269
10538
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9270
10539
|
sessionOngoing: ongoing
|
|
9271
10540
|
})) {
|
|
10541
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10542
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10543
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10544
|
+
watched_for_ms: pinnedForMs,
|
|
10545
|
+
run_state: "failed",
|
|
10546
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10547
|
+
});
|
|
10548
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10549
|
+
return;
|
|
10550
|
+
}
|
|
9272
10551
|
inFlight.ambiguousResolved = true;
|
|
9273
10552
|
this.log({
|
|
9274
10553
|
level: "warn",
|
|
@@ -9416,15 +10695,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9416
10695
|
}
|
|
9417
10696
|
}
|
|
9418
10697
|
/**
|
|
9419
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9420
|
-
*
|
|
9421
|
-
*
|
|
9422
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9423
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9424
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9425
|
-
* and risking the two copies silently drifting apart.
|
|
10698
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10699
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10700
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9426
10701
|
*/
|
|
9427
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10702
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9428
10703
|
const conv = watcher.conv;
|
|
9429
10704
|
this.anchorDeliveryDeadline(inFlight);
|
|
9430
10705
|
if (!inFlight.done) {
|
|
@@ -9451,7 +10726,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9451
10726
|
title,
|
|
9452
10727
|
usage,
|
|
9453
10728
|
usageAgentName,
|
|
9454
|
-
subagentInvocations
|
|
10729
|
+
subagentInvocations,
|
|
10730
|
+
note
|
|
9455
10731
|
);
|
|
9456
10732
|
} catch (err) {
|
|
9457
10733
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9493,6 +10769,79 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9493
10769
|
}
|
|
9494
10770
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9495
10771
|
}
|
|
10772
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10773
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10774
|
+
const conv = watcher.conv;
|
|
10775
|
+
const error2 = composeAbandonmentNote(
|
|
10776
|
+
finish,
|
|
10777
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10778
|
+
"without-answer"
|
|
10779
|
+
);
|
|
10780
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10781
|
+
this.log({
|
|
10782
|
+
level: "warn",
|
|
10783
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10784
|
+
conversation_id: conv.id,
|
|
10785
|
+
message_id: inFlight.evidentMessageId
|
|
10786
|
+
});
|
|
10787
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10788
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10789
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10790
|
+
messages,
|
|
10791
|
+
inFlight.opencodeMessageId,
|
|
10792
|
+
inFlight.evidentMessageId
|
|
10793
|
+
);
|
|
10794
|
+
try {
|
|
10795
|
+
await this.markFailed(
|
|
10796
|
+
conv.id,
|
|
10797
|
+
inFlight.evidentMessageId,
|
|
10798
|
+
sessionId,
|
|
10799
|
+
error2,
|
|
10800
|
+
usage,
|
|
10801
|
+
void 0,
|
|
10802
|
+
usageAgentName,
|
|
10803
|
+
subagentInvocations,
|
|
10804
|
+
true
|
|
10805
|
+
);
|
|
10806
|
+
} catch (err) {
|
|
10807
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10808
|
+
if (err instanceof ChannelTerminalError) {
|
|
10809
|
+
this.log({
|
|
10810
|
+
level: "warn",
|
|
10811
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10812
|
+
conversation_id: conv.id,
|
|
10813
|
+
message_id: inFlight.evidentMessageId
|
|
10814
|
+
});
|
|
10815
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10816
|
+
return;
|
|
10817
|
+
}
|
|
10818
|
+
if (this.now() >= inFlight.deadline) {
|
|
10819
|
+
this.log({
|
|
10820
|
+
level: "warn",
|
|
10821
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10822
|
+
conversation_id: conv.id,
|
|
10823
|
+
message_id: inFlight.evidentMessageId
|
|
10824
|
+
});
|
|
10825
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10826
|
+
return;
|
|
10827
|
+
}
|
|
10828
|
+
this.log({
|
|
10829
|
+
level: "warn",
|
|
10830
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10831
|
+
conversation_id: conv.id,
|
|
10832
|
+
message_id: inFlight.evidentMessageId
|
|
10833
|
+
});
|
|
10834
|
+
return;
|
|
10835
|
+
}
|
|
10836
|
+
inFlight.done = true;
|
|
10837
|
+
await this.reportSubagentAuthFailures(
|
|
10838
|
+
watcher.conv.id,
|
|
10839
|
+
inFlight.opencodeMessageId,
|
|
10840
|
+
inFlight.evidentMessageId,
|
|
10841
|
+
messages
|
|
10842
|
+
);
|
|
10843
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10844
|
+
}
|
|
9496
10845
|
/**
|
|
9497
10846
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9498
10847
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9548,6 +10897,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9548
10897
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9549
10898
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9550
10899
|
}
|
|
10900
|
+
/**
|
|
10901
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10902
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10903
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10904
|
+
*/
|
|
10905
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10906
|
+
const conv = watcher.conv;
|
|
10907
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10908
|
+
this.log({
|
|
10909
|
+
level: "error",
|
|
10910
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10911
|
+
conversation_id: conv.id,
|
|
10912
|
+
message_id: inFlight.evidentMessageId
|
|
10913
|
+
});
|
|
10914
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10915
|
+
try {
|
|
10916
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10917
|
+
} catch (err) {
|
|
10918
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10919
|
+
if (err instanceof ChannelTerminalError) {
|
|
10920
|
+
this.log({
|
|
10921
|
+
level: "warn",
|
|
10922
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10923
|
+
conversation_id: conv.id,
|
|
10924
|
+
message_id: inFlight.evidentMessageId
|
|
10925
|
+
});
|
|
10926
|
+
this.recordReleasedOpencodeId(
|
|
10927
|
+
inFlight.evidentMessageId,
|
|
10928
|
+
sessionId,
|
|
10929
|
+
inFlight.opencodeMessageId
|
|
10930
|
+
);
|
|
10931
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10932
|
+
return;
|
|
10933
|
+
}
|
|
10934
|
+
if (this.now() >= inFlight.deadline) {
|
|
10935
|
+
this.log({
|
|
10936
|
+
level: "warn",
|
|
10937
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10938
|
+
conversation_id: conv.id,
|
|
10939
|
+
message_id: inFlight.evidentMessageId
|
|
10940
|
+
});
|
|
10941
|
+
this.recordReleasedOpencodeId(
|
|
10942
|
+
inFlight.evidentMessageId,
|
|
10943
|
+
sessionId,
|
|
10944
|
+
inFlight.opencodeMessageId
|
|
10945
|
+
);
|
|
10946
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10947
|
+
return;
|
|
10948
|
+
}
|
|
10949
|
+
this.log({
|
|
10950
|
+
level: "warn",
|
|
10951
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10952
|
+
conversation_id: conv.id,
|
|
10953
|
+
message_id: inFlight.evidentMessageId
|
|
10954
|
+
});
|
|
10955
|
+
return;
|
|
10956
|
+
}
|
|
10957
|
+
inFlight.done = true;
|
|
10958
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10959
|
+
}
|
|
9551
10960
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9552
10961
|
/**
|
|
9553
10962
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9586,11 +10995,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9586
10995
|
}
|
|
9587
10996
|
}
|
|
9588
10997
|
}
|
|
9589
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10998
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9590
10999
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
11000
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
11001
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
11002
|
+
this.awaitingCorrelation.delete(id);
|
|
11003
|
+
this.log({
|
|
11004
|
+
level: "debug",
|
|
11005
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
11006
|
+
message_id: id
|
|
11007
|
+
});
|
|
11008
|
+
}
|
|
11009
|
+
}
|
|
9591
11010
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9592
11011
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9593
11012
|
this.untrackableAck.delete(id);
|
|
11013
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9594
11014
|
this.untrackableAckSignalled.delete(id);
|
|
9595
11015
|
this.untrackableAckWarned.delete(id);
|
|
9596
11016
|
this.log({
|
|
@@ -9630,7 +11050,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9630
11050
|
bySession.set(row.opencode_session_id, list);
|
|
9631
11051
|
}
|
|
9632
11052
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9633
|
-
let messages;
|
|
11053
|
+
let messages = null;
|
|
9634
11054
|
try {
|
|
9635
11055
|
if (this.isV2) {
|
|
9636
11056
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9639,9 +11059,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9639
11059
|
level: "warn",
|
|
9640
11060
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9641
11061
|
});
|
|
9642
|
-
|
|
11062
|
+
} else {
|
|
11063
|
+
messages = snapshot;
|
|
9643
11064
|
}
|
|
9644
|
-
messages = snapshot;
|
|
9645
11065
|
} else {
|
|
9646
11066
|
const polled = await pollSessionMessagesForRedrive(
|
|
9647
11067
|
this.port,
|
|
@@ -9654,15 +11074,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9654
11074
|
level: "warn",
|
|
9655
11075
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9656
11076
|
});
|
|
9657
|
-
|
|
11077
|
+
} else {
|
|
11078
|
+
messages = polled.messages;
|
|
9658
11079
|
}
|
|
9659
|
-
messages = polled.messages;
|
|
9660
11080
|
}
|
|
9661
11081
|
} catch (err) {
|
|
9662
11082
|
this.log({
|
|
9663
11083
|
level: "warn",
|
|
9664
11084
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9665
11085
|
});
|
|
11086
|
+
}
|
|
11087
|
+
if (messages === null) {
|
|
11088
|
+
for (const row of sessionRows) {
|
|
11089
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
11090
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
11091
|
+
}
|
|
11092
|
+
}
|
|
9666
11093
|
continue;
|
|
9667
11094
|
}
|
|
9668
11095
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9672,6 +11099,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9672
11099
|
}
|
|
9673
11100
|
}
|
|
9674
11101
|
}
|
|
11102
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
11103
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
11104
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
11105
|
+
if (!correlation) return false;
|
|
11106
|
+
correlation.attempts += 1;
|
|
11107
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
11108
|
+
if (correlatedId !== null) {
|
|
11109
|
+
this.awaitingCorrelation.delete(row.id);
|
|
11110
|
+
const conv = this.convForRow(sessionId, row);
|
|
11111
|
+
const message = this.queuedMessageForRow(row);
|
|
11112
|
+
const processedAtMs = this.processedAtMs(row);
|
|
11113
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
11114
|
+
this.dispatched.add(row.id);
|
|
11115
|
+
this.readopted.add(row.id);
|
|
11116
|
+
this.ensureWatcherRunning(sessionId);
|
|
11117
|
+
this.log({
|
|
11118
|
+
level: "info",
|
|
11119
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
11120
|
+
conversation_id: row.conversation_id,
|
|
11121
|
+
message_id: row.id
|
|
11122
|
+
});
|
|
11123
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
11124
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
11125
|
+
opencode_message_id: correlatedId
|
|
11126
|
+
});
|
|
11127
|
+
return true;
|
|
11128
|
+
}
|
|
11129
|
+
if (correlation.knownUserIds === null) {
|
|
11130
|
+
this.log({
|
|
11131
|
+
level: "debug",
|
|
11132
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
11133
|
+
conversation_id: row.conversation_id,
|
|
11134
|
+
message_id: row.id
|
|
11135
|
+
});
|
|
11136
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
11137
|
+
this.log({
|
|
11138
|
+
level: "warn",
|
|
11139
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
11140
|
+
conversation_id: row.conversation_id,
|
|
11141
|
+
message_id: row.id
|
|
11142
|
+
});
|
|
11143
|
+
}
|
|
11144
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
11145
|
+
await this.handoffUntrackableCorrelation(
|
|
11146
|
+
row.conversation_id,
|
|
11147
|
+
row.id,
|
|
11148
|
+
correlation.sessionId,
|
|
11149
|
+
correlation.attempts,
|
|
11150
|
+
"readopt"
|
|
11151
|
+
);
|
|
11152
|
+
}
|
|
11153
|
+
return true;
|
|
11154
|
+
}
|
|
9675
11155
|
/**
|
|
9676
11156
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9677
11157
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9679,8 +11159,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9679
11159
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9680
11160
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9681
11161
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9682
|
-
* before the restart)
|
|
9683
|
-
*
|
|
11162
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
11163
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
11164
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
11165
|
+
* `forceReadoptRun`):
|
|
9684
11166
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9685
11167
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9686
11168
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9689,7 +11171,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9689
11171
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9690
11172
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9691
11173
|
* tracking the stored id so the reply correlates by it;
|
|
9692
|
-
* - `unknown`/null id →
|
|
11174
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
11175
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9693
11176
|
*
|
|
9694
11177
|
* Only `ChannelAuthError` propagates.
|
|
9695
11178
|
*/
|
|
@@ -9703,6 +11186,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9703
11186
|
});
|
|
9704
11187
|
return;
|
|
9705
11188
|
}
|
|
11189
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9706
11190
|
const ocId = row.opencode_message_id;
|
|
9707
11191
|
const state = messageRunState(messages, ocId ?? "");
|
|
9708
11192
|
if (state === "done") {
|
|
@@ -9769,12 +11253,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9769
11253
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9770
11254
|
this.dontRedispatch.delete(row.id);
|
|
9771
11255
|
this.untrackableAck.delete(row.id);
|
|
11256
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9772
11257
|
this.untrackableAckSignalled.delete(row.id);
|
|
9773
11258
|
this.untrackableAckWarned.delete(row.id);
|
|
9774
11259
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9775
11260
|
return;
|
|
9776
11261
|
}
|
|
9777
|
-
if (this.
|
|
11262
|
+
if (this.untrackableAck.has(row.id)) {
|
|
11263
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
11264
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
11265
|
+
this.untrackableAckWarned.add(row.id);
|
|
11266
|
+
this.log({
|
|
11267
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
11268
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
11269
|
+
conversation_id: row.conversation_id,
|
|
11270
|
+
message_id: row.id
|
|
11271
|
+
});
|
|
11272
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
11273
|
+
const reportError = await this.reportUntrackableAck(
|
|
11274
|
+
row.conversation_id,
|
|
11275
|
+
row.id,
|
|
11276
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
11277
|
+
errorMessage3
|
|
11278
|
+
);
|
|
11279
|
+
if (reportError !== null) {
|
|
11280
|
+
this.log({
|
|
11281
|
+
level: "warn",
|
|
11282
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
11283
|
+
conversation_id: row.conversation_id,
|
|
11284
|
+
message_id: row.id
|
|
11285
|
+
});
|
|
11286
|
+
}
|
|
11287
|
+
} else {
|
|
11288
|
+
this.log({
|
|
11289
|
+
level: "debug",
|
|
11290
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
11291
|
+
conversation_id: row.conversation_id,
|
|
11292
|
+
message_id: row.id
|
|
11293
|
+
});
|
|
11294
|
+
}
|
|
11295
|
+
return;
|
|
11296
|
+
}
|
|
11297
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9778
11298
|
this.log({
|
|
9779
11299
|
level: "debug",
|
|
9780
11300
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9952,6 +11472,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9952
11472
|
}
|
|
9953
11473
|
this.dontRedispatch.delete(row.id);
|
|
9954
11474
|
this.untrackableAck.delete(row.id);
|
|
11475
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9955
11476
|
this.untrackableAckSignalled.delete(row.id);
|
|
9956
11477
|
this.untrackableAckWarned.delete(row.id);
|
|
9957
11478
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9971,8 +11492,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9971
11492
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9972
11493
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9973
11494
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9974
|
-
* re-entry)
|
|
9975
|
-
*
|
|
11495
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11496
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11497
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9976
11498
|
*
|
|
9977
11499
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9978
11500
|
* `processed_at` (Invariant 1).
|
|
@@ -10035,6 +11557,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10035
11557
|
);
|
|
10036
11558
|
}
|
|
10037
11559
|
let ocId;
|
|
11560
|
+
let preDispatchUserIds = null;
|
|
10038
11561
|
try {
|
|
10039
11562
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
10040
11563
|
sessionId,
|
|
@@ -10044,7 +11567,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10044
11567
|
row.content,
|
|
10045
11568
|
options,
|
|
10046
11569
|
sendAttachments,
|
|
10047
|
-
this.openCodeClient
|
|
11570
|
+
this.openCodeClient,
|
|
11571
|
+
{
|
|
11572
|
+
onKnownUserIds: (ids) => {
|
|
11573
|
+
preDispatchUserIds = ids;
|
|
11574
|
+
},
|
|
11575
|
+
onKnownUserIdsError: (error2) => {
|
|
11576
|
+
this.log({
|
|
11577
|
+
level: "warn",
|
|
11578
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11579
|
+
conversation_id: row.conversation_id,
|
|
11580
|
+
message_id: row.id
|
|
11581
|
+
});
|
|
11582
|
+
}
|
|
11583
|
+
}
|
|
10048
11584
|
)
|
|
10049
11585
|
);
|
|
10050
11586
|
} catch (err) {
|
|
@@ -10077,6 +11613,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10077
11613
|
}
|
|
10078
11614
|
} else {
|
|
10079
11615
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11616
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10080
11617
|
}
|
|
10081
11618
|
this.log({
|
|
10082
11619
|
level: "error",
|
|
@@ -10125,39 +11662,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10125
11662
|
}
|
|
10126
11663
|
if (ocId === null) {
|
|
10127
11664
|
this.awaitingReadopt.delete(row.id);
|
|
10128
|
-
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
|
|
10132
|
-
|
|
10133
|
-
|
|
10134
|
-
this.log({
|
|
10135
|
-
level: "error",
|
|
10136
|
-
message: errorMessage3,
|
|
10137
|
-
conversation_id: row.conversation_id,
|
|
10138
|
-
message_id: row.id
|
|
10139
|
-
});
|
|
10140
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10141
|
-
this.log({
|
|
10142
|
-
level: "warn",
|
|
10143
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10144
|
-
conversation_id: row.conversation_id,
|
|
10145
|
-
message_id: row.id
|
|
10146
|
-
});
|
|
10147
|
-
});
|
|
10148
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10149
|
-
return;
|
|
10150
|
-
}
|
|
11665
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11666
|
+
sessionId,
|
|
11667
|
+
content: row.content,
|
|
11668
|
+
attempts: 0,
|
|
11669
|
+
knownUserIds: preDispatchUserIds
|
|
11670
|
+
});
|
|
10151
11671
|
this.log({
|
|
10152
11672
|
level: "warn",
|
|
10153
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11673
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10154
11674
|
conversation_id: row.conversation_id,
|
|
10155
11675
|
message_id: row.id
|
|
10156
11676
|
});
|
|
10157
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10158
11677
|
return;
|
|
10159
11678
|
}
|
|
10160
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10161
11679
|
const processedAtMs = this.processedAtMs(row);
|
|
10162
11680
|
this.registerReadopted(
|
|
10163
11681
|
readoptConv,
|
|
@@ -10496,14 +12014,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10496
12014
|
const fetchChildWithoutBlocking = async (sessionId) => {
|
|
10497
12015
|
const pending = fetchChildMessages(sessionId);
|
|
10498
12016
|
let timer;
|
|
10499
|
-
const timeout = new Promise((
|
|
12017
|
+
const timeout = new Promise((resolve5) => {
|
|
10500
12018
|
timer = setTimeout(() => {
|
|
10501
12019
|
this.log({
|
|
10502
12020
|
level: "warn",
|
|
10503
12021
|
message: `Best-effort subagent session fetch for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} was slow \u2014 omitting invocation telemetry without delaying completion`,
|
|
10504
12022
|
message_id: messageId
|
|
10505
12023
|
});
|
|
10506
|
-
|
|
12024
|
+
resolve5(null);
|
|
10507
12025
|
}, SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS);
|
|
10508
12026
|
});
|
|
10509
12027
|
try {
|
|
@@ -11063,7 +12581,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11063
12581
|
* watcher retries next tick within the
|
|
11064
12582
|
* deadline, Finding 4).
|
|
11065
12583
|
*/
|
|
11066
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12584
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11067
12585
|
const res = await this.fetchImpl(
|
|
11068
12586
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11069
12587
|
{
|
|
@@ -11081,7 +12599,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11081
12599
|
...title ? { title } : {},
|
|
11082
12600
|
...usage ? usage : {},
|
|
11083
12601
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11084
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12602
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12603
|
+
...error2 ? { error: error2 } : {}
|
|
11085
12604
|
})
|
|
11086
12605
|
}
|
|
11087
12606
|
);
|
|
@@ -11128,7 +12647,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11128
12647
|
* exists but is wedged, so the next attempt must get a fresh one
|
|
11129
12648
|
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
11130
12649
|
*/
|
|
11131
|
-
async markFailed(conversationId, messageId, sessionId, error2, usage, failure, usageAgentName, subagentInvocations) {
|
|
12650
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage, failure, usageAgentName, subagentInvocations, abandonedWithoutAnswer) {
|
|
11132
12651
|
const body = { status: "failed" };
|
|
11133
12652
|
if (sessionId === null) {
|
|
11134
12653
|
body.opencode_session_id = null;
|
|
@@ -11141,12 +12660,19 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11141
12660
|
if (subagentInvocations && subagentInvocations.length > 0) {
|
|
11142
12661
|
body.subagent_invocations = subagentInvocations;
|
|
11143
12662
|
}
|
|
11144
|
-
if (failure) {
|
|
11145
|
-
body.
|
|
11146
|
-
|
|
11147
|
-
|
|
11148
|
-
body.
|
|
12663
|
+
if (failure && failure.errorShape.name !== null) {
|
|
12664
|
+
body.failure_error_name = failure.errorShape.name;
|
|
12665
|
+
}
|
|
12666
|
+
if (failure && failure.errorShape.statusCode !== null) {
|
|
12667
|
+
body.failure_status_code = failure.errorShape.statusCode;
|
|
11149
12668
|
}
|
|
12669
|
+
if (failure?.classification) {
|
|
12670
|
+
body.failure_kind = failure.classification.kind;
|
|
12671
|
+
body.failure_provider_id = failure.classification.providerId;
|
|
12672
|
+
body.failure_model_id = failure.classification.modelId;
|
|
12673
|
+
body.failure_reason = failure.classification.reason;
|
|
12674
|
+
}
|
|
12675
|
+
if (abandonedWithoutAnswer) body.abandoned_without_answer = true;
|
|
11150
12676
|
try {
|
|
11151
12677
|
await this.callWithRetry(
|
|
11152
12678
|
"marking message as failed",
|
|
@@ -11166,27 +12692,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11166
12692
|
this.clearSubagentInvocationCaches(messageId);
|
|
11167
12693
|
}
|
|
11168
12694
|
/**
|
|
11169
|
-
* Classify an errored turn
|
|
12695
|
+
* Classify an errored turn and extract its bounded error shape for `markFailed`.
|
|
11170
12696
|
*
|
|
11171
12697
|
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
11172
12698
|
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
11173
12699
|
* to the P1-2b zero-provider check — one extra loopback call to
|
|
11174
12700
|
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
11175
12701
|
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
11176
|
-
* (`null`/indeterminate) leaves the classification `null
|
|
11177
|
-
*
|
|
12702
|
+
* (`null`/indeterminate) leaves the classification `null`; the raw error shape
|
|
12703
|
+
* is still retained for diagnostics.
|
|
11178
12704
|
*/
|
|
11179
12705
|
async classifyModelAuthFailure(messages, userMessageId) {
|
|
11180
|
-
const classified = messageFailure(messages, userMessageId);
|
|
11181
|
-
if (classified != null) return classified;
|
|
11182
12706
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
11183
|
-
const
|
|
11184
|
-
|
|
12707
|
+
const errorShape = extractReplyErrorShape(reply);
|
|
12708
|
+
const classified = messageFailure(messages, userMessageId);
|
|
12709
|
+
if (classified != null) return { classification: classified, errorShape };
|
|
12710
|
+
const classification = applyZeroProviderFallback(
|
|
11185
12711
|
classified,
|
|
11186
|
-
|
|
12712
|
+
await this.hasAnyConfiguredProvider(),
|
|
11187
12713
|
reply?.info?.providerID ?? null,
|
|
11188
12714
|
reply?.info?.modelID ?? null
|
|
11189
12715
|
);
|
|
12716
|
+
return { classification, errorShape };
|
|
11190
12717
|
}
|
|
11191
12718
|
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
11192
12719
|
const providerId = failure.providerId ?? "(unknown)";
|
|
@@ -11744,7 +13271,7 @@ function commandError2(result) {
|
|
|
11744
13271
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
11745
13272
|
}
|
|
11746
13273
|
var runCommand2 = (command, args, opts) => {
|
|
11747
|
-
return new Promise((
|
|
13274
|
+
return new Promise((resolve5) => {
|
|
11748
13275
|
let child;
|
|
11749
13276
|
let stdout = "";
|
|
11750
13277
|
let stderr = "";
|
|
@@ -11754,7 +13281,7 @@ var runCommand2 = (command, args, opts) => {
|
|
|
11754
13281
|
if (settled) return;
|
|
11755
13282
|
settled = true;
|
|
11756
13283
|
if (timer.handle) clearTimeout(timer.handle);
|
|
11757
|
-
|
|
13284
|
+
resolve5(result);
|
|
11758
13285
|
};
|
|
11759
13286
|
try {
|
|
11760
13287
|
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
@@ -11809,7 +13336,9 @@ async function installRunnerSecret({
|
|
|
11809
13336
|
}) {
|
|
11810
13337
|
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
11811
13338
|
if (!arn) {
|
|
11812
|
-
|
|
13339
|
+
if (!env.GH_TOKEN) {
|
|
13340
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
13341
|
+
}
|
|
11813
13342
|
return false;
|
|
11814
13343
|
}
|
|
11815
13344
|
const result = await (commandRunner ?? runCommand2)(
|
|
@@ -12008,7 +13537,7 @@ async function configureGitHubAccess({ env, log: log3 }) {
|
|
|
12008
13537
|
// src/lib/opencode/config-overlay.ts
|
|
12009
13538
|
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
12010
13539
|
import { copyFileSync, existsSync as existsSync2, statSync as statSync6 } from "node:fs";
|
|
12011
|
-
import { isAbsolute as
|
|
13540
|
+
import { isAbsolute as isAbsolute3, join as join11, resolve as resolve4 } from "node:path";
|
|
12012
13541
|
function isFile(filePath) {
|
|
12013
13542
|
return existsSync2(filePath) && statSync6(filePath).isFile();
|
|
12014
13543
|
}
|
|
@@ -12021,8 +13550,8 @@ function applyRunnerOpenCodeConfig({
|
|
|
12021
13550
|
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
12022
13551
|
return;
|
|
12023
13552
|
}
|
|
12024
|
-
const source =
|
|
12025
|
-
const target = isFile(
|
|
13553
|
+
const source = isAbsolute3(overlayPath) ? overlayPath : resolve4(cwd, overlayPath);
|
|
13554
|
+
const target = isFile(join11(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
12026
13555
|
if (!isFile(source)) {
|
|
12027
13556
|
log3(
|
|
12028
13557
|
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
@@ -12030,7 +13559,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
12030
13559
|
);
|
|
12031
13560
|
return;
|
|
12032
13561
|
}
|
|
12033
|
-
copyFileSync(source,
|
|
13562
|
+
copyFileSync(source, join11(cwd, target));
|
|
12034
13563
|
try {
|
|
12035
13564
|
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
12036
13565
|
stdio: "ignore"
|
|
@@ -12039,7 +13568,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
12039
13568
|
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
12040
13569
|
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
12041
13570
|
}
|
|
12042
|
-
log3(`Applied runner OpenCode config ${source} to ${
|
|
13571
|
+
log3(`Applied runner OpenCode config ${source} to ${join11(cwd, target)}`);
|
|
12043
13572
|
}
|
|
12044
13573
|
|
|
12045
13574
|
// src/lib/credential-sync.ts
|
|
@@ -12057,14 +13586,14 @@ function errorMessage2(error2) {
|
|
|
12057
13586
|
return error2 instanceof Error ? error2.message : String(error2);
|
|
12058
13587
|
}
|
|
12059
13588
|
function waitForSettlement(promise, timeoutMs) {
|
|
12060
|
-
return new Promise((
|
|
13589
|
+
return new Promise((resolve5) => {
|
|
12061
13590
|
let settled = false;
|
|
12062
13591
|
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
12063
13592
|
const finish = (value) => {
|
|
12064
13593
|
if (settled) return;
|
|
12065
13594
|
settled = true;
|
|
12066
13595
|
clearTimeout(timer);
|
|
12067
|
-
|
|
13596
|
+
resolve5(value);
|
|
12068
13597
|
};
|
|
12069
13598
|
promise.then(
|
|
12070
13599
|
() => finish(true),
|
|
@@ -12281,6 +13810,8 @@ function createCredentialSync({
|
|
|
12281
13810
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
12282
13811
|
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12283
13812
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
13813
|
+
var CREDENTIAL_ADOPTION_PROBE_ATTEMPTS = 3;
|
|
13814
|
+
var CREDENTIAL_ADOPTION_PROBE_INTERVAL_MS = 1e3;
|
|
12284
13815
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12285
13816
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
12286
13817
|
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
@@ -12315,8 +13846,8 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
12315
13846
|
if (trimmed === "") {
|
|
12316
13847
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
12317
13848
|
}
|
|
12318
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
12319
|
-
if (!
|
|
13849
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join12(homeDir, trimmed.slice(2)) : trimmed;
|
|
13850
|
+
if (!isAbsolute4(expanded)) {
|
|
12320
13851
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
12321
13852
|
}
|
|
12322
13853
|
const normalized = resolvePath2(expanded);
|
|
@@ -12416,8 +13947,7 @@ function resolveMaxActiveSessions(options, env = process.env) {
|
|
|
12416
13947
|
function meetsThreshold(state, level) {
|
|
12417
13948
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
12418
13949
|
}
|
|
12419
|
-
function
|
|
12420
|
-
if (!meetsThreshold(state, level)) return;
|
|
13950
|
+
function writeConsoleLog(state, message, level) {
|
|
12421
13951
|
if (state.json) {
|
|
12422
13952
|
console.log(
|
|
12423
13953
|
JSON.stringify({
|
|
@@ -12426,10 +13956,18 @@ function log2(state, message, level = "info") {
|
|
|
12426
13956
|
message
|
|
12427
13957
|
})
|
|
12428
13958
|
);
|
|
12429
|
-
|
|
12430
|
-
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
12431
|
-
console.log(`${prefix} ${message}`);
|
|
13959
|
+
return;
|
|
12432
13960
|
}
|
|
13961
|
+
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
13962
|
+
console.log(`${prefix} ${message}`);
|
|
13963
|
+
}
|
|
13964
|
+
function log2(state, message, level = "info") {
|
|
13965
|
+
if (!meetsThreshold(state, level)) return;
|
|
13966
|
+
if (state.json || !state.interactive) writeConsoleLog(state, message, level);
|
|
13967
|
+
}
|
|
13968
|
+
function logLocalOnly(state, message, level = "info") {
|
|
13969
|
+
if (!meetsThreshold(state, level)) return;
|
|
13970
|
+
writeConsoleLog(state, message, level);
|
|
12433
13971
|
}
|
|
12434
13972
|
function logActivity(state, entry) {
|
|
12435
13973
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
@@ -12621,6 +14159,125 @@ async function handleAuthError(state, error2) {
|
|
|
12621
14159
|
return { success: false };
|
|
12622
14160
|
}
|
|
12623
14161
|
}
|
|
14162
|
+
function logCredentialAdoptionActivity(state, activity, level) {
|
|
14163
|
+
const message = (() => {
|
|
14164
|
+
switch (activity.outcome) {
|
|
14165
|
+
case "deferred":
|
|
14166
|
+
return `CREDENTIAL-ADOPTION: deferred generation ${activity.generation} with ${activity.active_sessions} active session(s)`;
|
|
14167
|
+
case "reloaded_ready":
|
|
14168
|
+
return `CREDENTIAL-ADOPTION: adopted generation ${activity.generation} with ${activity.active_sessions} active session(s); readiness probe succeeded after ${activity.probe_attempts} attempt(s)`;
|
|
14169
|
+
case "reloaded_interrupted": {
|
|
14170
|
+
const response = activity.probe_status === null ? "without an HTTP response" : `with HTTP ${activity.probe_status} and ${activity.probe_body_empty ? "an empty" : "a non-empty"} body`;
|
|
14171
|
+
return `CREDENTIAL-ADOPTION: generation ${activity.generation} with ${activity.active_sessions} active session(s); replacement instance answered ${response} after ${activity.probe_attempts} probe attempt(s)`;
|
|
14172
|
+
}
|
|
14173
|
+
case "reload_failed":
|
|
14174
|
+
return `CREDENTIAL-ADOPTION: failed to adopt generation ${activity.generation} with ${activity.active_sessions} active session(s)`;
|
|
14175
|
+
case "skipped_v2":
|
|
14176
|
+
return `CREDENTIAL-ADOPTION: skipped generation ${activity.generation} with ${activity.active_sessions} active session(s) on OpenCode V2`;
|
|
14177
|
+
}
|
|
14178
|
+
})();
|
|
14179
|
+
logActivity(state, {
|
|
14180
|
+
type: level === "error" ? "error" : "info",
|
|
14181
|
+
level,
|
|
14182
|
+
...level === "error" ? { error: message } : { message },
|
|
14183
|
+
metadata: {
|
|
14184
|
+
...activity,
|
|
14185
|
+
opencode_version: state.opencodeVersion,
|
|
14186
|
+
port: state.port
|
|
14187
|
+
}
|
|
14188
|
+
});
|
|
14189
|
+
}
|
|
14190
|
+
async function adoptCredentialGeneration(state, generation, adoption, activeSessions, deferredForMs) {
|
|
14191
|
+
const reload = await disposeAndReloadProviderConfig(
|
|
14192
|
+
state.port,
|
|
14193
|
+
state.opencodeClient ?? void 0
|
|
14194
|
+
);
|
|
14195
|
+
if (reload.outcome === "failed") {
|
|
14196
|
+
logCredentialAdoptionActivity(
|
|
14197
|
+
state,
|
|
14198
|
+
{
|
|
14199
|
+
outcome: "reload_failed",
|
|
14200
|
+
generation,
|
|
14201
|
+
active_sessions: activeSessions,
|
|
14202
|
+
adoption,
|
|
14203
|
+
deferred_for_ms: deferredForMs,
|
|
14204
|
+
reload_status: reload.status,
|
|
14205
|
+
reload_detail: reload.detail,
|
|
14206
|
+
probe_attempts: null,
|
|
14207
|
+
probe_status: null,
|
|
14208
|
+
probe_body_empty: null,
|
|
14209
|
+
probe_detail: null
|
|
14210
|
+
},
|
|
14211
|
+
"error"
|
|
14212
|
+
);
|
|
14213
|
+
return;
|
|
14214
|
+
}
|
|
14215
|
+
if (reload.outcome === "skipped_v2") {
|
|
14216
|
+
logCredentialAdoptionActivity(
|
|
14217
|
+
state,
|
|
14218
|
+
{
|
|
14219
|
+
outcome: "skipped_v2",
|
|
14220
|
+
generation,
|
|
14221
|
+
active_sessions: activeSessions,
|
|
14222
|
+
adoption,
|
|
14223
|
+
deferred_for_ms: deferredForMs,
|
|
14224
|
+
reload_status: null,
|
|
14225
|
+
reload_detail: null,
|
|
14226
|
+
probe_attempts: null,
|
|
14227
|
+
probe_status: null,
|
|
14228
|
+
probe_body_empty: null,
|
|
14229
|
+
probe_detail: null
|
|
14230
|
+
},
|
|
14231
|
+
"debug"
|
|
14232
|
+
);
|
|
14233
|
+
return;
|
|
14234
|
+
}
|
|
14235
|
+
let lastProbe = null;
|
|
14236
|
+
let attempts = 0;
|
|
14237
|
+
for (attempts = 1; attempts <= CREDENTIAL_ADOPTION_PROBE_ATTEMPTS; attempts++) {
|
|
14238
|
+
lastProbe = await probeOpenCodeRequestReadiness(state.port, state.opencodeClient ?? void 0);
|
|
14239
|
+
if (lastProbe.ok) {
|
|
14240
|
+
logCredentialAdoptionActivity(
|
|
14241
|
+
state,
|
|
14242
|
+
{
|
|
14243
|
+
outcome: "reloaded_ready",
|
|
14244
|
+
generation,
|
|
14245
|
+
active_sessions: activeSessions,
|
|
14246
|
+
adoption,
|
|
14247
|
+
deferred_for_ms: deferredForMs,
|
|
14248
|
+
reload_status: null,
|
|
14249
|
+
reload_detail: null,
|
|
14250
|
+
probe_attempts: attempts,
|
|
14251
|
+
probe_status: lastProbe.status,
|
|
14252
|
+
probe_body_empty: lastProbe.bodyEmpty,
|
|
14253
|
+
probe_detail: lastProbe.detail
|
|
14254
|
+
},
|
|
14255
|
+
"warn"
|
|
14256
|
+
);
|
|
14257
|
+
return;
|
|
14258
|
+
}
|
|
14259
|
+
if (attempts < CREDENTIAL_ADOPTION_PROBE_ATTEMPTS) {
|
|
14260
|
+
await new Promise((resolve5) => setTimeout(resolve5, CREDENTIAL_ADOPTION_PROBE_INTERVAL_MS));
|
|
14261
|
+
}
|
|
14262
|
+
}
|
|
14263
|
+
logCredentialAdoptionActivity(
|
|
14264
|
+
state,
|
|
14265
|
+
{
|
|
14266
|
+
outcome: "reloaded_interrupted",
|
|
14267
|
+
generation,
|
|
14268
|
+
active_sessions: activeSessions,
|
|
14269
|
+
adoption,
|
|
14270
|
+
deferred_for_ms: deferredForMs,
|
|
14271
|
+
reload_status: null,
|
|
14272
|
+
reload_detail: null,
|
|
14273
|
+
probe_attempts: CREDENTIAL_ADOPTION_PROBE_ATTEMPTS,
|
|
14274
|
+
probe_status: lastProbe?.status ?? null,
|
|
14275
|
+
probe_body_empty: lastProbe?.bodyEmpty ?? null,
|
|
14276
|
+
probe_detail: lastProbe?.detail ?? null
|
|
14277
|
+
},
|
|
14278
|
+
"error"
|
|
14279
|
+
);
|
|
14280
|
+
}
|
|
12624
14281
|
async function driveChannels(state, driver) {
|
|
12625
14282
|
let idlePolls = 0;
|
|
12626
14283
|
let idleMs = 0;
|
|
@@ -12630,6 +14287,15 @@ async function driveChannels(state, driver) {
|
|
|
12630
14287
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
12631
14288
|
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
12632
14289
|
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
14290
|
+
const credentialAdoption = {
|
|
14291
|
+
generation: 0,
|
|
14292
|
+
pendingGeneration: null,
|
|
14293
|
+
pendingSinceMs: null,
|
|
14294
|
+
deferredGeneration: null,
|
|
14295
|
+
adoption: 0
|
|
14296
|
+
};
|
|
14297
|
+
const stayAwakeHolds = () => state.stayAwakeUntilMs !== null && Date.now() < state.stayAwakeUntilMs;
|
|
14298
|
+
let stayAwakeSuppressionLogged = false;
|
|
12633
14299
|
while (state.running) {
|
|
12634
14300
|
const cycleStartedAtMs = performance.now();
|
|
12635
14301
|
let idleThisCycle = false;
|
|
@@ -12671,12 +14337,30 @@ async function driveChannels(state, driver) {
|
|
|
12671
14337
|
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
12672
14338
|
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
12673
14339
|
if (claudeCredentialApplied || opencodeAuthApplied) {
|
|
12674
|
-
|
|
12675
|
-
|
|
12676
|
-
|
|
12677
|
-
|
|
12678
|
-
|
|
12679
|
-
|
|
14340
|
+
credentialAdoption.generation++;
|
|
14341
|
+
if (state.opencodeApiVersion === "v2") {
|
|
14342
|
+
logCredentialAdoptionActivity(
|
|
14343
|
+
state,
|
|
14344
|
+
{
|
|
14345
|
+
outcome: "skipped_v2",
|
|
14346
|
+
generation: credentialAdoption.generation,
|
|
14347
|
+
active_sessions: driver.protectedSessionIds().size,
|
|
14348
|
+
adoption: null,
|
|
14349
|
+
deferred_for_ms: null,
|
|
14350
|
+
reload_status: null,
|
|
14351
|
+
reload_detail: null,
|
|
14352
|
+
probe_attempts: null,
|
|
14353
|
+
probe_status: null,
|
|
14354
|
+
probe_body_empty: null,
|
|
14355
|
+
probe_detail: null
|
|
14356
|
+
},
|
|
14357
|
+
"debug"
|
|
14358
|
+
);
|
|
14359
|
+
} else {
|
|
14360
|
+
credentialAdoption.pendingGeneration = credentialAdoption.generation;
|
|
14361
|
+
credentialAdoption.pendingSinceMs = Date.now();
|
|
14362
|
+
credentialAdoption.deferredGeneration = null;
|
|
14363
|
+
}
|
|
12680
14364
|
}
|
|
12681
14365
|
if (opencodeAuthApplied && state.opencodeApiVersion === "v2") {
|
|
12682
14366
|
void importOpenCodeV2CredentialsForState(state).catch(
|
|
@@ -12731,7 +14415,50 @@ async function driveChannels(state, driver) {
|
|
|
12731
14415
|
}
|
|
12732
14416
|
}
|
|
12733
14417
|
}
|
|
12734
|
-
|
|
14418
|
+
if (credentialAdoption.pendingGeneration !== null) {
|
|
14419
|
+
const activeSessions = driver.protectedSessionIds();
|
|
14420
|
+
const generation = credentialAdoption.pendingGeneration;
|
|
14421
|
+
if (activeSessions.size > 0) {
|
|
14422
|
+
if (credentialAdoption.deferredGeneration !== generation) {
|
|
14423
|
+
credentialAdoption.deferredGeneration = generation;
|
|
14424
|
+
logCredentialAdoptionActivity(
|
|
14425
|
+
state,
|
|
14426
|
+
{
|
|
14427
|
+
outcome: "deferred",
|
|
14428
|
+
generation,
|
|
14429
|
+
active_sessions: activeSessions.size,
|
|
14430
|
+
adoption: null,
|
|
14431
|
+
deferred_for_ms: null,
|
|
14432
|
+
reload_status: null,
|
|
14433
|
+
reload_detail: null,
|
|
14434
|
+
probe_attempts: null,
|
|
14435
|
+
probe_status: null,
|
|
14436
|
+
probe_body_empty: null,
|
|
14437
|
+
probe_detail: null
|
|
14438
|
+
},
|
|
14439
|
+
"warn"
|
|
14440
|
+
);
|
|
14441
|
+
}
|
|
14442
|
+
} else {
|
|
14443
|
+
const deferredForMs = credentialAdoption.deferredGeneration === generation && credentialAdoption.pendingSinceMs !== null ? Math.max(0, Date.now() - credentialAdoption.pendingSinceMs) : null;
|
|
14444
|
+
credentialAdoption.pendingGeneration = null;
|
|
14445
|
+
credentialAdoption.pendingSinceMs = null;
|
|
14446
|
+
credentialAdoption.adoption++;
|
|
14447
|
+
void adoptCredentialGeneration(
|
|
14448
|
+
state,
|
|
14449
|
+
generation,
|
|
14450
|
+
credentialAdoption.adoption,
|
|
14451
|
+
activeSessions.size,
|
|
14452
|
+
deferredForMs
|
|
14453
|
+
).catch(
|
|
14454
|
+
(error2) => logActivity(state, {
|
|
14455
|
+
type: "error",
|
|
14456
|
+
error: `CREDENTIAL-ADOPTION: adoption failed for generation ${generation} with ${activeSessions.size} active session(s) on port ${state.port}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
14457
|
+
})
|
|
14458
|
+
);
|
|
14459
|
+
}
|
|
14460
|
+
}
|
|
14461
|
+
await new Promise((resolve5) => setTimeout(resolve5, CHANNEL_POLL_INTERVAL_MS));
|
|
12735
14462
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
12736
14463
|
if (idleThisCycle) idleMs += cycleMs;
|
|
12737
14464
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -12744,17 +14471,29 @@ async function driveChannels(state, driver) {
|
|
|
12744
14471
|
if (state.interactive) displayStatus(state);
|
|
12745
14472
|
break;
|
|
12746
14473
|
}
|
|
12747
|
-
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
|
|
14474
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && !stayAwakeHolds()) {
|
|
12748
14475
|
logActivity(state, { type: "info", message: "Idle timeout reached" });
|
|
12749
14476
|
if (state.interactive) displayStatus(state);
|
|
12750
14477
|
break;
|
|
12751
14478
|
}
|
|
14479
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && stayAwakeHolds()) {
|
|
14480
|
+
if (!stayAwakeSuppressionLogged) {
|
|
14481
|
+
stayAwakeSuppressionLogged = true;
|
|
14482
|
+
logActivity(state, {
|
|
14483
|
+
type: "info",
|
|
14484
|
+
message: "Idle, but staying awake until the requested minimum passes"
|
|
14485
|
+
});
|
|
14486
|
+
}
|
|
14487
|
+
}
|
|
12752
14488
|
}
|
|
12753
14489
|
}
|
|
12754
14490
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12755
|
-
var
|
|
14491
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
14492
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
14493
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
14494
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
12756
14495
|
function shouldWarnForReclaimSkip(reason) {
|
|
12757
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
14496
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12758
14497
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12759
14498
|
if (!version2) return false;
|
|
12760
14499
|
const major = Number(version2[1]);
|
|
@@ -12763,7 +14502,7 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12763
14502
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12764
14503
|
}
|
|
12765
14504
|
function sessionDbPath() {
|
|
12766
|
-
return
|
|
14505
|
+
return join12(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
12767
14506
|
}
|
|
12768
14507
|
function logSessionCleanupActivity(state, phase, entry) {
|
|
12769
14508
|
logActivity(state, {
|
|
@@ -12811,61 +14550,146 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12811
14550
|
async function runSweep(state, driver, config) {
|
|
12812
14551
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12813
14552
|
try {
|
|
12814
|
-
const
|
|
12815
|
-
|
|
12816
|
-
|
|
14553
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
14554
|
+
const sessions = enumeration.sessions;
|
|
14555
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
14556
|
+
if (enumerationReason !== null) {
|
|
14557
|
+
logActivity(state, {
|
|
12817
14558
|
type: "info",
|
|
12818
|
-
|
|
12819
|
-
|
|
12820
|
-
outcome: "sweep-skipped",
|
|
12821
|
-
reason: "opencode-unreachable"
|
|
12822
|
-
}
|
|
14559
|
+
level: "warn",
|
|
14560
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12823
14561
|
});
|
|
12824
|
-
return;
|
|
12825
14562
|
}
|
|
12826
|
-
const toDelete = selectSessionsToDelete(
|
|
12827
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12828
|
-
{
|
|
12829
|
-
maxAgeMs: config.maxAgeMs,
|
|
12830
|
-
maxCount: config.maxCount,
|
|
12831
|
-
nowMs: Date.now(),
|
|
12832
|
-
protectedIds: driver.protectedSessionIds()
|
|
12833
|
-
}
|
|
12834
|
-
);
|
|
12835
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12836
14563
|
let deleted = 0;
|
|
12837
14564
|
let failed = 0;
|
|
14565
|
+
let failed4xx = 0;
|
|
14566
|
+
let failed5xx = 0;
|
|
14567
|
+
let failedOtherStatus = 0;
|
|
14568
|
+
let failedThrew = 0;
|
|
12838
14569
|
let skippedNewlyActive = 0;
|
|
12839
|
-
|
|
12840
|
-
|
|
12841
|
-
|
|
12842
|
-
|
|
14570
|
+
let remaining = 0;
|
|
14571
|
+
let protectedNow = driver.protectedSessionIds();
|
|
14572
|
+
if (sessions !== null) {
|
|
14573
|
+
const toDelete = selectSessionsToDelete(
|
|
14574
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
14575
|
+
{
|
|
14576
|
+
maxAgeMs: config.maxAgeMs,
|
|
14577
|
+
maxCount: config.maxCount,
|
|
14578
|
+
nowMs: Date.now(),
|
|
14579
|
+
protectedIds: protectedNow
|
|
14580
|
+
}
|
|
14581
|
+
);
|
|
14582
|
+
const eligibleToDelete = toDelete.filter(
|
|
14583
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
14584
|
+
);
|
|
14585
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
14586
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
14587
|
+
const answeredFailures = [];
|
|
14588
|
+
protectedNow = driver.protectedSessionIds();
|
|
14589
|
+
for (const id of deleteCandidates) {
|
|
14590
|
+
if (protectedNow.has(id)) {
|
|
14591
|
+
skippedNewlyActive++;
|
|
14592
|
+
logActivity(state, {
|
|
14593
|
+
type: "info",
|
|
14594
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
14595
|
+
});
|
|
14596
|
+
continue;
|
|
14597
|
+
}
|
|
14598
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
14599
|
+
if (outcome.ok) {
|
|
14600
|
+
deleted++;
|
|
14601
|
+
} else if (outcome.status === "threw") {
|
|
14602
|
+
failed++;
|
|
14603
|
+
failedThrew++;
|
|
14604
|
+
} else {
|
|
14605
|
+
failed++;
|
|
14606
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
14607
|
+
if (statusClass === 4) failed4xx++;
|
|
14608
|
+
else if (statusClass === 5) failed5xx++;
|
|
14609
|
+
else failedOtherStatus++;
|
|
14610
|
+
answeredFailures.push({
|
|
14611
|
+
id,
|
|
14612
|
+
status: outcome.status,
|
|
14613
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
14614
|
+
});
|
|
14615
|
+
}
|
|
14616
|
+
}
|
|
14617
|
+
const givenUpFailures = [];
|
|
14618
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
14619
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
14620
|
+
} else {
|
|
14621
|
+
for (const failure of answeredFailures) {
|
|
14622
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
14623
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
14624
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
14625
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
14626
|
+
givenUpFailures.push(failure);
|
|
14627
|
+
}
|
|
14628
|
+
}
|
|
14629
|
+
}
|
|
14630
|
+
for (const failure of givenUpFailures) {
|
|
14631
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
14632
|
+
0,
|
|
14633
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
14634
|
+
);
|
|
14635
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12843
14636
|
type: "info",
|
|
12844
|
-
|
|
14637
|
+
level: "warn",
|
|
14638
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
14639
|
+
metadata: {
|
|
14640
|
+
outcome: "session-delete-given-up",
|
|
14641
|
+
session_id: failure.id,
|
|
14642
|
+
status: failure.status,
|
|
14643
|
+
body_excerpt: excerpt,
|
|
14644
|
+
given_up_count: givenUpFailures.length
|
|
14645
|
+
}
|
|
12845
14646
|
});
|
|
12846
|
-
continue;
|
|
12847
14647
|
}
|
|
12848
|
-
|
|
12849
|
-
|
|
14648
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
14649
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
14650
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
14651
|
+
logActivity(state, {
|
|
14652
|
+
type: "info",
|
|
14653
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
14654
|
+
});
|
|
12850
14655
|
}
|
|
12851
|
-
const
|
|
12852
|
-
|
|
12853
|
-
|
|
12854
|
-
|
|
12855
|
-
|
|
12856
|
-
|
|
14656
|
+
const sessionMetadata = {
|
|
14657
|
+
sessions_inspected: sessions?.length ?? null,
|
|
14658
|
+
sessions_deleted: deleted,
|
|
14659
|
+
sessions_failed: failed,
|
|
14660
|
+
sessions_failed_4xx: failed4xx,
|
|
14661
|
+
sessions_failed_5xx: failed5xx,
|
|
14662
|
+
sessions_failed_other: failedOtherStatus,
|
|
14663
|
+
sessions_failed_threw: failedThrew,
|
|
14664
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
14665
|
+
sessions_remaining: remaining,
|
|
14666
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
14667
|
+
};
|
|
14668
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
12857
14669
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12858
14670
|
dbPath: sessionDbPath(),
|
|
12859
|
-
|
|
12860
|
-
|
|
14671
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
14672
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
14673
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
14674
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14675
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
12861
14676
|
});
|
|
12862
14677
|
if (reclaimResult.ok) {
|
|
14678
|
+
state.sessionStoreHealth.observeWritable();
|
|
12863
14679
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12864
14680
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12865
14681
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
14682
|
+
const loopElapsedMs = Math.max(
|
|
14683
|
+
0,
|
|
14684
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14685
|
+
);
|
|
14686
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14687
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14688
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
12866
14689
|
logSessionCleanupActivity(state, "sweep", {
|
|
12867
14690
|
type: "info",
|
|
12868
|
-
|
|
14691
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14692
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
12869
14693
|
metadata: {
|
|
12870
14694
|
outcome: "reclaimed",
|
|
12871
14695
|
mode: reclaimResult.mode,
|
|
@@ -12873,25 +14697,28 @@ async function runSweep(state, driver, config) {
|
|
|
12873
14697
|
after_bytes: reclaimResult.afterBytes,
|
|
12874
14698
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12875
14699
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
12879
|
-
|
|
14700
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
14701
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
14702
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
14703
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
14704
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14705
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
14706
|
+
...sessionMetadata
|
|
12880
14707
|
}
|
|
12881
14708
|
});
|
|
12882
14709
|
} else {
|
|
14710
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
14711
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
14712
|
+
}
|
|
12883
14713
|
logSessionCleanupActivity(state, "sweep", {
|
|
12884
14714
|
type: "info",
|
|
12885
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
14715
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12886
14716
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12887
14717
|
metadata: {
|
|
12888
14718
|
outcome: "reclaim-skipped",
|
|
12889
14719
|
reason: reclaimResult.skipped,
|
|
12890
14720
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12891
|
-
|
|
12892
|
-
sessions_deleted: deleted,
|
|
12893
|
-
sessions_failed: failed,
|
|
12894
|
-
sessions_skipped_active: skippedNewlyActive
|
|
14721
|
+
...sessionMetadata
|
|
12895
14722
|
}
|
|
12896
14723
|
});
|
|
12897
14724
|
}
|
|
@@ -12986,10 +14813,12 @@ function scheduleUsageReporting(state, params) {
|
|
|
12986
14813
|
let phase = "dormant";
|
|
12987
14814
|
let rearmRequested = false;
|
|
12988
14815
|
const armProbe = () => {
|
|
14816
|
+
if (!state.running) return;
|
|
12989
14817
|
phase = "probe-pending";
|
|
12990
14818
|
params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
|
|
12991
14819
|
};
|
|
12992
14820
|
const scheduleNextTick = () => {
|
|
14821
|
+
if (!state.running) return;
|
|
12993
14822
|
if (rearmRequested) {
|
|
12994
14823
|
rearmRequested = false;
|
|
12995
14824
|
armProbe();
|
|
@@ -13023,6 +14852,7 @@ function scheduleUsageReporting(state, params) {
|
|
|
13023
14852
|
phase = "tick-in-flight";
|
|
13024
14853
|
try {
|
|
13025
14854
|
const usage = await params.fetchUsage();
|
|
14855
|
+
if (!state.running) return;
|
|
13026
14856
|
const result = await params.report(usage);
|
|
13027
14857
|
if (result.ok) {
|
|
13028
14858
|
if (consecutiveFailures > 0) {
|
|
@@ -13148,6 +14978,20 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13148
14978
|
message: `Resource usage collection: ${warning2}`
|
|
13149
14979
|
});
|
|
13150
14980
|
}
|
|
14981
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14982
|
+
...usage,
|
|
14983
|
+
nowMs: Date.now(),
|
|
14984
|
+
state: state.lowDiskWarningState
|
|
14985
|
+
});
|
|
14986
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14987
|
+
if (lowDiskDecision.warning !== null) {
|
|
14988
|
+
logActivity(state, {
|
|
14989
|
+
type: "info",
|
|
14990
|
+
level: "warn",
|
|
14991
|
+
message: lowDiskDecision.warning
|
|
14992
|
+
});
|
|
14993
|
+
}
|
|
14994
|
+
if (!state.running) return;
|
|
13151
14995
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13152
14996
|
if (result.ok) {
|
|
13153
14997
|
if (consecutiveFailures > 0) {
|
|
@@ -13180,7 +15024,9 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13180
15024
|
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
13181
15025
|
});
|
|
13182
15026
|
} finally {
|
|
13183
|
-
state.
|
|
15027
|
+
if (state.running) {
|
|
15028
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), usageReportDelayMs());
|
|
15029
|
+
}
|
|
13184
15030
|
}
|
|
13185
15031
|
};
|
|
13186
15032
|
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
@@ -13342,8 +15188,8 @@ async function beginGracefulShutdown(state, trigger) {
|
|
|
13342
15188
|
return true;
|
|
13343
15189
|
}
|
|
13344
15190
|
);
|
|
13345
|
-
const timedOut = new Promise((
|
|
13346
|
-
timer = setTimeout(() =>
|
|
15191
|
+
const timedOut = new Promise((resolve5) => {
|
|
15192
|
+
timer = setTimeout(() => resolve5(false), telemetryBudgetMs);
|
|
13347
15193
|
});
|
|
13348
15194
|
if (!await Promise.race([flushed, timedOut])) {
|
|
13349
15195
|
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
@@ -13383,6 +15229,8 @@ async function run(options) {
|
|
|
13383
15229
|
port: options.port ?? 4096,
|
|
13384
15230
|
conversationFilter: options.conversation ?? null,
|
|
13385
15231
|
idleTimeout: options.idleTimeout ?? null,
|
|
15232
|
+
stayAwakeSeconds: null,
|
|
15233
|
+
stayAwakeUntilMs: null,
|
|
13386
15234
|
json: options.json ?? false,
|
|
13387
15235
|
interactive,
|
|
13388
15236
|
logLevel,
|
|
@@ -13394,6 +15242,7 @@ async function run(options) {
|
|
|
13394
15242
|
opencodeClient: null,
|
|
13395
15243
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13396
15244
|
sessionDbProvenanceAnomaly: false,
|
|
15245
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13397
15246
|
opencodeProcess: null,
|
|
13398
15247
|
stopOpenCodeLogTail: null,
|
|
13399
15248
|
litestreamProcess: null,
|
|
@@ -13405,12 +15254,14 @@ async function run(options) {
|
|
|
13405
15254
|
messageCount: 0,
|
|
13406
15255
|
lastProxiedActivityAt: null,
|
|
13407
15256
|
sessionCleanupTimers: [],
|
|
15257
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13408
15258
|
claudeUsageTimer: null,
|
|
13409
15259
|
claudeUsageRearm: null,
|
|
13410
15260
|
openaiUsageTimer: null,
|
|
13411
15261
|
openaiUsageRearm: null,
|
|
13412
15262
|
resourceUsageTimer: null,
|
|
13413
15263
|
stopResourceUsageSampling: null,
|
|
15264
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13414
15265
|
credentialSync: null,
|
|
13415
15266
|
authHeader: ""
|
|
13416
15267
|
};
|
|
@@ -13569,6 +15420,17 @@ async function run(options) {
|
|
|
13569
15420
|
}
|
|
13570
15421
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
13571
15422
|
state.agentName = validation.agent.name;
|
|
15423
|
+
const requestedStayAwakeSeconds = validation.agent?.wake_stay_awake_seconds;
|
|
15424
|
+
if (typeof requestedStayAwakeSeconds === "number" && Number.isFinite(requestedStayAwakeSeconds) && requestedStayAwakeSeconds > 0) {
|
|
15425
|
+
state.stayAwakeSeconds = requestedStayAwakeSeconds;
|
|
15426
|
+
} else if (requestedStayAwakeSeconds !== void 0 && requestedStayAwakeSeconds !== null) {
|
|
15427
|
+
state.stayAwakeSeconds = null;
|
|
15428
|
+
log2(
|
|
15429
|
+
state,
|
|
15430
|
+
`Ignoring invalid wake_stay_awake_seconds (type: ${typeof requestedStayAwakeSeconds})`,
|
|
15431
|
+
"warn"
|
|
15432
|
+
);
|
|
15433
|
+
}
|
|
13572
15434
|
const microvmId = process.env.MICROVM_ID?.trim();
|
|
13573
15435
|
if (microvmId) {
|
|
13574
15436
|
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
@@ -13584,27 +15446,74 @@ async function run(options) {
|
|
|
13584
15446
|
} else {
|
|
13585
15447
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13586
15448
|
}
|
|
13587
|
-
if (options.restoreRunnerCredentials) {
|
|
13588
|
-
|
|
13589
|
-
|
|
13590
|
-
|
|
13591
|
-
|
|
13592
|
-
if (level === "error") {
|
|
13593
|
-
logActivity(state, { type: "error", error: message });
|
|
13594
|
-
} else {
|
|
13595
|
-
logActivity(state, { type: "info", level, message });
|
|
13596
|
-
}
|
|
15449
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
15450
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
15451
|
+
log2(state, message, level);
|
|
15452
|
+
if (state.interactive && !state.json) {
|
|
15453
|
+
logActivity(state, { type: "info", level, message });
|
|
13597
15454
|
}
|
|
13598
15455
|
};
|
|
13599
|
-
const
|
|
15456
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
15457
|
+
if (vaultResolution.error) {
|
|
15458
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15459
|
+
logCredentialVaultResolution(
|
|
15460
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
15461
|
+
"warn"
|
|
15462
|
+
);
|
|
15463
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
15464
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
15465
|
+
logCredentialVaultResolution(
|
|
15466
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
15467
|
+
"info"
|
|
15468
|
+
);
|
|
15469
|
+
} else {
|
|
15470
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15471
|
+
logCredentialVaultResolution(
|
|
15472
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
15473
|
+
"debug"
|
|
15474
|
+
);
|
|
15475
|
+
}
|
|
15476
|
+
}
|
|
15477
|
+
const credentialContext = {
|
|
15478
|
+
env: process.env,
|
|
15479
|
+
log: (message, level = "info") => {
|
|
15480
|
+
if (level === "error") {
|
|
15481
|
+
logActivity(state, { type: "error", error: message });
|
|
15482
|
+
} else {
|
|
15483
|
+
logActivity(state, { type: "info", level, message });
|
|
15484
|
+
}
|
|
15485
|
+
}
|
|
15486
|
+
};
|
|
15487
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
15488
|
+
if (options.restoreRunnerCredentials) {
|
|
13600
15489
|
await restoreCredentialStores(credentialContext);
|
|
13601
|
-
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13602
15490
|
}
|
|
15491
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13603
15492
|
state.credentialSync?.arm();
|
|
13604
|
-
const
|
|
13605
|
-
|
|
13606
|
-
|
|
13607
|
-
|
|
15493
|
+
const dependencyInstallDiagnosticReporter = createDependencyInstallDiagnosticReporter({
|
|
15494
|
+
context: {
|
|
15495
|
+
homeDir: homedir7(),
|
|
15496
|
+
cwd: process.cwd(),
|
|
15497
|
+
microvmId: process.env.MICROVM_ID?.trim(),
|
|
15498
|
+
euid: process.getuid?.(),
|
|
15499
|
+
egid: process.getgid?.()
|
|
15500
|
+
},
|
|
15501
|
+
emitEvent: (message) => logActivity(state, { type: "info", level: "warn", message }),
|
|
15502
|
+
emitLocal: (message) => logLocalOnly(state, message, "warn")
|
|
15503
|
+
});
|
|
15504
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
15505
|
+
() => ({
|
|
15506
|
+
agentId: state.agentId,
|
|
15507
|
+
authHeader: state.authHeader
|
|
15508
|
+
}),
|
|
15509
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
15510
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
15511
|
+
if (cause !== null && sessionId !== null) {
|
|
15512
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
15513
|
+
}
|
|
15514
|
+
},
|
|
15515
|
+
dependencyInstallDiagnosticReporter
|
|
15516
|
+
);
|
|
13608
15517
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13609
15518
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13610
15519
|
openCodeActivityForwarder
|
|
@@ -13614,22 +15523,27 @@ async function run(options) {
|
|
|
13614
15523
|
openCodeLogTail.stop();
|
|
13615
15524
|
};
|
|
13616
15525
|
let sessionDbVerifyFatal = false;
|
|
15526
|
+
let sessionDbVerifyFatalReason;
|
|
13617
15527
|
if (!options.restoreSessionDb) {
|
|
13618
|
-
|
|
15528
|
+
logActivity(state, {
|
|
15529
|
+
type: "info",
|
|
15530
|
+
level: "warn",
|
|
15531
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
15532
|
+
});
|
|
13619
15533
|
} else {
|
|
13620
15534
|
const health = await checkOpenCodeHealth(state.port);
|
|
13621
15535
|
if (health.healthy) {
|
|
13622
|
-
|
|
13623
|
-
|
|
13624
|
-
|
|
13625
|
-
"
|
|
13626
|
-
);
|
|
15536
|
+
logActivity(state, {
|
|
15537
|
+
type: "info",
|
|
15538
|
+
level: "warn",
|
|
15539
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
15540
|
+
});
|
|
13627
15541
|
} else if (health.authFailed) {
|
|
13628
|
-
|
|
13629
|
-
|
|
13630
|
-
|
|
13631
|
-
"
|
|
13632
|
-
);
|
|
15542
|
+
logActivity(state, {
|
|
15543
|
+
type: "info",
|
|
15544
|
+
level: "warn",
|
|
15545
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
15546
|
+
});
|
|
13633
15547
|
} else {
|
|
13634
15548
|
const result = await restoreAndVerifySessionDb({
|
|
13635
15549
|
dbPath: sessionDbPath(),
|
|
@@ -13646,12 +15560,13 @@ async function run(options) {
|
|
|
13646
15560
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13647
15561
|
});
|
|
13648
15562
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
15563
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13649
15564
|
}
|
|
13650
15565
|
}
|
|
13651
15566
|
reportSessionDbRecovery(state);
|
|
13652
15567
|
if (sessionDbVerifyFatal) {
|
|
13653
15568
|
throw new Error(
|
|
13654
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
15569
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13655
15570
|
);
|
|
13656
15571
|
}
|
|
13657
15572
|
applyRunnerOpenCodeConfig({
|
|
@@ -13759,7 +15674,7 @@ async function run(options) {
|
|
|
13759
15674
|
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
13760
15675
|
}
|
|
13761
15676
|
}
|
|
13762
|
-
await
|
|
15677
|
+
await disposeAndReloadProviderConfig(state.port, state.opencodeClient ?? void 0);
|
|
13763
15678
|
if (state.opencodeApiVersion === "v2" && state.opencodeClient !== null) {
|
|
13764
15679
|
await importOpenCodeV2CredentialsForState(state);
|
|
13765
15680
|
}
|
|
@@ -13896,6 +15811,7 @@ async function run(options) {
|
|
|
13896
15811
|
getAuthHeader: () => state.authHeader,
|
|
13897
15812
|
conversationFilter: state.conversationFilter,
|
|
13898
15813
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15814
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13899
15815
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13900
15816
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13901
15817
|
fileSyncDirectories,
|
|
@@ -13929,6 +15845,13 @@ async function run(options) {
|
|
|
13929
15845
|
type: "info",
|
|
13930
15846
|
message: connectionMessage
|
|
13931
15847
|
});
|
|
15848
|
+
if (!isReconnect && state.stayAwakeSeconds !== null && state.stayAwakeUntilMs === null) {
|
|
15849
|
+
state.stayAwakeUntilMs = Date.now() + state.stayAwakeSeconds * 1e3;
|
|
15850
|
+
logActivity(state, {
|
|
15851
|
+
type: "info",
|
|
15852
|
+
message: `Staying awake for at least ${state.stayAwakeSeconds}s as requested by the wake`
|
|
15853
|
+
});
|
|
15854
|
+
}
|
|
13932
15855
|
if (options.tunnelReadyFile) {
|
|
13933
15856
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
13934
15857
|
if (marker.ok) {
|
|
@@ -13977,14 +15900,14 @@ async function run(options) {
|
|
|
13977
15900
|
},
|
|
13978
15901
|
onDisconnected: (code, reason, transition) => {
|
|
13979
15902
|
state.connected = false;
|
|
13980
|
-
const
|
|
15903
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13981
15904
|
logActivity(state, {
|
|
13982
15905
|
type: "info",
|
|
13983
|
-
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${
|
|
15906
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
13984
15907
|
});
|
|
13985
15908
|
emitAgentDisconnected(state.agentId, {
|
|
13986
15909
|
code,
|
|
13987
|
-
reason:
|
|
15910
|
+
reason: safeReason2,
|
|
13988
15911
|
connection_generation: transition.generation
|
|
13989
15912
|
});
|
|
13990
15913
|
if (state.interactive) displayStatus(state);
|
|
@@ -14063,6 +15986,7 @@ async function run(options) {
|
|
|
14063
15986
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
14064
15987
|
throw error2;
|
|
14065
15988
|
}
|
|
15989
|
+
if (!state.running) return;
|
|
14066
15990
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
14067
15991
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
14068
15992
|
state.openaiUsageRearm = scheduleUsageReporting(state, {
|
|
@@ -14129,7 +16053,7 @@ async function run(options) {
|
|
|
14129
16053
|
}
|
|
14130
16054
|
|
|
14131
16055
|
// src/index.ts
|
|
14132
|
-
var
|
|
16056
|
+
var version = getCliVersion();
|
|
14133
16057
|
var program = new Command();
|
|
14134
16058
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
14135
16059
|
"--endpoint <url>",
|
|
@@ -14147,6 +16071,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14147
16071
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14148
16072
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14149
16073
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
16074
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14150
16075
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14151
16076
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14152
16077
|
"-a, --agent [id]",
|