@evident-ai/cli 3.5.2-dev.4181600 → 3.5.2-dev.4763a76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/index.js +2580 -603
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -316,14 +315,14 @@ function blank() {
|
|
|
316
315
|
console.log();
|
|
317
316
|
}
|
|
318
317
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
319
|
-
return new Promise((
|
|
318
|
+
return new Promise((resolve5) => {
|
|
320
319
|
process.stdout.write(chalk.dim(prompt));
|
|
321
320
|
const handler = () => {
|
|
322
321
|
process.stdin.removeListener("data", handler);
|
|
323
322
|
process.stdin.setRawMode?.(false);
|
|
324
323
|
process.stdin.pause();
|
|
325
324
|
console.log();
|
|
326
|
-
|
|
325
|
+
resolve5();
|
|
327
326
|
};
|
|
328
327
|
if (process.stdin.isTTY) {
|
|
329
328
|
process.stdin.setRawMode?.(true);
|
|
@@ -333,7 +332,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
333
332
|
});
|
|
334
333
|
}
|
|
335
334
|
function sleep(ms) {
|
|
336
|
-
return new Promise((
|
|
335
|
+
return new Promise((resolve5) => setTimeout(resolve5, ms));
|
|
337
336
|
}
|
|
338
337
|
|
|
339
338
|
// src/commands/login.ts
|
|
@@ -411,19 +410,19 @@ async function tokenLogin() {
|
|
|
411
410
|
);
|
|
412
411
|
blank();
|
|
413
412
|
process.stdout.write("Paste token: ");
|
|
414
|
-
const token = await new Promise((
|
|
413
|
+
const token = await new Promise((resolve5) => {
|
|
415
414
|
let data = "";
|
|
416
415
|
process.stdin.setEncoding("utf8");
|
|
417
416
|
process.stdin.on("data", (chunk) => {
|
|
418
417
|
data += chunk;
|
|
419
418
|
});
|
|
420
419
|
process.stdin.on("end", () => {
|
|
421
|
-
|
|
420
|
+
resolve5(data.trim());
|
|
422
421
|
});
|
|
423
422
|
if (process.stdin.isTTY) {
|
|
424
423
|
process.stdin.once("data", (chunk) => {
|
|
425
424
|
process.stdin.pause();
|
|
426
|
-
|
|
425
|
+
resolve5(chunk.toString().trim());
|
|
427
426
|
});
|
|
428
427
|
process.stdin.resume();
|
|
429
428
|
}
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1064,12 +1158,82 @@ async function claudeUsage() {
|
|
|
1064
1158
|
// src/commands/run.ts
|
|
1065
1159
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
1160
|
import { homedir as homedir7 } from "node:os";
|
|
1067
|
-
import { isAbsolute as
|
|
1161
|
+
import { isAbsolute as isAbsolute4, join as join12, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1162
|
import chalk7 from "chalk";
|
|
1069
1163
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1129,7 +1293,11 @@ var FORWARD_FAILURE_REASON_HEADER = "X-Evident-Failure-Reason";
|
|
|
1129
1293
|
function log(level, event, fields) {
|
|
1130
1294
|
const method = level === "debug" ? "log" : level;
|
|
1131
1295
|
try {
|
|
1132
|
-
|
|
1296
|
+
const payload = Object.fromEntries(
|
|
1297
|
+
Object.entries({ level, event, ...fields }).filter(([, value]) => value !== void 0)
|
|
1298
|
+
);
|
|
1299
|
+
const secondArgument = globalThis.navigator?.userAgent === "Cloudflare-Workers" ? payload : JSON.stringify(payload);
|
|
1300
|
+
console[method]("[evident]", secondArgument);
|
|
1133
1301
|
} catch (err) {
|
|
1134
1302
|
console.error(
|
|
1135
1303
|
"[evident] log_serialize_failed",
|
|
@@ -1158,7 +1326,7 @@ import ora4 from "ora";
|
|
|
1158
1326
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1327
|
|
|
1160
1328
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1329
|
+
var CLI_VERSION = (true ? "3.5.2-dev.4763a76" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1330
|
function getCliVersion() {
|
|
1163
1331
|
return CLI_VERSION;
|
|
1164
1332
|
}
|
|
@@ -1319,8 +1487,591 @@ var EventTypes = {
|
|
|
1319
1487
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1320
1488
|
};
|
|
1321
1489
|
|
|
1322
|
-
// src/lib/
|
|
1490
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1491
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1492
|
+
import { dirname as dirname2 } from "node:path";
|
|
1493
|
+
|
|
1494
|
+
// src/lib/diagnostic-redaction.ts
|
|
1323
1495
|
import { homedir as homedir2 } from "node:os";
|
|
1496
|
+
function redactHomeDirectory(message) {
|
|
1497
|
+
const home = homedir2();
|
|
1498
|
+
if (!home || home === "/") {
|
|
1499
|
+
return message;
|
|
1500
|
+
}
|
|
1501
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1502
|
+
let result = "";
|
|
1503
|
+
let searchFrom = 0;
|
|
1504
|
+
let homeIndex;
|
|
1505
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1506
|
+
const homeEnd = homeIndex + home.length;
|
|
1507
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1508
|
+
const nextCharacter = message[homeEnd];
|
|
1509
|
+
const atPathBoundary = (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1510
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1511
|
+
result += atPathBoundary ? "~" : home;
|
|
1512
|
+
searchFrom = homeEnd;
|
|
1513
|
+
}
|
|
1514
|
+
return result + message.slice(searchFrom);
|
|
1515
|
+
}
|
|
1516
|
+
function redactDiagnosticText(message) {
|
|
1517
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1518
|
+
return redactHomeDirectory(redacted);
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1522
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 100;
|
|
1523
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1524
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1525
|
+
var SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS = 5e3;
|
|
1526
|
+
function errorMessage(error2) {
|
|
1527
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1528
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1529
|
+
}
|
|
1530
|
+
function errorMessages(error2) {
|
|
1531
|
+
const messages = [];
|
|
1532
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1533
|
+
let current = error2;
|
|
1534
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1535
|
+
visited.add(current);
|
|
1536
|
+
if (typeof current === "string") {
|
|
1537
|
+
messages.push(current);
|
|
1538
|
+
break;
|
|
1539
|
+
}
|
|
1540
|
+
if (current instanceof Error) {
|
|
1541
|
+
messages.push(current.name, current.message);
|
|
1542
|
+
current = current.cause;
|
|
1543
|
+
continue;
|
|
1544
|
+
}
|
|
1545
|
+
if (typeof current === "object") {
|
|
1546
|
+
const record = current;
|
|
1547
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1548
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1549
|
+
current = record.cause;
|
|
1550
|
+
continue;
|
|
1551
|
+
}
|
|
1552
|
+
break;
|
|
1553
|
+
}
|
|
1554
|
+
return messages;
|
|
1555
|
+
}
|
|
1556
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1557
|
+
return errorMessages(error2).some(
|
|
1558
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1559
|
+
message
|
|
1560
|
+
)
|
|
1561
|
+
);
|
|
1562
|
+
}
|
|
1563
|
+
var SQLITE_CAUSE_TOKEN = /(?:^|cause:\s+)(SQLiteError:\s+[A-Za-z][A-Za-z0-9]*(?:\s+[A-Za-z][A-Za-z0-9]*){0,7})(?=\s*(?:[)\]"',;]|$))/i;
|
|
1564
|
+
var EFFECT_SQL_CAUSE_TOKEN = /(?:^|cause:\s+)(effect\/sql\/SqlError\/[A-Za-z][A-Za-z0-9_]*)/i;
|
|
1565
|
+
var PARAMS_FIELD = /\bparams\s*(?:=|:)/i;
|
|
1566
|
+
function isCauseBearingText(text) {
|
|
1567
|
+
return /^(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text.trim()) || /(?:^|\s)cause:\s+(?:SQLiteError:\s+|effect\/sql\/SqlError\/)/i.test(text);
|
|
1568
|
+
}
|
|
1569
|
+
function structuredLogFields(line) {
|
|
1570
|
+
const fields = [];
|
|
1571
|
+
for (let index = 0; index < line.length; index++) {
|
|
1572
|
+
if (line[index] === '"') {
|
|
1573
|
+
index++;
|
|
1574
|
+
while (index < line.length) {
|
|
1575
|
+
if (line[index] === "\\") {
|
|
1576
|
+
index++;
|
|
1577
|
+
} else if (line[index] === '"') {
|
|
1578
|
+
break;
|
|
1579
|
+
}
|
|
1580
|
+
index++;
|
|
1581
|
+
}
|
|
1582
|
+
continue;
|
|
1583
|
+
}
|
|
1584
|
+
const atFieldBoundary = index === 0 || /\s/.test(line[index - 1]);
|
|
1585
|
+
const key = line.startsWith("error=", index) ? "error=" : line.startsWith("cause=", index) ? "cause=" : null;
|
|
1586
|
+
if (!atFieldBoundary || key === null) continue;
|
|
1587
|
+
const valueStart = index + key.length;
|
|
1588
|
+
if (line[valueStart] !== '"') {
|
|
1589
|
+
const valueEnd2 = line.indexOf(" ", valueStart);
|
|
1590
|
+
fields.push(line.slice(valueStart, valueEnd2 === -1 ? line.length : valueEnd2));
|
|
1591
|
+
index = valueEnd2 === -1 ? line.length : valueEnd2;
|
|
1592
|
+
continue;
|
|
1593
|
+
}
|
|
1594
|
+
let valueEnd = valueStart + 1;
|
|
1595
|
+
for (; valueEnd < line.length; valueEnd++) {
|
|
1596
|
+
if (line[valueEnd] === "\\") {
|
|
1597
|
+
valueEnd++;
|
|
1598
|
+
} else if (line[valueEnd] === '"') {
|
|
1599
|
+
break;
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
fields.push(line.slice(valueStart + 1, valueEnd));
|
|
1603
|
+
index = valueEnd;
|
|
1604
|
+
}
|
|
1605
|
+
return fields;
|
|
1606
|
+
}
|
|
1607
|
+
function structuredCauseMessages(text) {
|
|
1608
|
+
const fields = structuredLogFields(text);
|
|
1609
|
+
if (fields.length > 0) return fields;
|
|
1610
|
+
return isCauseBearingText(text) ? [text] : [];
|
|
1611
|
+
}
|
|
1612
|
+
function causeTokenFromText(text) {
|
|
1613
|
+
const paramsIndex = text.search(PARAMS_FIELD);
|
|
1614
|
+
for (const pattern of [SQLITE_CAUSE_TOKEN, EFFECT_SQL_CAUSE_TOKEN]) {
|
|
1615
|
+
const match = pattern.exec(text);
|
|
1616
|
+
if (match?.[1] && (paramsIndex === -1 || match.index < paramsIndex)) {
|
|
1617
|
+
return match[1];
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
return null;
|
|
1621
|
+
}
|
|
1622
|
+
function sessionStoreCauseToken(value) {
|
|
1623
|
+
for (const message of structuredCauseMessages(value)) {
|
|
1624
|
+
const token = causeTokenFromText(message);
|
|
1625
|
+
if (token !== null) return redactDiagnosticText(token).slice(0, 120);
|
|
1626
|
+
}
|
|
1627
|
+
return null;
|
|
1628
|
+
}
|
|
1629
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1630
|
+
try {
|
|
1631
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1632
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1633
|
+
if (availableBytes < requiredBytes) {
|
|
1634
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1635
|
+
}
|
|
1636
|
+
return null;
|
|
1637
|
+
} catch (err) {
|
|
1638
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
function readLogicalBytes(db) {
|
|
1642
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1643
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1644
|
+
return pageCount * pageSize;
|
|
1645
|
+
}
|
|
1646
|
+
function readFreelistCount(db) {
|
|
1647
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1648
|
+
}
|
|
1649
|
+
function readCheckpointResult(db) {
|
|
1650
|
+
db.exec("PRAGMA busy_timeout=0");
|
|
1651
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1652
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1653
|
+
}
|
|
1654
|
+
async function probeReclaimAvailability(input) {
|
|
1655
|
+
const { dbPath, requiredBytes } = input;
|
|
1656
|
+
let sqlite;
|
|
1657
|
+
try {
|
|
1658
|
+
sqlite = await import("node:sqlite");
|
|
1659
|
+
} catch (err) {
|
|
1660
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1661
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1662
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1663
|
+
}
|
|
1664
|
+
let autoVacuum = null;
|
|
1665
|
+
try {
|
|
1666
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1667
|
+
try {
|
|
1668
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1669
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1670
|
+
} finally {
|
|
1671
|
+
db.close();
|
|
1672
|
+
}
|
|
1673
|
+
} catch (err) {
|
|
1674
|
+
console.warn(
|
|
1675
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1676
|
+
);
|
|
1677
|
+
}
|
|
1678
|
+
if (autoVacuum !== 0) return null;
|
|
1679
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1680
|
+
}
|
|
1681
|
+
async function reclaimSessionDbSpace(input) {
|
|
1682
|
+
const {
|
|
1683
|
+
dbPath,
|
|
1684
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1685
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1686
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1687
|
+
now = () => performance.now(),
|
|
1688
|
+
hasLiveTurn = false,
|
|
1689
|
+
hasProvenanceAnomaly = false
|
|
1690
|
+
} = input;
|
|
1691
|
+
let sqlite;
|
|
1692
|
+
try {
|
|
1693
|
+
sqlite = await import("node:sqlite");
|
|
1694
|
+
} catch (err) {
|
|
1695
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1696
|
+
console.warn(
|
|
1697
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1698
|
+
);
|
|
1699
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1700
|
+
}
|
|
1701
|
+
const { DatabaseSync } = sqlite;
|
|
1702
|
+
let db;
|
|
1703
|
+
try {
|
|
1704
|
+
db = new DatabaseSync(dbPath);
|
|
1705
|
+
db.exec(`PRAGMA busy_timeout=${SESSION_DB_RECLAIM_BUSY_TIMEOUT_MS}`);
|
|
1706
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1707
|
+
if (autoVacuum === 0) {
|
|
1708
|
+
if (hasLiveTurn || hasProvenanceAnomaly) {
|
|
1709
|
+
const detail = [
|
|
1710
|
+
...hasLiveTurn ? ["a session turn is live"] : [],
|
|
1711
|
+
...hasProvenanceAnomaly ? ["session-DB provenance is anomalous"] : []
|
|
1712
|
+
].join(" and ");
|
|
1713
|
+
console.warn(`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${detail}`);
|
|
1714
|
+
return {
|
|
1715
|
+
ok: false,
|
|
1716
|
+
skipped: "full-vacuum-blocked",
|
|
1717
|
+
detail
|
|
1718
|
+
};
|
|
1719
|
+
}
|
|
1720
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1721
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1722
|
+
if (skipReason !== null) {
|
|
1723
|
+
console.warn(
|
|
1724
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1725
|
+
);
|
|
1726
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1727
|
+
}
|
|
1728
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1729
|
+
const freelistBefore = readFreelistCount(db);
|
|
1730
|
+
const startedAt = now();
|
|
1731
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1732
|
+
db.exec("VACUUM");
|
|
1733
|
+
const afterBytes = readLogicalBytes(db);
|
|
1734
|
+
const freelistAfter = readFreelistCount(db);
|
|
1735
|
+
const checkpointStartedAt = now();
|
|
1736
|
+
const checkpoint = readCheckpointResult(db);
|
|
1737
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1738
|
+
return {
|
|
1739
|
+
ok: true,
|
|
1740
|
+
mode: "convert",
|
|
1741
|
+
beforeBytes,
|
|
1742
|
+
afterBytes,
|
|
1743
|
+
freelistBefore,
|
|
1744
|
+
freelistAfter,
|
|
1745
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1746
|
+
stoppedBy: "freelist-drained",
|
|
1747
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1748
|
+
checkpointElapsedMs,
|
|
1749
|
+
checkpoint
|
|
1750
|
+
};
|
|
1751
|
+
}
|
|
1752
|
+
if (autoVacuum === 2) {
|
|
1753
|
+
if (hasLiveTurn) {
|
|
1754
|
+
const detail = "a session turn is live";
|
|
1755
|
+
console.warn(
|
|
1756
|
+
`[reclaimSessionDbSpace] skipping incremental reclaim of ${dbPath}: ${detail}`
|
|
1757
|
+
);
|
|
1758
|
+
return { ok: false, skipped: "live-turn", detail };
|
|
1759
|
+
}
|
|
1760
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1761
|
+
const freelistBefore = readFreelistCount(db);
|
|
1762
|
+
const startedAt = now();
|
|
1763
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1764
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1765
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1766
|
+
let pagesAttempted = 0;
|
|
1767
|
+
let freelistAfter = freelistBefore;
|
|
1768
|
+
let stoppedBy = "freelist-drained";
|
|
1769
|
+
while (freelistAfter > 0) {
|
|
1770
|
+
if (pagesAttempted >= pageLimit) {
|
|
1771
|
+
stoppedBy = "page-ceiling";
|
|
1772
|
+
break;
|
|
1773
|
+
}
|
|
1774
|
+
if (now() - startedAt >= budget) {
|
|
1775
|
+
stoppedBy = "time-budget";
|
|
1776
|
+
break;
|
|
1777
|
+
}
|
|
1778
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1779
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1780
|
+
pagesAttempted += pages;
|
|
1781
|
+
freelistAfter = readFreelistCount(db);
|
|
1782
|
+
}
|
|
1783
|
+
const afterBytes = readLogicalBytes(db);
|
|
1784
|
+
const checkpointStartedAt = now();
|
|
1785
|
+
const checkpoint = readCheckpointResult(db);
|
|
1786
|
+
const checkpointElapsedMs = Math.max(0, now() - checkpointStartedAt);
|
|
1787
|
+
return {
|
|
1788
|
+
ok: true,
|
|
1789
|
+
mode: "incremental",
|
|
1790
|
+
beforeBytes,
|
|
1791
|
+
afterBytes,
|
|
1792
|
+
freelistBefore,
|
|
1793
|
+
freelistAfter,
|
|
1794
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1795
|
+
stoppedBy,
|
|
1796
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1797
|
+
checkpointElapsedMs,
|
|
1798
|
+
checkpoint
|
|
1799
|
+
};
|
|
1800
|
+
}
|
|
1801
|
+
console.warn(
|
|
1802
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1803
|
+
);
|
|
1804
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1805
|
+
} catch (err) {
|
|
1806
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1807
|
+
return {
|
|
1808
|
+
ok: false,
|
|
1809
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1810
|
+
detail: errorMessage(err)
|
|
1811
|
+
};
|
|
1812
|
+
} finally {
|
|
1813
|
+
db?.close();
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1817
|
+
// src/lib/opencode/dependency-install-diagnostic.ts
|
|
1818
|
+
import { lstatSync, realpathSync } from "node:fs";
|
|
1819
|
+
import { basename, dirname as dirname3, isAbsolute, join as join2, relative, resolve, sep } from "node:path";
|
|
1820
|
+
var DEPENDENCY_INSTALL_FAILURE_MESSAGE = "background dependency install failed";
|
|
1821
|
+
var MAX_DISTINCT_DIAGNOSTICS = 32;
|
|
1822
|
+
var EFFECT = "OpenCode could not install background dependencies for this scope; its plugins and tools may be unavailable";
|
|
1823
|
+
function readOpenCodeLogField(line, key) {
|
|
1824
|
+
const match = line.match(new RegExp(`(?:^|\\s)${key}=(?:"((?:\\\\.|[^"])*)"|(\\S+))`));
|
|
1825
|
+
if (!match) return void 0;
|
|
1826
|
+
if (match[1] === void 0) return match[2];
|
|
1827
|
+
return match[1].replace(/\\(["\\n])/g, (_match, escaped) => {
|
|
1828
|
+
return escaped === "n" ? "\n" : escaped;
|
|
1829
|
+
});
|
|
1830
|
+
}
|
|
1831
|
+
function parseTrigger(line) {
|
|
1832
|
+
if (readOpenCodeLogField(line, "message") !== DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
1833
|
+
return null;
|
|
1834
|
+
}
|
|
1835
|
+
const error2 = readOpenCodeLogField(line, "error");
|
|
1836
|
+
if (!error2 || !/\bEACCES\b/.test(error2)) return null;
|
|
1837
|
+
const errnoMatch = error2.match(/\bEACCES:\s*permission denied,\s*([a-z_]+)\s+'([^']*)'/);
|
|
1838
|
+
const capturedPath = errnoMatch?.[2];
|
|
1839
|
+
return {
|
|
1840
|
+
dir: readOpenCodeLogField(line, "dir"),
|
|
1841
|
+
op: errnoMatch?.[1],
|
|
1842
|
+
path: capturedPath && isAbsolute(capturedPath) ? capturedPath : void 0
|
|
1843
|
+
};
|
|
1844
|
+
}
|
|
1845
|
+
function getErrorCode(error2) {
|
|
1846
|
+
if (typeof error2 === "object" && error2 !== null && "code" in error2) {
|
|
1847
|
+
const code = error2.code;
|
|
1848
|
+
if (typeof code === "string") return code;
|
|
1849
|
+
}
|
|
1850
|
+
return "UNKNOWN";
|
|
1851
|
+
}
|
|
1852
|
+
function isWithin(path, root) {
|
|
1853
|
+
const candidate = resolve(path);
|
|
1854
|
+
const resolvedRoot = resolve(root);
|
|
1855
|
+
return candidate === resolvedRoot || candidate.startsWith(`${resolvedRoot}${sep}`);
|
|
1856
|
+
}
|
|
1857
|
+
function rootSpecs(context) {
|
|
1858
|
+
return [
|
|
1859
|
+
{ token: "~", configured: resolve(context.homeDir) },
|
|
1860
|
+
{ token: "$WORKSPACE", configured: resolve(context.cwd) }
|
|
1861
|
+
];
|
|
1862
|
+
}
|
|
1863
|
+
function resolveRoots(context) {
|
|
1864
|
+
const resolved = [];
|
|
1865
|
+
const unresolved = [];
|
|
1866
|
+
for (const spec of rootSpecs(context)) {
|
|
1867
|
+
try {
|
|
1868
|
+
resolved.push({ ...spec, real: realpathSync(spec.configured) });
|
|
1869
|
+
} catch (_error) {
|
|
1870
|
+
unresolved.push(spec);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
return { resolved, unresolved };
|
|
1874
|
+
}
|
|
1875
|
+
function resolveNearestAncestor(failingPath, remaining, error2) {
|
|
1876
|
+
let ancestor = dirname3(failingPath);
|
|
1877
|
+
const suffix = [basename(failingPath), ...remaining];
|
|
1878
|
+
while (ancestor) {
|
|
1879
|
+
try {
|
|
1880
|
+
return {
|
|
1881
|
+
path: resolve(realpathSync(ancestor), ...suffix),
|
|
1882
|
+
failure: { path: resolve(failingPath), error: error2 }
|
|
1883
|
+
};
|
|
1884
|
+
} catch (ancestorError) {
|
|
1885
|
+
if (ancestor === dirname3(ancestor) || !getErrorCode(ancestorError).startsWith("E")) {
|
|
1886
|
+
return null;
|
|
1887
|
+
}
|
|
1888
|
+
suffix.unshift(basename(ancestor));
|
|
1889
|
+
ancestor = dirname3(ancestor);
|
|
1890
|
+
}
|
|
1891
|
+
}
|
|
1892
|
+
return null;
|
|
1893
|
+
}
|
|
1894
|
+
function resolveExistingPath(absolutePath) {
|
|
1895
|
+
if (!isAbsolute(absolutePath)) return null;
|
|
1896
|
+
const candidate = resolve(absolutePath);
|
|
1897
|
+
const remaining = [];
|
|
1898
|
+
let current = candidate;
|
|
1899
|
+
while (current) {
|
|
1900
|
+
try {
|
|
1901
|
+
const stat2 = lstatSync(current);
|
|
1902
|
+
if (stat2.isSymbolicLink()) {
|
|
1903
|
+
if (remaining.length === 0) {
|
|
1904
|
+
return { path: join2(realpathSync(dirname3(current)), basename(current)) };
|
|
1905
|
+
}
|
|
1906
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1907
|
+
}
|
|
1908
|
+
return { path: resolve(realpathSync(current), ...remaining) };
|
|
1909
|
+
} catch (error2) {
|
|
1910
|
+
const code = getErrorCode(error2);
|
|
1911
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
1912
|
+
return resolveNearestAncestor(current, remaining, error2);
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
if (current === dirname3(current)) return null;
|
|
1916
|
+
remaining.unshift(basename(current));
|
|
1917
|
+
current = dirname3(current);
|
|
1918
|
+
}
|
|
1919
|
+
return null;
|
|
1920
|
+
}
|
|
1921
|
+
function tokenise(absolutePath, roots) {
|
|
1922
|
+
const pathResolution = resolveExistingPath(absolutePath);
|
|
1923
|
+
if (!pathResolution) return null;
|
|
1924
|
+
const root = roots.reduce((selected, candidate) => {
|
|
1925
|
+
if (!isWithin(pathResolution.path, candidate.real)) return selected;
|
|
1926
|
+
if (!selected || candidate.real.length > selected.real.length || candidate.real.length === selected.real.length && candidate.token === "$WORKSPACE") {
|
|
1927
|
+
return candidate;
|
|
1928
|
+
}
|
|
1929
|
+
return selected;
|
|
1930
|
+
}, null);
|
|
1931
|
+
if (!root) return null;
|
|
1932
|
+
const suffix = relative(root.real, pathResolution.path);
|
|
1933
|
+
return {
|
|
1934
|
+
value: suffix ? `${root.token}/${suffix.split(sep).join("/")}` : root.token,
|
|
1935
|
+
root,
|
|
1936
|
+
resolutionFailure: pathResolution.failure
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
function classifyScope(dir, roots) {
|
|
1940
|
+
if (!dir) return "other";
|
|
1941
|
+
const tokenisedDir = tokenise(dir, roots.resolved);
|
|
1942
|
+
if (tokenisedDir) return tokenisedDir.root.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1943
|
+
for (const spec of roots.unresolved) {
|
|
1944
|
+
if (isWithin(dir, spec.configured)) {
|
|
1945
|
+
return spec.token === "$WORKSPACE" ? "workspace" : "global";
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
return "other";
|
|
1949
|
+
}
|
|
1950
|
+
function unresolvedRootForPath(path, unresolved) {
|
|
1951
|
+
return unresolved.find((spec) => isWithin(path, spec.configured));
|
|
1952
|
+
}
|
|
1953
|
+
function statType(stat2) {
|
|
1954
|
+
if (stat2.isDirectory()) return "d";
|
|
1955
|
+
if (stat2.isFile()) return "f";
|
|
1956
|
+
if (stat2.isSymbolicLink()) return "l";
|
|
1957
|
+
return "o";
|
|
1958
|
+
}
|
|
1959
|
+
function renderStat(name, stat2) {
|
|
1960
|
+
const mode = (stat2.mode & 4095).toString(8).padStart(4, "0");
|
|
1961
|
+
return `${name}:${statType(stat2)}:${stat2.uid}:${stat2.gid}:${mode}`;
|
|
1962
|
+
}
|
|
1963
|
+
function renderStatError(name, error2) {
|
|
1964
|
+
const code = getErrorCode(error2);
|
|
1965
|
+
return `${name}:${renderErrorCode(code)}`;
|
|
1966
|
+
}
|
|
1967
|
+
function renderErrorCode(code) {
|
|
1968
|
+
if (code === "ENOENT") return "missing";
|
|
1969
|
+
return code.startsWith("E") ? code : `E${code}`;
|
|
1970
|
+
}
|
|
1971
|
+
function inspectChain(rootReal, rootToken, absolutePath, resolutionFailure) {
|
|
1972
|
+
const target = resolve(absolutePath);
|
|
1973
|
+
const relativePath = relative(rootReal, target);
|
|
1974
|
+
const components = relativePath ? relativePath.split(sep) : [];
|
|
1975
|
+
const entries = [];
|
|
1976
|
+
let current = rootReal;
|
|
1977
|
+
let status2 = "complete";
|
|
1978
|
+
let finalStat;
|
|
1979
|
+
for (const name of [rootToken, ...components]) {
|
|
1980
|
+
if (name !== rootToken) current = join2(current, name);
|
|
1981
|
+
if (resolutionFailure && resolve(current) === resolve(resolutionFailure.path)) {
|
|
1982
|
+
entries.push(renderStatError(name, resolutionFailure.error));
|
|
1983
|
+
status2 = "stat_failed";
|
|
1984
|
+
break;
|
|
1985
|
+
}
|
|
1986
|
+
try {
|
|
1987
|
+
const stat2 = lstatSync(current);
|
|
1988
|
+
entries.push(renderStat(name, stat2));
|
|
1989
|
+
finalStat = stat2;
|
|
1990
|
+
} catch (error2) {
|
|
1991
|
+
entries.push(renderStatError(name, error2));
|
|
1992
|
+
if (getErrorCode(error2) !== "ENOENT") status2 = "stat_failed";
|
|
1993
|
+
break;
|
|
1994
|
+
}
|
|
1995
|
+
}
|
|
1996
|
+
let symlinkTarget;
|
|
1997
|
+
let symlinkMeta;
|
|
1998
|
+
if (finalStat?.isSymbolicLink() && entries.length === components.length + 1) {
|
|
1999
|
+
try {
|
|
2000
|
+
symlinkTarget = realpathSync(current);
|
|
2001
|
+
try {
|
|
2002
|
+
symlinkMeta = renderStat("", lstatSync(symlinkTarget)).slice(1);
|
|
2003
|
+
} catch (error2) {
|
|
2004
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2005
|
+
status2 = "stat_failed";
|
|
2006
|
+
}
|
|
2007
|
+
} catch (error2) {
|
|
2008
|
+
symlinkTarget = "<unresolved>";
|
|
2009
|
+
symlinkMeta = renderErrorCode(getErrorCode(error2));
|
|
2010
|
+
status2 = "stat_failed";
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
return { status: status2, chain: entries.join(","), symlinkTarget, symlinkMeta };
|
|
2014
|
+
}
|
|
2015
|
+
function buildDiagnostic(line, context) {
|
|
2016
|
+
const trigger = parseTrigger(line);
|
|
2017
|
+
if (!trigger) return null;
|
|
2018
|
+
const roots = resolveRoots(context);
|
|
2019
|
+
const scope = classifyScope(trigger.dir, roots);
|
|
2020
|
+
const pathSource = trigger.path ? "error" : trigger.dir ? "dir" : "none";
|
|
2021
|
+
const inspectedPath = trigger.path ?? trigger.dir;
|
|
2022
|
+
const microvmId = context.microvmId?.trim() || "unknown";
|
|
2023
|
+
const euid = context.euid === void 0 ? "unavailable" : String(context.euid);
|
|
2024
|
+
const egid = context.egid === void 0 ? "unavailable" : String(context.egid);
|
|
2025
|
+
const op = trigger.op ?? "unknown";
|
|
2026
|
+
if (!inspectedPath) {
|
|
2027
|
+
return {
|
|
2028
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=no_path path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2029
|
+
localDetail: null
|
|
2030
|
+
};
|
|
2031
|
+
}
|
|
2032
|
+
const tokenisedPath = tokenise(inspectedPath, roots.resolved);
|
|
2033
|
+
if (!tokenisedPath) {
|
|
2034
|
+
const unresolvedRoot = unresolvedRootForPath(inspectedPath, roots.unresolved);
|
|
2035
|
+
if (unresolvedRoot) {
|
|
2036
|
+
return {
|
|
2037
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=root_unresolved path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=unknown cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2038
|
+
localDetail: null
|
|
2039
|
+
};
|
|
2040
|
+
}
|
|
2041
|
+
return {
|
|
2042
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=outside_roots path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=<outside-approved-roots> cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=unavailable`,
|
|
2043
|
+
localDetail: `OPENCODE-DEPENDENCY-INSTALL-EACCES (local only): outside approved roots: ${inspectedPath}`
|
|
2044
|
+
};
|
|
2045
|
+
}
|
|
2046
|
+
const inspection = inspectChain(
|
|
2047
|
+
tokenisedPath.root.real,
|
|
2048
|
+
tokenisedPath.root.token,
|
|
2049
|
+
inspectedPath,
|
|
2050
|
+
tokenisedPath.resolutionFailure
|
|
2051
|
+
);
|
|
2052
|
+
const optionalSymlinkFields = inspection.symlinkTarget ? ` symlink_target=${tokenise(inspection.symlinkTarget, roots.resolved)?.value ?? "<outside-approved-roots>"}` + (inspection.symlinkMeta ? ` symlink_meta=${inspection.symlinkMeta}` : "") : "";
|
|
2053
|
+
return {
|
|
2054
|
+
event: `OPENCODE-DEPENDENCY-INSTALL-EACCES: status=${inspection.status} path_source=${pathSource} scope=${scope} microvm_id=${microvmId} op=${op} path=${tokenisedPath.value} cli_euid=${euid} cli_egid=${egid} effect="${EFFECT}" chain=${inspection.chain}${optionalSymlinkFields}`,
|
|
2055
|
+
localDetail: null
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
function createDependencyInstallDiagnosticReporter({
|
|
2059
|
+
context,
|
|
2060
|
+
emitEvent: emitEvent2,
|
|
2061
|
+
emitLocal
|
|
2062
|
+
}) {
|
|
2063
|
+
const emittedEvents = /* @__PURE__ */ new Set();
|
|
2064
|
+
return (line) => {
|
|
2065
|
+
const diagnostic = buildDiagnostic(line, context);
|
|
2066
|
+
if (!diagnostic || emittedEvents.has(diagnostic.event)) return;
|
|
2067
|
+
if (emittedEvents.size >= MAX_DISTINCT_DIAGNOSTICS) return;
|
|
2068
|
+
emittedEvents.add(diagnostic.event);
|
|
2069
|
+
emitEvent2(diagnostic.event);
|
|
2070
|
+
if (diagnostic.localDetail) emitLocal(diagnostic.localDetail);
|
|
2071
|
+
};
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1324
2075
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1325
2076
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1326
2077
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1337,11 +2088,13 @@ function parseOpenCodeLogLine(line) {
|
|
|
1337
2088
|
if (!levelMatch) return null;
|
|
1338
2089
|
const level = levelMatch[1].toUpperCase();
|
|
1339
2090
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1340
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
2091
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
2092
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1341
2093
|
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1342
2094
|
const parsed = {
|
|
1343
2095
|
level: level === "WARN" ? "warn" : "error",
|
|
1344
|
-
sessionID: sessionMatch?.[1]
|
|
2096
|
+
sessionID: sessionMatch?.[1],
|
|
2097
|
+
messageID: messageIDMatch?.[1]
|
|
1345
2098
|
};
|
|
1346
2099
|
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1347
2100
|
if (message !== void 0) parsed.message = message;
|
|
@@ -1349,18 +2102,29 @@ function parseOpenCodeLogLine(line) {
|
|
|
1349
2102
|
}
|
|
1350
2103
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1351
2104
|
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1352
|
-
function
|
|
2105
|
+
function sanitiseDependencyInstallWarning(line) {
|
|
2106
|
+
const withoutDirectory = line.replace(
|
|
2107
|
+
/((?:^|\s)dir=)(?:"(?:\\.|[^"])*"|\/[^\s)"']+|\S+)/g,
|
|
2108
|
+
"$1<path>"
|
|
2109
|
+
);
|
|
2110
|
+
return withoutDirectory.replace(
|
|
2111
|
+
/((?:^|\s)error=)"((?:\\.|[^"])*)"/,
|
|
2112
|
+
(_match, prefix, error2) => `${prefix}"${error2.replace(/'\/[^']*'/g, "'<path>'")}"`
|
|
2113
|
+
);
|
|
2114
|
+
}
|
|
2115
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure, onDependencyInstallFailure) {
|
|
1353
2116
|
let buffer = Buffer.alloc(0);
|
|
1354
2117
|
const repeatStates = /* @__PURE__ */ new Map();
|
|
1355
2118
|
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1356
2119
|
const metadata = {
|
|
1357
2120
|
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
2121
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1358
2122
|
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1359
2123
|
};
|
|
1360
2124
|
forwardRunnerActivity(
|
|
1361
2125
|
{
|
|
1362
2126
|
level: parsed.level,
|
|
1363
|
-
error: line,
|
|
2127
|
+
error: parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE ? sanitiseDependencyInstallWarning(line) : line,
|
|
1364
2128
|
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1365
2129
|
source: "opencode"
|
|
1366
2130
|
},
|
|
@@ -1378,11 +2142,9 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1378
2142
|
repeatStates.delete(key);
|
|
1379
2143
|
}
|
|
1380
2144
|
};
|
|
1381
|
-
const
|
|
1382
|
-
const parsed = parseOpenCodeLogLine(line);
|
|
1383
|
-
if (!parsed) return;
|
|
2145
|
+
const forwardWithRepeatSuppression = (line, parsed) => {
|
|
1384
2146
|
const now = Date.now();
|
|
1385
|
-
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
2147
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1386
2148
|
expireRepeatStates(now, key);
|
|
1387
2149
|
if (key === void 0) {
|
|
1388
2150
|
forwardLine(line, parsed);
|
|
@@ -1403,6 +2165,35 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1403
2165
|
previous.parsed = parsed;
|
|
1404
2166
|
previous.repeatsSuppressed++;
|
|
1405
2167
|
};
|
|
2168
|
+
const flushLine = (line) => {
|
|
2169
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
2170
|
+
const isDiskFailure = isSessionStoreWriteFailure(line);
|
|
2171
|
+
const cause = sessionStoreCauseToken(line);
|
|
2172
|
+
if (onSessionStoreWriteFailure && (isDiskFailure || cause !== null)) {
|
|
2173
|
+
try {
|
|
2174
|
+
onSessionStoreWriteFailure({
|
|
2175
|
+
isDiskFailure,
|
|
2176
|
+
sessionId: parsed?.sessionID ?? null,
|
|
2177
|
+
cause
|
|
2178
|
+
});
|
|
2179
|
+
} catch (err) {
|
|
2180
|
+
reportTelemetryDiagnostic(
|
|
2181
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
2182
|
+
);
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
if (!parsed) return;
|
|
2186
|
+
forwardWithRepeatSuppression(line, parsed);
|
|
2187
|
+
if (onDependencyInstallFailure && parsed.message === DEPENDENCY_INSTALL_FAILURE_MESSAGE) {
|
|
2188
|
+
try {
|
|
2189
|
+
onDependencyInstallFailure(line);
|
|
2190
|
+
} catch (err) {
|
|
2191
|
+
reportTelemetryDiagnostic(
|
|
2192
|
+
`Runner activity telemetry could not record dependency-install failure diagnostic: ${err instanceof Error ? err.message : String(err)}`
|
|
2193
|
+
);
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
};
|
|
1406
2197
|
const forward = ((chunk) => {
|
|
1407
2198
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1408
2199
|
let newlineIndex;
|
|
@@ -1428,34 +2219,38 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1428
2219
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1429
2220
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1430
2221
|
var MAX_METADATA_ENTRIES = 20;
|
|
1431
|
-
var TRUNCATION_MARKER = "\u2026";
|
|
1432
|
-
function redactHomeDirectory(message) {
|
|
1433
|
-
const home = homedir2();
|
|
1434
|
-
if (!home || home === "/") {
|
|
1435
|
-
return message;
|
|
1436
|
-
}
|
|
1437
|
-
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1438
|
-
let result = "";
|
|
1439
|
-
let searchFrom = 0;
|
|
1440
|
-
let homeIndex;
|
|
1441
|
-
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1442
|
-
const homeEnd = homeIndex + home.length;
|
|
1443
|
-
const previousCharacter = message[homeIndex - 1];
|
|
1444
|
-
const nextCharacter = message[homeEnd];
|
|
1445
|
-
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1446
|
-
result += message.slice(searchFrom, homeIndex);
|
|
1447
|
-
result += atPathBoundary ? "~" : home;
|
|
1448
|
-
searchFrom = homeEnd;
|
|
1449
|
-
}
|
|
1450
|
-
return result + message.slice(searchFrom);
|
|
1451
|
-
}
|
|
1452
|
-
function redact(message) {
|
|
1453
|
-
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1454
|
-
return redactHomeDirectory(redacted);
|
|
1455
|
-
}
|
|
1456
2222
|
function truncate(message) {
|
|
1457
2223
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1458
|
-
|
|
2224
|
+
let dropped = message.length - MAX_MESSAGE_LENGTH;
|
|
2225
|
+
let marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2226
|
+
let previousDropped = -1;
|
|
2227
|
+
while (dropped !== previousDropped) {
|
|
2228
|
+
previousDropped = dropped;
|
|
2229
|
+
const nextDropped = message.length - MAX_MESSAGE_LENGTH + marker.length;
|
|
2230
|
+
dropped = nextDropped;
|
|
2231
|
+
marker = `\u2026${dropped} chars omitted\u2026`;
|
|
2232
|
+
}
|
|
2233
|
+
const remaining = MAX_MESSAGE_LENGTH - marker.length;
|
|
2234
|
+
const headLength = Math.ceil(remaining * 0.6);
|
|
2235
|
+
const tailLength = remaining - headLength;
|
|
2236
|
+
const effectMatch = message.startsWith("OPENCODE-DEPENDENCY-INSTALL-EACCES:") ? message.match(/ effect="[^"]*"/) : null;
|
|
2237
|
+
if (effectMatch?.index !== void 0) {
|
|
2238
|
+
const effectEnd = effectMatch.index + effectMatch[0].length;
|
|
2239
|
+
if (effectEnd > headLength && effectEnd < message.length) {
|
|
2240
|
+
const effectLength = effectMatch[0].length;
|
|
2241
|
+
const nonEffectLength = remaining - effectLength;
|
|
2242
|
+
const preservedTailLength = Math.min(
|
|
2243
|
+
message.length - effectEnd,
|
|
2244
|
+
nonEffectLength - Math.min(effectMatch.index, Math.ceil(nonEffectLength * 0.6))
|
|
2245
|
+
);
|
|
2246
|
+
const preservedHeadLength = Math.min(
|
|
2247
|
+
effectMatch.index,
|
|
2248
|
+
nonEffectLength - preservedTailLength
|
|
2249
|
+
);
|
|
2250
|
+
return message.slice(0, preservedHeadLength) + marker + message.slice(effectMatch.index, effectEnd) + message.slice(message.length - preservedTailLength);
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
return message.slice(0, headLength) + marker + message.slice(message.length - tailLength);
|
|
1459
2254
|
}
|
|
1460
2255
|
function sanitiseMetadata(metadata) {
|
|
1461
2256
|
if (!metadata) return {};
|
|
@@ -1468,7 +2263,10 @@ function sanitiseMetadata(metadata) {
|
|
|
1468
2263
|
const sanitised = [];
|
|
1469
2264
|
for (const [key, value] of entries) {
|
|
1470
2265
|
if (typeof value === "string") {
|
|
1471
|
-
sanitised.push([
|
|
2266
|
+
sanitised.push([
|
|
2267
|
+
key,
|
|
2268
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
2269
|
+
]);
|
|
1472
2270
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1473
2271
|
sanitised.push([key, value]);
|
|
1474
2272
|
}
|
|
@@ -1513,7 +2311,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1513
2311
|
if (!context.agentId || !context.authHeader) return;
|
|
1514
2312
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1515
2313
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1516
|
-
const message = truncate(
|
|
2314
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1517
2315
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1518
2316
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1519
2317
|
message,
|
|
@@ -1529,10 +2327,10 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1529
2327
|
|
|
1530
2328
|
// src/lib/opencode/session-db-recovery-report.ts
|
|
1531
2329
|
import { readFileSync as readFileSync2, unlinkSync } from "node:fs";
|
|
1532
|
-
import { join as
|
|
2330
|
+
import { join as join3 } from "node:path";
|
|
1533
2331
|
function sessionDbRecoveryReportPath(homeDir, env) {
|
|
1534
2332
|
const override = env.EVIDENT_SESSION_DB_RECOVERY_REPORT?.trim();
|
|
1535
|
-
return override ||
|
|
2333
|
+
return override || join3(homeDir, ".local", "state", "evident", "session-db-recovery.jsonl");
|
|
1536
2334
|
}
|
|
1537
2335
|
function drainSessionDbRecoveryReport({
|
|
1538
2336
|
homeDir,
|
|
@@ -1662,12 +2460,23 @@ function buildSessionDbRecoveryActivity(record) {
|
|
|
1662
2460
|
metadata: withoutContractFields(record),
|
|
1663
2461
|
message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
|
|
1664
2462
|
};
|
|
1665
|
-
case "session_db_boot_refused":
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
2463
|
+
case "session_db_boot_refused": {
|
|
2464
|
+
const message = (() => {
|
|
2465
|
+
switch (record.reason) {
|
|
2466
|
+
case "local_discard_failed":
|
|
2467
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
|
|
2468
|
+
case "replica_separation_unproven":
|
|
2469
|
+
return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
|
|
2470
|
+
case "verification_timeout_before_separation":
|
|
2471
|
+
return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2472
|
+
case "verification_timeout_during_separation":
|
|
2473
|
+
return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
|
|
2474
|
+
default:
|
|
2475
|
+
return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
|
|
2476
|
+
}
|
|
2477
|
+
})();
|
|
2478
|
+
return { level, metadata: withoutContractFields(record), message };
|
|
2479
|
+
}
|
|
1671
2480
|
case "schema_provenance_mismatch":
|
|
1672
2481
|
return {
|
|
1673
2482
|
level,
|
|
@@ -1794,7 +2603,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1794
2603
|
if (health.healthy || health.authFailed) {
|
|
1795
2604
|
return health;
|
|
1796
2605
|
}
|
|
1797
|
-
await new Promise((
|
|
2606
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1798
2607
|
}
|
|
1799
2608
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1800
2609
|
}
|
|
@@ -1805,15 +2614,53 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1805
2614
|
if (health.healthy || health.authFailed) {
|
|
1806
2615
|
return health;
|
|
1807
2616
|
}
|
|
1808
|
-
await new Promise((
|
|
2617
|
+
await new Promise((resolve5) => setTimeout(resolve5, 1e3));
|
|
1809
2618
|
}
|
|
1810
2619
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1811
2620
|
}
|
|
1812
2621
|
|
|
1813
2622
|
// src/lib/http-timeout.ts
|
|
1814
2623
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2624
|
+
var RequestTimeoutError = class extends Error {
|
|
2625
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2626
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2627
|
+
this.operation = operation;
|
|
2628
|
+
this.timeoutMs = timeoutMs;
|
|
2629
|
+
this.elapsedMs = elapsedMs;
|
|
2630
|
+
this.name = "TimeoutError";
|
|
2631
|
+
}
|
|
2632
|
+
};
|
|
2633
|
+
function describeRequestOperation(input, init) {
|
|
2634
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2635
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2636
|
+
let path;
|
|
2637
|
+
try {
|
|
2638
|
+
path = new URL(rawUrl).pathname;
|
|
2639
|
+
} catch (error2) {
|
|
2640
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2641
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2642
|
+
}
|
|
2643
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2644
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2645
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2646
|
+
return `${method} ${maskedPath}`;
|
|
2647
|
+
}
|
|
1815
2648
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1816
|
-
return ((input, init) =>
|
|
2649
|
+
return (async (input, init) => {
|
|
2650
|
+
const startedAt = Date.now();
|
|
2651
|
+
try {
|
|
2652
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2653
|
+
} catch (error2) {
|
|
2654
|
+
if (error2?.name === "TimeoutError") {
|
|
2655
|
+
throw new RequestTimeoutError(
|
|
2656
|
+
describeRequestOperation(input, init),
|
|
2657
|
+
timeoutMs,
|
|
2658
|
+
Date.now() - startedAt
|
|
2659
|
+
);
|
|
2660
|
+
}
|
|
2661
|
+
throw error2;
|
|
2662
|
+
}
|
|
2663
|
+
});
|
|
1817
2664
|
}
|
|
1818
2665
|
|
|
1819
2666
|
// src/lib/opencode/client.ts
|
|
@@ -1879,7 +2726,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1879
2726
|
return null;
|
|
1880
2727
|
}
|
|
1881
2728
|
}
|
|
1882
|
-
function
|
|
2729
|
+
function roleOf2(m) {
|
|
1883
2730
|
if (!m || typeof m !== "object") return void 0;
|
|
1884
2731
|
if (typeof m.role === "string") return m.role;
|
|
1885
2732
|
const infoRole = m.info?.role;
|
|
@@ -1893,13 +2740,13 @@ function createdOf(m) {
|
|
|
1893
2740
|
if (!m || typeof m !== "object") return void 0;
|
|
1894
2741
|
return m.info?.time?.created ?? m.time?.created;
|
|
1895
2742
|
}
|
|
1896
|
-
function
|
|
2743
|
+
function idOf2(m) {
|
|
1897
2744
|
if (!m || typeof m !== "object") return void 0;
|
|
1898
2745
|
if (typeof m.id === "string") return m.id;
|
|
1899
2746
|
const infoId = m.info?.id;
|
|
1900
2747
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1901
2748
|
}
|
|
1902
|
-
function
|
|
2749
|
+
function parentIdOf2(m) {
|
|
1903
2750
|
if (!m || typeof m !== "object") return void 0;
|
|
1904
2751
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1905
2752
|
const infoParent = m.info?.parentID;
|
|
@@ -1915,8 +2762,12 @@ function errorOf(m) {
|
|
|
1915
2762
|
if (!m || typeof m !== "object") return void 0;
|
|
1916
2763
|
return m.info?.error ?? m.error;
|
|
1917
2764
|
}
|
|
2765
|
+
function isTerminalStopAnswer(m) {
|
|
2766
|
+
return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
|
|
2767
|
+
}
|
|
1918
2768
|
function isAssistantInFlight(m) {
|
|
1919
2769
|
if (completedOf(m) == null) return true;
|
|
2770
|
+
if (errorOf(m) != null) return false;
|
|
1920
2771
|
return finishOf(m) === "tool-calls";
|
|
1921
2772
|
}
|
|
1922
2773
|
async function getSessionMessages(port, sessionId, client) {
|
|
@@ -1964,9 +2815,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1964
2815
|
function isSessionActivelyGenerating(messages) {
|
|
1965
2816
|
if (!messages || messages.length === 0) return false;
|
|
1966
2817
|
const last = messages[messages.length - 1];
|
|
1967
|
-
if (
|
|
2818
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1968
2819
|
return completedOf(last) == null;
|
|
1969
2820
|
}
|
|
2821
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1970
2822
|
function sessionLastActivityMs(session) {
|
|
1971
2823
|
const candidates = [
|
|
1972
2824
|
session.time?.updated,
|
|
@@ -1981,30 +2833,71 @@ function sessionLastActivityMs(session) {
|
|
|
1981
2833
|
}
|
|
1982
2834
|
return null;
|
|
1983
2835
|
}
|
|
1984
|
-
|
|
1985
|
-
|
|
2836
|
+
function sessionListPath(options) {
|
|
2837
|
+
const params = new URLSearchParams();
|
|
2838
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2839
|
+
const query = params.toString();
|
|
2840
|
+
return query ? `/session?${query}` : "/session";
|
|
2841
|
+
}
|
|
2842
|
+
async function listSessionPage(port, client, options) {
|
|
2843
|
+
const path = sessionListPath(options);
|
|
1986
2844
|
try {
|
|
1987
|
-
const res = await requestWithClient(port, client,
|
|
2845
|
+
const res = await requestWithClient(port, client, path);
|
|
1988
2846
|
if (!res.ok) return null;
|
|
1989
2847
|
const body = await res.json();
|
|
1990
2848
|
return Array.isArray(body) ? body : null;
|
|
1991
2849
|
} catch (error2) {
|
|
1992
2850
|
console.error(
|
|
1993
|
-
`[
|
|
2851
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1994
2852
|
);
|
|
1995
2853
|
return null;
|
|
1996
2854
|
}
|
|
1997
2855
|
}
|
|
2856
|
+
async function listSessions(port, client, options) {
|
|
2857
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2858
|
+
return listSessionPage(port, client, options);
|
|
2859
|
+
}
|
|
2860
|
+
async function listAllSessions(port, client) {
|
|
2861
|
+
if (client?.version === "v2") {
|
|
2862
|
+
const sessions2 = await listV2Sessions(client);
|
|
2863
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2864
|
+
}
|
|
2865
|
+
const sessions = await listSessionPage(port, client, {
|
|
2866
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2867
|
+
});
|
|
2868
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2869
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2870
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2871
|
+
}
|
|
2872
|
+
return { sessions };
|
|
2873
|
+
}
|
|
1998
2874
|
async function deleteSession(port, id, client) {
|
|
1999
2875
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
2000
2876
|
try {
|
|
2001
|
-
const res = await requestWithClient(
|
|
2002
|
-
|
|
2877
|
+
const res = await requestWithClient(
|
|
2878
|
+
port,
|
|
2879
|
+
client,
|
|
2880
|
+
`/session/${id}`,
|
|
2881
|
+
{ method: "DELETE" },
|
|
2882
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2883
|
+
);
|
|
2884
|
+
if (res.status >= 200 && res.status < 300) {
|
|
2885
|
+
return { ok: true, status: res.status };
|
|
2886
|
+
}
|
|
2887
|
+
try {
|
|
2888
|
+
const body = await res.text();
|
|
2889
|
+
return body ? { ok: false, status: res.status, body } : { ok: false, status: res.status };
|
|
2890
|
+
} catch (error2) {
|
|
2891
|
+
console.error(
|
|
2892
|
+
`[deleteSession] DELETE /session/${id} response body read failed for HTTP ${res.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2893
|
+
);
|
|
2894
|
+
return { ok: false, status: res.status };
|
|
2895
|
+
}
|
|
2003
2896
|
} catch (error2) {
|
|
2004
2897
|
console.error(
|
|
2005
2898
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2006
2899
|
);
|
|
2007
|
-
return false;
|
|
2900
|
+
return { ok: false, status: "threw" };
|
|
2008
2901
|
}
|
|
2009
2902
|
}
|
|
2010
2903
|
async function sessionExists(port, id, client) {
|
|
@@ -2242,11 +3135,31 @@ function messageText(m) {
|
|
|
2242
3135
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2243
3136
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2244
3137
|
}
|
|
2245
|
-
|
|
3138
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
3139
|
+
if (!messages || messages.length === 0) return null;
|
|
3140
|
+
let best = null;
|
|
3141
|
+
for (const message of messages) {
|
|
3142
|
+
if (roleOf2(message) !== "user") continue;
|
|
3143
|
+
const id = idOf2(message);
|
|
3144
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
3145
|
+
if (messageText(message) !== content) continue;
|
|
3146
|
+
const created = createdOf(message) ?? 0;
|
|
3147
|
+
if (best === null || created > best.created) best = { id, created };
|
|
3148
|
+
}
|
|
3149
|
+
return best?.id ?? null;
|
|
3150
|
+
}
|
|
3151
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2246
3152
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2247
3153
|
const knownUserIds = new Set(
|
|
2248
|
-
(before ?? []).filter((m) =>
|
|
3154
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2249
3155
|
);
|
|
3156
|
+
if (callbacks) {
|
|
3157
|
+
try {
|
|
3158
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
3159
|
+
} catch (error2) {
|
|
3160
|
+
callbacks.onKnownUserIdsError(error2);
|
|
3161
|
+
}
|
|
3162
|
+
}
|
|
2250
3163
|
const parts = [{ type: "text", text: content }];
|
|
2251
3164
|
let pendingOutcomes = null;
|
|
2252
3165
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2285,25 +3198,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2285
3198
|
const READ_BACK_DELAY_MS = 150;
|
|
2286
3199
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2287
3200
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
const id = idOf(m);
|
|
2293
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2294
|
-
if (messageText(m) !== content) continue;
|
|
2295
|
-
const created = createdOf(m) ?? 0;
|
|
2296
|
-
if (best === null || created > best.created) {
|
|
2297
|
-
best = { id, created };
|
|
2298
|
-
}
|
|
2299
|
-
}
|
|
2300
|
-
if (best) {
|
|
2301
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2302
|
-
return best.id;
|
|
2303
|
-
}
|
|
3201
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
3202
|
+
if (dispatchedUserMessageId !== null) {
|
|
3203
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
3204
|
+
return dispatchedUserMessageId;
|
|
2304
3205
|
}
|
|
2305
3206
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2306
|
-
await new Promise((
|
|
3207
|
+
await new Promise((resolve5) => setTimeout(resolve5, READ_BACK_DELAY_MS));
|
|
2307
3208
|
}
|
|
2308
3209
|
}
|
|
2309
3210
|
return null;
|
|
@@ -2311,57 +3212,68 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2311
3212
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2312
3213
|
if (!messages || messages.length === 0) return null;
|
|
2313
3214
|
const byParent = messages.find(
|
|
2314
|
-
(m) =>
|
|
3215
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2315
3216
|
);
|
|
2316
3217
|
if (byParent) return byParent;
|
|
2317
|
-
const userIndex = messages.findIndex((m) =>
|
|
3218
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2318
3219
|
if (userIndex === -1) return null;
|
|
2319
3220
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2320
|
-
if (
|
|
3221
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2321
3222
|
}
|
|
2322
3223
|
return null;
|
|
2323
3224
|
}
|
|
2324
3225
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
2325
3226
|
if (!messages || messages.length === 0) return null;
|
|
2326
3227
|
let lastCorrelated = null;
|
|
2327
|
-
let lastNonErrored = null;
|
|
2328
3228
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2329
3229
|
const m = messages[i];
|
|
2330
|
-
if (
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
3230
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
3231
|
+
lastCorrelated = m;
|
|
3232
|
+
break;
|
|
3233
|
+
}
|
|
3234
|
+
if (lastCorrelated) {
|
|
3235
|
+
if (errorOf(lastCorrelated) == null) return lastCorrelated;
|
|
3236
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
3237
|
+
const m = messages[i];
|
|
3238
|
+
if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
|
|
3239
|
+
return m;
|
|
3240
|
+
}
|
|
2335
3241
|
}
|
|
3242
|
+
return lastCorrelated;
|
|
2336
3243
|
}
|
|
2337
|
-
|
|
2338
|
-
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
3244
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2339
3245
|
if (userIndex === -1) return null;
|
|
2340
3246
|
let last = null;
|
|
2341
|
-
let
|
|
3247
|
+
let lastAssistantIndex = -1;
|
|
2342
3248
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2343
|
-
const role =
|
|
3249
|
+
const role = roleOf2(messages[i]);
|
|
2344
3250
|
if (role === "user") break;
|
|
2345
3251
|
if (role === "assistant") {
|
|
2346
3252
|
last = messages[i];
|
|
2347
|
-
|
|
3253
|
+
lastAssistantIndex = i;
|
|
2348
3254
|
}
|
|
2349
3255
|
}
|
|
2350
|
-
|
|
3256
|
+
if (last === null || errorOf(last) == null) return last;
|
|
3257
|
+
for (let i = lastAssistantIndex; i > userIndex; i--) {
|
|
3258
|
+
const m = messages[i];
|
|
3259
|
+
if (roleOf2(m) === "user") break;
|
|
3260
|
+
if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
|
|
3261
|
+
}
|
|
3262
|
+
return last;
|
|
2351
3263
|
}
|
|
2352
3264
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2353
3265
|
if (!messages || messages.length === 0) return [];
|
|
2354
3266
|
const byParent = messages.filter(
|
|
2355
|
-
(message) =>
|
|
3267
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2356
3268
|
);
|
|
2357
3269
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2358
3270
|
if (assistants.length === 0) {
|
|
2359
|
-
const userIndex = messages.findIndex((message) =>
|
|
3271
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2360
3272
|
if (userIndex === -1) return [];
|
|
2361
3273
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2362
3274
|
const message = messages[i];
|
|
2363
|
-
if (
|
|
2364
|
-
if (
|
|
3275
|
+
if (roleOf2(message) === "user") break;
|
|
3276
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2365
3277
|
}
|
|
2366
3278
|
}
|
|
2367
3279
|
const refs = [];
|
|
@@ -2401,7 +3313,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2401
3313
|
if (!messages || messages.length === 0) return [];
|
|
2402
3314
|
const calls = [];
|
|
2403
3315
|
for (const message of messages) {
|
|
2404
|
-
if (
|
|
3316
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2405
3317
|
for (const part of message.parts ?? []) {
|
|
2406
3318
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2407
3319
|
continue;
|
|
@@ -2431,7 +3343,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2431
3343
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2432
3344
|
const unattributed = [];
|
|
2433
3345
|
for (const message of messages ?? []) {
|
|
2434
|
-
if (
|
|
3346
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2435
3347
|
const created = finiteNumber(createdOf(message));
|
|
2436
3348
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2437
3349
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2453,8 +3365,9 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2453
3365
|
}
|
|
2454
3366
|
function sumAssistantUsage(messages) {
|
|
2455
3367
|
if (!messages || messages.length === 0) return null;
|
|
2456
|
-
const
|
|
2457
|
-
const
|
|
3368
|
+
const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
|
|
3369
|
+
const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
|
|
3370
|
+
const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
|
|
2458
3371
|
let sawAnyUsage = false;
|
|
2459
3372
|
let inputSum = 0;
|
|
2460
3373
|
let outputSum = 0;
|
|
@@ -2463,9 +3376,11 @@ function sumAssistantUsage(messages) {
|
|
|
2463
3376
|
let cacheWriteSum = 0;
|
|
2464
3377
|
let costSum = 0;
|
|
2465
3378
|
let sawCost = false;
|
|
3379
|
+
let toolCallCount = 0;
|
|
2466
3380
|
let modelId = null;
|
|
2467
3381
|
let providerId = null;
|
|
2468
3382
|
for (const m of selected) {
|
|
3383
|
+
toolCallCount += m.parts?.filter((part) => part.type === "tool").length ?? 0;
|
|
2469
3384
|
const info = m.info;
|
|
2470
3385
|
if (!info) continue;
|
|
2471
3386
|
const tokens = info.tokens;
|
|
@@ -2502,19 +3417,18 @@ function sumAssistantUsage(messages) {
|
|
|
2502
3417
|
usage_tokens_cache_write: cacheWriteSum,
|
|
2503
3418
|
// NULL means OpenCode never reported a cost; it is distinct from a genuine
|
|
2504
3419
|
// zero-cost message, which sets `sawCost` with `costSum === 0`.
|
|
2505
|
-
usage_cost_usd: sawCost ? costSum : null
|
|
3420
|
+
usage_cost_usd: sawCost ? costSum : null,
|
|
3421
|
+
turn_tool_calls: toolCallCount
|
|
2506
3422
|
};
|
|
2507
3423
|
}
|
|
2508
3424
|
function messageUsage(messages, userMessageId) {
|
|
2509
3425
|
if (!messages || messages.length === 0) return null;
|
|
2510
3426
|
const byParentAll = messages.filter(
|
|
2511
|
-
(m) =>
|
|
3427
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2512
3428
|
);
|
|
2513
|
-
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2514
|
-
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
2515
3429
|
let correlated;
|
|
2516
|
-
if (
|
|
2517
|
-
correlated =
|
|
3430
|
+
if (byParentAll.length > 0) {
|
|
3431
|
+
correlated = byParentAll;
|
|
2518
3432
|
} else {
|
|
2519
3433
|
const reply = findAssistantReplyAfter(messages, userMessageId);
|
|
2520
3434
|
correlated = reply ? [reply] : [];
|
|
@@ -2524,7 +3438,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2524
3438
|
}
|
|
2525
3439
|
function messageRunState(messages, userMessageId) {
|
|
2526
3440
|
if (!messages || messages.length === 0) return "unknown";
|
|
2527
|
-
const hasUser = messages.some((m) =>
|
|
3441
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2528
3442
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2529
3443
|
if (!hasUser) {
|
|
2530
3444
|
if (!reply) return "unknown";
|
|
@@ -2560,14 +3474,38 @@ function messageError(messages, userMessageId) {
|
|
|
2560
3474
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2561
3475
|
const error2 = errorOf(reply);
|
|
2562
3476
|
if (error2 == null) return null;
|
|
2563
|
-
|
|
2564
|
-
if (typeof error2 === "
|
|
3477
|
+
let rendered;
|
|
3478
|
+
if (typeof error2 === "string") {
|
|
3479
|
+
rendered = error2;
|
|
3480
|
+
} else if (typeof error2 === "object") {
|
|
2565
3481
|
const e = error2;
|
|
2566
3482
|
const dataMessage = e.data?.message;
|
|
2567
|
-
if (typeof dataMessage === "string")
|
|
2568
|
-
|
|
3483
|
+
if (typeof dataMessage === "string") {
|
|
3484
|
+
rendered = dataMessage;
|
|
3485
|
+
} else if (typeof e.message === "string") {
|
|
3486
|
+
rendered = e.message;
|
|
3487
|
+
} else {
|
|
3488
|
+
rendered = "The agent run failed.";
|
|
3489
|
+
}
|
|
3490
|
+
} else {
|
|
3491
|
+
rendered = "The agent run failed.";
|
|
2569
3492
|
}
|
|
2570
|
-
|
|
3493
|
+
const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
|
|
3494
|
+
(part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
|
|
3495
|
+
) : void 0;
|
|
3496
|
+
return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
|
|
3497
|
+
}
|
|
3498
|
+
function describeAbandonedReplyShape(messages, userMessageId) {
|
|
3499
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
3500
|
+
if (!reply) return null;
|
|
3501
|
+
const parts = Array.isArray(reply.parts) ? reply.parts : [];
|
|
3502
|
+
const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
|
|
3503
|
+
return {
|
|
3504
|
+
partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
|
|
3505
|
+
droppedPartCount: Math.max(0, parts.length - 12),
|
|
3506
|
+
lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
|
|
3507
|
+
lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
|
|
3508
|
+
};
|
|
2571
3509
|
}
|
|
2572
3510
|
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2573
3511
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
@@ -2584,19 +3522,32 @@ function isAbortedTerminalReply(messages, userMessageId) {
|
|
|
2584
3522
|
}
|
|
2585
3523
|
return false;
|
|
2586
3524
|
}
|
|
3525
|
+
var CLAUDE_AUTH_EXPIRY_MESSAGES = Object.freeze([
|
|
3526
|
+
"Claude Code credentials are unavailable or expired. Run `claude` to refresh them."
|
|
3527
|
+
]);
|
|
3528
|
+
function extractReplyErrorShape(reply) {
|
|
3529
|
+
const error2 = errorOf(reply);
|
|
3530
|
+
if (error2 == null || typeof error2 !== "object") {
|
|
3531
|
+
return { name: null, statusCode: null };
|
|
3532
|
+
}
|
|
3533
|
+
const e = error2;
|
|
3534
|
+
const name = typeof e.name === "string" ? e.name.slice(0, 100) : null;
|
|
3535
|
+
const data = e.data;
|
|
3536
|
+
const statusCode = typeof data?.statusCode === "number" && Number.isFinite(data.statusCode) ? data.statusCode : null;
|
|
3537
|
+
return { name, statusCode };
|
|
3538
|
+
}
|
|
2587
3539
|
function classifyReplyAuthError(reply) {
|
|
2588
3540
|
const error2 = errorOf(reply);
|
|
2589
3541
|
if (error2 == null || typeof error2 !== "object") return null;
|
|
2590
3542
|
const e = error2;
|
|
2591
3543
|
const replyProviderId = reply?.info?.providerID ?? null;
|
|
2592
3544
|
const replyModelId = reply?.info?.modelID ?? null;
|
|
3545
|
+
const data = e.data;
|
|
2593
3546
|
if (e.name === "ProviderAuthError") {
|
|
2594
|
-
const data = e.data;
|
|
2595
3547
|
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
2596
3548
|
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
2597
3549
|
}
|
|
2598
3550
|
if (e.name === "APIError") {
|
|
2599
|
-
const data = e.data;
|
|
2600
3551
|
const statusCode = data?.statusCode;
|
|
2601
3552
|
if (statusCode === 401 || statusCode === 403) {
|
|
2602
3553
|
return {
|
|
@@ -2607,16 +3558,28 @@ function classifyReplyAuthError(reply) {
|
|
|
2607
3558
|
};
|
|
2608
3559
|
}
|
|
2609
3560
|
}
|
|
3561
|
+
const errorMessage3 = typeof data?.message === "string" ? data.message : typeof e.message === "string" ? e.message : null;
|
|
3562
|
+
if (errorMessage3 !== null && CLAUDE_AUTH_EXPIRY_MESSAGES.some((knownMessage) => errorMessage3.includes(knownMessage))) {
|
|
3563
|
+
return {
|
|
3564
|
+
kind: "model_auth",
|
|
3565
|
+
providerId: typeof data?.providerID === "string" && data.providerID || replyProviderId,
|
|
3566
|
+
modelId: replyModelId,
|
|
3567
|
+
reason: "rejected"
|
|
3568
|
+
};
|
|
3569
|
+
}
|
|
2610
3570
|
return null;
|
|
2611
3571
|
}
|
|
2612
3572
|
function messageFailure(messages, userMessageId) {
|
|
2613
3573
|
return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
|
|
2614
3574
|
}
|
|
3575
|
+
function isProviderModelNotFoundError(reason) {
|
|
3576
|
+
return reason.includes("Model not found:");
|
|
3577
|
+
}
|
|
2615
3578
|
function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
2616
3579
|
if (!messages || messages.length === 0) return null;
|
|
2617
3580
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2618
3581
|
const message = messages[i];
|
|
2619
|
-
if (
|
|
3582
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2620
3583
|
const created = createdOf(message);
|
|
2621
3584
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2622
3585
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2648,25 +3611,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2648
3611
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2649
3612
|
if (!messages || messages.length === 0) return false;
|
|
2650
3613
|
return messages.some(
|
|
2651
|
-
(m) =>
|
|
3614
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2652
3615
|
);
|
|
2653
3616
|
}
|
|
2654
3617
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2655
3618
|
if (!messages || messages.length === 0) return false;
|
|
2656
|
-
const userIndex = messages.findIndex((message) =>
|
|
3619
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2657
3620
|
if (userIndex === -1) return false;
|
|
2658
3621
|
let hasLaterUser = false;
|
|
2659
3622
|
let hasStartedLaterUser = false;
|
|
2660
3623
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2661
3624
|
const message = messages[i];
|
|
2662
|
-
if (
|
|
3625
|
+
if (roleOf2(message) !== "user") continue;
|
|
2663
3626
|
hasLaterUser = true;
|
|
2664
|
-
const laterUserMessageId =
|
|
3627
|
+
const laterUserMessageId = idOf2(message);
|
|
2665
3628
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2666
3629
|
return false;
|
|
2667
3630
|
}
|
|
2668
3631
|
if (messages.some(
|
|
2669
|
-
(candidate) =>
|
|
3632
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2670
3633
|
)) {
|
|
2671
3634
|
hasStartedLaterUser = true;
|
|
2672
3635
|
}
|
|
@@ -2820,23 +3783,49 @@ async function readSessionErrorStream(port, options, client) {
|
|
|
2820
3783
|
if (reader) void reader.cancel().catch(() => void 0);
|
|
2821
3784
|
}
|
|
2822
3785
|
}
|
|
2823
|
-
async function
|
|
2824
|
-
if (client?.version === "v2") return;
|
|
3786
|
+
async function disposeAndReloadProviderConfig(port, client) {
|
|
3787
|
+
if (client?.version === "v2") return { outcome: "skipped_v2" };
|
|
2825
3788
|
try {
|
|
2826
|
-
const res = await requestWithClient(
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
3789
|
+
const res = await requestWithClient(
|
|
3790
|
+
port,
|
|
3791
|
+
client,
|
|
3792
|
+
"/config",
|
|
3793
|
+
{
|
|
3794
|
+
method: "PATCH",
|
|
3795
|
+
headers: { "Content-Type": "application/json" },
|
|
3796
|
+
body: JSON.stringify({})
|
|
3797
|
+
},
|
|
3798
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
3799
|
+
);
|
|
2831
3800
|
if (!res.ok) {
|
|
2832
3801
|
console.error(
|
|
2833
|
-
`[
|
|
3802
|
+
`[disposeAndReloadProviderConfig] PATCH /config returned HTTP ${res.status} (port ${port})`
|
|
2834
3803
|
);
|
|
3804
|
+
return { outcome: "failed", status: res.status, detail: null };
|
|
2835
3805
|
}
|
|
3806
|
+
return { outcome: "reloaded" };
|
|
2836
3807
|
} catch (err) {
|
|
2837
3808
|
console.error(
|
|
2838
|
-
`[
|
|
3809
|
+
`[disposeAndReloadProviderConfig] PATCH /config failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
2839
3810
|
);
|
|
3811
|
+
return {
|
|
3812
|
+
outcome: "failed",
|
|
3813
|
+
status: null,
|
|
3814
|
+
detail: err instanceof Error ? err.message : String(err)
|
|
3815
|
+
};
|
|
3816
|
+
}
|
|
3817
|
+
}
|
|
3818
|
+
async function probeOpenCodeRequestReadiness(port, client) {
|
|
3819
|
+
try {
|
|
3820
|
+
const res = await requestWithClient(port, client, "/config", void 0, {
|
|
3821
|
+
allowStatuses: ALL_HTTP_STATUSES
|
|
3822
|
+
});
|
|
3823
|
+
const body = await res.text();
|
|
3824
|
+
return { ok: res.ok, status: res.status, bodyEmpty: body.trim().length === 0, detail: null };
|
|
3825
|
+
} catch (err) {
|
|
3826
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
3827
|
+
console.error(`[probeOpenCodeRequestReadiness] GET /config failed (port ${port}): ${detail}`);
|
|
3828
|
+
return { ok: false, status: null, bodyEmpty: null, detail };
|
|
2840
3829
|
}
|
|
2841
3830
|
}
|
|
2842
3831
|
|
|
@@ -3137,13 +4126,28 @@ async function listV2Sessions(client) {
|
|
|
3137
4126
|
}
|
|
3138
4127
|
async function deleteV2Session(client, sessionId) {
|
|
3139
4128
|
try {
|
|
3140
|
-
await client.request(
|
|
3141
|
-
|
|
4129
|
+
const response = await client.request(
|
|
4130
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
4131
|
+
{ method: "DELETE" },
|
|
4132
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
4133
|
+
);
|
|
4134
|
+
if (response.status >= 200 && response.status < 300) {
|
|
4135
|
+
return { ok: true, status: response.status };
|
|
4136
|
+
}
|
|
4137
|
+
try {
|
|
4138
|
+
const body = await response.text();
|
|
4139
|
+
return body ? { ok: false, status: response.status, body } : { ok: false, status: response.status };
|
|
4140
|
+
} catch (error2) {
|
|
4141
|
+
console.error(
|
|
4142
|
+
`[deleteV2Session] DELETE /api/session/${sessionId} response body read failed for HTTP ${response.status}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
4143
|
+
);
|
|
4144
|
+
return { ok: false, status: response.status };
|
|
4145
|
+
}
|
|
3142
4146
|
} catch (error2) {
|
|
3143
4147
|
console.error(
|
|
3144
4148
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3145
4149
|
);
|
|
3146
|
-
return false;
|
|
4150
|
+
return { ok: false, status: "threw" };
|
|
3147
4151
|
}
|
|
3148
4152
|
}
|
|
3149
4153
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3314,12 +4318,12 @@ async function readV2SessionErrorStream(client, options) {
|
|
|
3314
4318
|
|
|
3315
4319
|
// src/lib/opencode/credential-import-v2.ts
|
|
3316
4320
|
import { readFileSync as readFileSync4 } from "node:fs";
|
|
3317
|
-
import { join as
|
|
4321
|
+
import { join as join5 } from "node:path";
|
|
3318
4322
|
|
|
3319
4323
|
// src/lib/openai-usage.ts
|
|
3320
4324
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3321
4325
|
import { homedir as homedir3 } from "node:os";
|
|
3322
|
-
import { join as
|
|
4326
|
+
import { join as join4 } from "node:path";
|
|
3323
4327
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3324
4328
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
3325
4329
|
var OpenAiUsageError = class extends Error {
|
|
@@ -3333,7 +4337,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3333
4337
|
}
|
|
3334
4338
|
function readOpenCodeChatGptCredentials() {
|
|
3335
4339
|
try {
|
|
3336
|
-
const raw = readFileSync3(
|
|
4340
|
+
const raw = readFileSync3(join4(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3337
4341
|
let parsed;
|
|
3338
4342
|
try {
|
|
3339
4343
|
parsed = JSON.parse(raw);
|
|
@@ -3474,16 +4478,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3474
4478
|
].some((name) => headers.has(name));
|
|
3475
4479
|
}
|
|
3476
4480
|
async function getOpenAiUsage(port, client) {
|
|
4481
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3477
4482
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3478
4483
|
if (!credentials2) {
|
|
3479
4484
|
throw new OpenAiUsageError(
|
|
3480
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
4485
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3481
4486
|
"no_credentials"
|
|
3482
4487
|
);
|
|
3483
4488
|
}
|
|
3484
4489
|
if (credentials2.expiresAt < Date.now()) {
|
|
3485
4490
|
throw new OpenAiUsageError(
|
|
3486
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
4491
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3487
4492
|
"credentials_expired"
|
|
3488
4493
|
);
|
|
3489
4494
|
}
|
|
@@ -3558,13 +4563,13 @@ var INTEGRATION_LIST_RETRY_DELAY_MS = 1e3;
|
|
|
3558
4563
|
function isRecord3(value) {
|
|
3559
4564
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3560
4565
|
}
|
|
3561
|
-
var sleep2 = (milliseconds) => new Promise((
|
|
4566
|
+
var sleep2 = (milliseconds) => new Promise((resolve5) => setTimeout(resolve5, milliseconds));
|
|
3562
4567
|
function emptyResult(unsupported = [], outcome = "completed") {
|
|
3563
4568
|
return { outcome, imported: [], alreadyConnected: [], unsupported, failed: [] };
|
|
3564
4569
|
}
|
|
3565
4570
|
function readOpenCodeCredentials(homeDir, log3) {
|
|
3566
4571
|
try {
|
|
3567
|
-
const raw = readFileSync4(
|
|
4572
|
+
const raw = readFileSync4(join5(homeDir, ...OPENCODE_AUTH_SEGMENTS), "utf8");
|
|
3568
4573
|
let parsed;
|
|
3569
4574
|
try {
|
|
3570
4575
|
parsed = JSON.parse(raw);
|
|
@@ -3764,9 +4769,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3764
4769
|
|
|
3765
4770
|
// src/lib/opencode/session-db-boot.ts
|
|
3766
4771
|
import { spawn as spawn2 } from "node:child_process";
|
|
3767
|
-
import { mkdirSync, statSync as
|
|
4772
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3768
4773
|
import { homedir as homedir4 } from "node:os";
|
|
3769
|
-
import { dirname as
|
|
4774
|
+
import { dirname as dirname4, resolve as resolvePath } from "node:path";
|
|
3770
4775
|
|
|
3771
4776
|
// src/lib/runner-synchroniser.ts
|
|
3772
4777
|
import { spawn } from "node:child_process";
|
|
@@ -3776,7 +4781,7 @@ function appendError(stderr, error2) {
|
|
|
3776
4781
|
${message}`;
|
|
3777
4782
|
}
|
|
3778
4783
|
function runSynchroniser(args, opts) {
|
|
3779
|
-
return new Promise((
|
|
4784
|
+
return new Promise((resolve5) => {
|
|
3780
4785
|
let child;
|
|
3781
4786
|
let stdout = "";
|
|
3782
4787
|
let stderr = "";
|
|
@@ -3790,7 +4795,7 @@ function runSynchroniser(args, opts) {
|
|
|
3790
4795
|
if (timer.handle) clearTimeout(timer.handle);
|
|
3791
4796
|
if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
|
|
3792
4797
|
if (spawnListener) child.removeListener("spawn", spawnListener);
|
|
3793
|
-
|
|
4798
|
+
resolve5(result);
|
|
3794
4799
|
};
|
|
3795
4800
|
try {
|
|
3796
4801
|
child = spawn("runner-synchroniser", args, {
|
|
@@ -3846,9 +4851,12 @@ function runSynchroniser(args, opts) {
|
|
|
3846
4851
|
|
|
3847
4852
|
// src/lib/opencode/session-db-boot.ts
|
|
3848
4853
|
var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
|
|
3849
|
-
var SESSION_DB_VERIFY_TIMEOUT_MS =
|
|
4854
|
+
var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
|
|
3850
4855
|
var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
|
|
3851
|
-
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS =
|
|
4856
|
+
var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
|
|
4857
|
+
var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
|
|
4858
|
+
var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
|
|
4859
|
+
var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
|
|
3852
4860
|
function commandError(result) {
|
|
3853
4861
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
3854
4862
|
}
|
|
@@ -3891,7 +4899,7 @@ function clearMarker(options) {
|
|
|
3891
4899
|
function markNoReplicate(options, message) {
|
|
3892
4900
|
if (options.noReplicateMarker) {
|
|
3893
4901
|
try {
|
|
3894
|
-
mkdirSync(
|
|
4902
|
+
mkdirSync(dirname4(options.noReplicateMarker), { recursive: true });
|
|
3895
4903
|
writeFileSync(options.noReplicateMarker, "");
|
|
3896
4904
|
} catch (error2) {
|
|
3897
4905
|
options.log(
|
|
@@ -3950,7 +4958,7 @@ function parseSynchroniserEnv(stdout) {
|
|
|
3950
4958
|
return values;
|
|
3951
4959
|
}
|
|
3952
4960
|
function runCommand(command, args, options) {
|
|
3953
|
-
return new Promise((
|
|
4961
|
+
return new Promise((resolve5) => {
|
|
3954
4962
|
let child;
|
|
3955
4963
|
let stdout = "";
|
|
3956
4964
|
let stderr = "";
|
|
@@ -3959,7 +4967,7 @@ function runCommand(command, args, options) {
|
|
|
3959
4967
|
if (settled) return;
|
|
3960
4968
|
settled = true;
|
|
3961
4969
|
if (timer) clearTimeout(timer);
|
|
3962
|
-
|
|
4970
|
+
resolve5(result);
|
|
3963
4971
|
};
|
|
3964
4972
|
try {
|
|
3965
4973
|
child = spawn2(command, args, {
|
|
@@ -3967,7 +4975,7 @@ function runCommand(command, args, options) {
|
|
|
3967
4975
|
stdio: ["ignore", "pipe", "pipe"]
|
|
3968
4976
|
});
|
|
3969
4977
|
} catch (error2) {
|
|
3970
|
-
|
|
4978
|
+
resolve5({
|
|
3971
4979
|
code: null,
|
|
3972
4980
|
stdout,
|
|
3973
4981
|
stderr: error2 instanceof Error ? error2.message : String(error2),
|
|
@@ -4016,7 +5024,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4016
5024
|
return null;
|
|
4017
5025
|
}
|
|
4018
5026
|
try {
|
|
4019
|
-
if (
|
|
5027
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
4020
5028
|
} catch (error2) {
|
|
4021
5029
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
4022
5030
|
options.log(
|
|
@@ -4046,7 +5054,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
4046
5054
|
return null;
|
|
4047
5055
|
}
|
|
4048
5056
|
try {
|
|
4049
|
-
mkdirSync(
|
|
5057
|
+
mkdirSync(dirname4(configPath), { recursive: true });
|
|
4050
5058
|
writeFileSync(configPath, rendered.stdout);
|
|
4051
5059
|
} catch (error2) {
|
|
4052
5060
|
options.log(
|
|
@@ -4159,20 +5167,37 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4159
5167
|
timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
|
|
4160
5168
|
env: {
|
|
4161
5169
|
...env,
|
|
4162
|
-
// The synchroniser reads
|
|
4163
|
-
// 120_000, so the walkback gives up before the outer process bound.
|
|
5170
|
+
// The synchroniser reads all four values in SECONDS.
|
|
4164
5171
|
EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
|
|
4165
5172
|
SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
|
|
5173
|
+
),
|
|
5174
|
+
EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
|
|
5175
|
+
SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
|
|
5176
|
+
),
|
|
5177
|
+
EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
|
|
5178
|
+
SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
|
|
5179
|
+
),
|
|
5180
|
+
EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
|
|
5181
|
+
SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
|
|
4166
5182
|
)
|
|
4167
5183
|
}
|
|
4168
5184
|
});
|
|
4169
5185
|
logSynchroniserDiagnostics(result, options);
|
|
4170
5186
|
if (result.timedOut || result.code === 124 || result.code === 137) {
|
|
5187
|
+
if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
|
|
5188
|
+
const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
|
|
5189
|
+
options.log(
|
|
5190
|
+
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
|
|
5191
|
+
"error"
|
|
5192
|
+
);
|
|
5193
|
+
reportRecord("verify", "session_db_boot_refused", reason, null, options);
|
|
5194
|
+
return { verifyFatal: true, verifyFatalReason: reason };
|
|
5195
|
+
}
|
|
4171
5196
|
options.log(
|
|
4172
5197
|
`SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
|
|
4173
5198
|
"warn"
|
|
4174
5199
|
);
|
|
4175
|
-
return false;
|
|
5200
|
+
return { verifyFatal: false };
|
|
4176
5201
|
}
|
|
4177
5202
|
if (result.code === 34) {
|
|
4178
5203
|
reportRecord(
|
|
@@ -4182,14 +5207,14 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4182
5207
|
null,
|
|
4183
5208
|
options
|
|
4184
5209
|
);
|
|
4185
|
-
return true;
|
|
5210
|
+
return { verifyFatal: true };
|
|
4186
5211
|
}
|
|
4187
5212
|
if (result.code === 33) {
|
|
4188
5213
|
options.log(
|
|
4189
5214
|
"SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
|
|
4190
5215
|
"warn"
|
|
4191
5216
|
);
|
|
4192
|
-
return false;
|
|
5217
|
+
return { verifyFatal: false };
|
|
4193
5218
|
}
|
|
4194
5219
|
if (result.code !== 0) {
|
|
4195
5220
|
options.log(
|
|
@@ -4197,17 +5222,17 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4197
5222
|
"warn"
|
|
4198
5223
|
);
|
|
4199
5224
|
}
|
|
4200
|
-
return false;
|
|
5225
|
+
return { verifyFatal: false };
|
|
4201
5226
|
}
|
|
4202
5227
|
options.log(
|
|
4203
5228
|
"skipping session-DB verification: this boot's session DB was not proven safe to replicate",
|
|
4204
5229
|
"debug"
|
|
4205
5230
|
);
|
|
4206
|
-
return false;
|
|
5231
|
+
return { verifyFatal: false };
|
|
4207
5232
|
}
|
|
4208
5233
|
function fileExists(path) {
|
|
4209
5234
|
try {
|
|
4210
|
-
|
|
5235
|
+
statSync3(path);
|
|
4211
5236
|
return true;
|
|
4212
5237
|
} catch (error2) {
|
|
4213
5238
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4290,13 +5315,13 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4290
5315
|
if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
|
|
4291
5316
|
return { verifyFatal: false };
|
|
4292
5317
|
}
|
|
4293
|
-
return
|
|
5318
|
+
return verifySessionDb(options, configPath, env);
|
|
4294
5319
|
}
|
|
4295
5320
|
|
|
4296
5321
|
// src/lib/opencode/session-db-provenance.ts
|
|
4297
5322
|
import { createRequire } from "node:module";
|
|
4298
5323
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4299
|
-
import { dirname as
|
|
5324
|
+
import { dirname as dirname5, join as join6 } from "node:path";
|
|
4300
5325
|
var require2 = createRequire(import.meta.url);
|
|
4301
5326
|
function readSessionDbMigrationIds(dbPath) {
|
|
4302
5327
|
let db;
|
|
@@ -4333,7 +5358,7 @@ function readSessionDbMigrationIds(dbPath) {
|
|
|
4333
5358
|
}
|
|
4334
5359
|
function sessionDbProvenanceStatePath(homeDir, env) {
|
|
4335
5360
|
const override = env.EVIDENT_SESSION_DB_PROVENANCE_STATE?.trim();
|
|
4336
|
-
return override ||
|
|
5361
|
+
return override || join6(homeDir, ".local", "state", "evident", "session-db-provenance.json");
|
|
4337
5362
|
}
|
|
4338
5363
|
function loadSessionDbProvenanceState(path) {
|
|
4339
5364
|
let value;
|
|
@@ -4362,7 +5387,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4362
5387
|
}
|
|
4363
5388
|
function saveSessionDbProvenanceState(path, state) {
|
|
4364
5389
|
try {
|
|
4365
|
-
mkdirSync2(
|
|
5390
|
+
mkdirSync2(dirname5(path), { recursive: true });
|
|
4366
5391
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4367
5392
|
`, "utf8");
|
|
4368
5393
|
} catch (error2) {
|
|
@@ -4453,7 +5478,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4453
5478
|
if (child.exitCode !== null || child.signalCode !== null) {
|
|
4454
5479
|
return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
|
|
4455
5480
|
}
|
|
4456
|
-
return new Promise((
|
|
5481
|
+
return new Promise((resolve5, reject) => {
|
|
4457
5482
|
let forced = false;
|
|
4458
5483
|
let settled = false;
|
|
4459
5484
|
const timer = setTimeout(() => {
|
|
@@ -4473,7 +5498,7 @@ async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
|
|
|
4473
5498
|
settled = true;
|
|
4474
5499
|
clearTimeout(timer);
|
|
4475
5500
|
child.removeListener("exit", onExit);
|
|
4476
|
-
|
|
5501
|
+
resolve5(result);
|
|
4477
5502
|
};
|
|
4478
5503
|
const fail = (error2) => {
|
|
4479
5504
|
if (settled) return;
|
|
@@ -4943,10 +5968,13 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4943
5968
|
toDelete.push(s.id);
|
|
4944
5969
|
}
|
|
4945
5970
|
}
|
|
4946
|
-
|
|
5971
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5972
|
+
return toDelete.sort(
|
|
5973
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5974
|
+
);
|
|
4947
5975
|
}
|
|
4948
5976
|
var DEFAULT_INTERVAL = "1h";
|
|
4949
|
-
function
|
|
5977
|
+
function resolve2(flag, envValue, fallback) {
|
|
4950
5978
|
return flag ?? envValue ?? fallback;
|
|
4951
5979
|
}
|
|
4952
5980
|
function parseMaxCount(input) {
|
|
@@ -4962,9 +5990,9 @@ function parseMaxCount(input) {
|
|
|
4962
5990
|
}
|
|
4963
5991
|
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
4964
5992
|
const warnings = [];
|
|
4965
|
-
const maxAgeRaw =
|
|
4966
|
-
const maxCountRaw =
|
|
4967
|
-
const intervalRaw =
|
|
5993
|
+
const maxAgeRaw = resolve2(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
5994
|
+
const maxCountRaw = resolve2(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
5995
|
+
const intervalRaw = resolve2(
|
|
4968
5996
|
flags.interval,
|
|
4969
5997
|
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
4970
5998
|
DEFAULT_INTERVAL
|
|
@@ -5003,13 +6031,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
5003
6031
|
}
|
|
5004
6032
|
|
|
5005
6033
|
// src/lib/opencode/session-db-size.ts
|
|
5006
|
-
import { statSync as
|
|
5007
|
-
import { join as
|
|
6034
|
+
import { statSync as statSync4 } from "node:fs";
|
|
6035
|
+
import { join as join7 } from "node:path";
|
|
5008
6036
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
5009
6037
|
function statSessionDbBytes(homeDir) {
|
|
5010
|
-
const dbPath =
|
|
6038
|
+
const dbPath = join7(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
5011
6039
|
try {
|
|
5012
|
-
return
|
|
6040
|
+
return statSync4(dbPath).size;
|
|
5013
6041
|
} catch (err) {
|
|
5014
6042
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
5015
6043
|
if (!isMissingFile) {
|
|
@@ -5035,14 +6063,14 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
5035
6063
|
}
|
|
5036
6064
|
|
|
5037
6065
|
// src/lib/opencode/log-tail.ts
|
|
5038
|
-
import { statSync as
|
|
6066
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5039
6067
|
import { homedir as homedir5 } from "node:os";
|
|
5040
|
-
import { join as
|
|
6068
|
+
import { join as join8 } from "node:path";
|
|
5041
6069
|
import { open as open2, stat } from "node:fs/promises";
|
|
5042
6070
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
5043
6071
|
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
5044
|
-
const dataDir = env.XDG_DATA_HOME ||
|
|
5045
|
-
return
|
|
6072
|
+
const dataDir = env.XDG_DATA_HOME || join8(homeDir, ".local", "share");
|
|
6073
|
+
return join8(dataDir, "opencode", "log", "opencode.log");
|
|
5046
6074
|
}
|
|
5047
6075
|
function isEnoent(error2) {
|
|
5048
6076
|
return error2?.code === "ENOENT";
|
|
@@ -5058,7 +6086,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5058
6086
|
let inode = null;
|
|
5059
6087
|
let baselineReady = true;
|
|
5060
6088
|
try {
|
|
5061
|
-
const initial =
|
|
6089
|
+
const initial = statSync5(logPath);
|
|
5062
6090
|
offset = initial.size;
|
|
5063
6091
|
inode = initial.ino;
|
|
5064
6092
|
} catch (error2) {
|
|
@@ -5117,129 +6145,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5117
6145
|
};
|
|
5118
6146
|
}
|
|
5119
6147
|
|
|
5120
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5121
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5122
|
-
import { dirname as dirname4 } from "node:path";
|
|
5123
|
-
function errorMessage(error2) {
|
|
5124
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5125
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5126
|
-
}
|
|
5127
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5128
|
-
try {
|
|
5129
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5130
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5131
|
-
if (availableBytes < requiredBytes) {
|
|
5132
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5133
|
-
}
|
|
5134
|
-
return null;
|
|
5135
|
-
} catch (err) {
|
|
5136
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5137
|
-
}
|
|
5138
|
-
}
|
|
5139
|
-
function readLogicalBytes(db) {
|
|
5140
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5141
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5142
|
-
return pageCount * pageSize;
|
|
5143
|
-
}
|
|
5144
|
-
function readCheckpointResult(db) {
|
|
5145
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5146
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5147
|
-
}
|
|
5148
|
-
async function probeReclaimAvailability(input) {
|
|
5149
|
-
const { dbPath, requiredBytes } = input;
|
|
5150
|
-
let sqlite;
|
|
5151
|
-
try {
|
|
5152
|
-
sqlite = await import("node:sqlite");
|
|
5153
|
-
} catch (err) {
|
|
5154
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5155
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5156
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5157
|
-
}
|
|
5158
|
-
let autoVacuum = null;
|
|
5159
|
-
try {
|
|
5160
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5161
|
-
try {
|
|
5162
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5163
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5164
|
-
} finally {
|
|
5165
|
-
db.close();
|
|
5166
|
-
}
|
|
5167
|
-
} catch (err) {
|
|
5168
|
-
console.warn(
|
|
5169
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5170
|
-
);
|
|
5171
|
-
}
|
|
5172
|
-
if (autoVacuum !== 0) return null;
|
|
5173
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5174
|
-
}
|
|
5175
|
-
async function reclaimSessionDbSpace(input) {
|
|
5176
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5177
|
-
let sqlite;
|
|
5178
|
-
try {
|
|
5179
|
-
sqlite = await import("node:sqlite");
|
|
5180
|
-
} catch (err) {
|
|
5181
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5182
|
-
console.warn(
|
|
5183
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5184
|
-
);
|
|
5185
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5186
|
-
}
|
|
5187
|
-
const { DatabaseSync } = sqlite;
|
|
5188
|
-
let db;
|
|
5189
|
-
try {
|
|
5190
|
-
db = new DatabaseSync(dbPath);
|
|
5191
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5192
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5193
|
-
if (autoVacuum === 0) {
|
|
5194
|
-
if (!allowFullVacuum) {
|
|
5195
|
-
console.warn(
|
|
5196
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5197
|
-
);
|
|
5198
|
-
return {
|
|
5199
|
-
ok: false,
|
|
5200
|
-
skipped: "full-vacuum-blocked",
|
|
5201
|
-
detail: "a session turn is live"
|
|
5202
|
-
};
|
|
5203
|
-
}
|
|
5204
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5205
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5206
|
-
if (skipReason !== null) {
|
|
5207
|
-
console.warn(
|
|
5208
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5209
|
-
);
|
|
5210
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5211
|
-
}
|
|
5212
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5213
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5214
|
-
db.exec("VACUUM");
|
|
5215
|
-
const afterBytes = readLogicalBytes(db);
|
|
5216
|
-
const checkpoint = readCheckpointResult(db);
|
|
5217
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5218
|
-
}
|
|
5219
|
-
if (autoVacuum === 2) {
|
|
5220
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5221
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5222
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5223
|
-
const afterBytes = readLogicalBytes(db);
|
|
5224
|
-
const checkpoint = readCheckpointResult(db);
|
|
5225
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5226
|
-
}
|
|
5227
|
-
console.warn(
|
|
5228
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5229
|
-
);
|
|
5230
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5231
|
-
} catch (err) {
|
|
5232
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5233
|
-
return {
|
|
5234
|
-
ok: false,
|
|
5235
|
-
skipped: "reclaim-error",
|
|
5236
|
-
detail: errorMessage(err)
|
|
5237
|
-
};
|
|
5238
|
-
} finally {
|
|
5239
|
-
db?.close();
|
|
5240
|
-
}
|
|
5241
|
-
}
|
|
5242
|
-
|
|
5243
6148
|
// src/lib/tunnel/connection.ts
|
|
5244
6149
|
import WebSocket2 from "ws";
|
|
5245
6150
|
|
|
@@ -5320,8 +6225,8 @@ var StreamForwarder = class {
|
|
|
5320
6225
|
pauseStream(sid, stream) {
|
|
5321
6226
|
if (!stream.paused) {
|
|
5322
6227
|
stream.paused = true;
|
|
5323
|
-
stream.pausePromise = new Promise((
|
|
5324
|
-
stream.resume =
|
|
6228
|
+
stream.pausePromise = new Promise((resolve5) => {
|
|
6229
|
+
stream.resume = resolve5;
|
|
5325
6230
|
});
|
|
5326
6231
|
}
|
|
5327
6232
|
if (stream.pauseTimer !== null) clearTimeout(stream.pauseTimer);
|
|
@@ -5388,12 +6293,12 @@ var StreamForwarder = class {
|
|
|
5388
6293
|
let endBody;
|
|
5389
6294
|
if (has_body) {
|
|
5390
6295
|
const chunks = [];
|
|
5391
|
-
bodyPromise = new Promise((
|
|
6296
|
+
bodyPromise = new Promise((resolve5) => {
|
|
5392
6297
|
pushBody = (buf) => {
|
|
5393
6298
|
chunks.push(buf);
|
|
5394
6299
|
};
|
|
5395
6300
|
endBody = () => {
|
|
5396
|
-
|
|
6301
|
+
resolve5(Buffer.concat(chunks));
|
|
5397
6302
|
};
|
|
5398
6303
|
});
|
|
5399
6304
|
}
|
|
@@ -5469,6 +6374,8 @@ var StreamForwarder = class {
|
|
|
5469
6374
|
if (done) break;
|
|
5470
6375
|
const chunk = Buffer.from(value);
|
|
5471
6376
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
6377
|
+
await stream.pausePromise;
|
|
6378
|
+
if (ac.signal.aborted) break;
|
|
5472
6379
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5473
6380
|
if (this.binaryFramesSupported) {
|
|
5474
6381
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5511,6 +6418,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5511
6418
|
}
|
|
5512
6419
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5513
6420
|
var BASE_RECONNECT_DELAY = 500;
|
|
6421
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5514
6422
|
function getReconnectDelay(attempt) {
|
|
5515
6423
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5516
6424
|
const jitter = Math.random() * 1e3;
|
|
@@ -5561,7 +6469,7 @@ function connectTunnel(options) {
|
|
|
5561
6469
|
} = options;
|
|
5562
6470
|
const tunnelUrl = getTunnelUrlConfig();
|
|
5563
6471
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
5564
|
-
return new Promise((
|
|
6472
|
+
return new Promise((resolve5, reject) => {
|
|
5565
6473
|
const ws = new WebSocket2(url, {
|
|
5566
6474
|
headers: {
|
|
5567
6475
|
Authorization: authHeader,
|
|
@@ -5582,10 +6490,27 @@ function connectTunnel(options) {
|
|
|
5582
6490
|
ws.close();
|
|
5583
6491
|
reject(new Error("Connection timeout"));
|
|
5584
6492
|
}, 3e4);
|
|
6493
|
+
let relaySilenceTimeout;
|
|
6494
|
+
let relaySilenceTimedOut = false;
|
|
6495
|
+
const clearRelaySilenceTimeout = () => {
|
|
6496
|
+
if (relaySilenceTimeout === void 0) return;
|
|
6497
|
+
clearTimeout(relaySilenceTimeout);
|
|
6498
|
+
relaySilenceTimeout = void 0;
|
|
6499
|
+
};
|
|
6500
|
+
const armRelaySilenceTimeout = () => {
|
|
6501
|
+
clearRelaySilenceTimeout();
|
|
6502
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
6503
|
+
relaySilenceTimeout = void 0;
|
|
6504
|
+
relaySilenceTimedOut = true;
|
|
6505
|
+
ws.terminate();
|
|
6506
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
6507
|
+
relaySilenceTimeout.unref?.();
|
|
6508
|
+
};
|
|
5585
6509
|
let upgradeRejection = null;
|
|
5586
6510
|
let upgradeRejectionReason = null;
|
|
5587
6511
|
ws.on("unexpected-response", (_req, res) => {
|
|
5588
6512
|
clearTimeout(connectionTimeout);
|
|
6513
|
+
clearRelaySilenceTimeout();
|
|
5589
6514
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5590
6515
|
upgradeRejectionReason = reason;
|
|
5591
6516
|
const chunks = [];
|
|
@@ -5615,6 +6540,7 @@ function connectTunnel(options) {
|
|
|
5615
6540
|
onInfo?.("WebSocket connection established");
|
|
5616
6541
|
});
|
|
5617
6542
|
ws.on("message", (data, isBinary) => {
|
|
6543
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5618
6544
|
if (isBinary) {
|
|
5619
6545
|
try {
|
|
5620
6546
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5648,10 +6574,11 @@ function connectTunnel(options) {
|
|
|
5648
6574
|
switch (message.type) {
|
|
5649
6575
|
case "connected": {
|
|
5650
6576
|
clearTimeout(connectionTimeout);
|
|
6577
|
+
armRelaySilenceTimeout();
|
|
5651
6578
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5652
6579
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5653
6580
|
onConnected?.(connectedAgentId);
|
|
5654
|
-
|
|
6581
|
+
resolve5({
|
|
5655
6582
|
ws,
|
|
5656
6583
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
5657
6584
|
});
|
|
@@ -5672,6 +6599,7 @@ function connectTunnel(options) {
|
|
|
5672
6599
|
});
|
|
5673
6600
|
ws.on("error", (error2) => {
|
|
5674
6601
|
clearTimeout(connectionTimeout);
|
|
6602
|
+
clearRelaySilenceTimeout();
|
|
5675
6603
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5676
6604
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5677
6605
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5683,7 +6611,8 @@ function connectTunnel(options) {
|
|
|
5683
6611
|
);
|
|
5684
6612
|
});
|
|
5685
6613
|
ws.on("close", (code, reason) => {
|
|
5686
|
-
|
|
6614
|
+
clearRelaySilenceTimeout();
|
|
6615
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5687
6616
|
forwarder.abortAll();
|
|
5688
6617
|
onDisconnected?.(code, reasonStr);
|
|
5689
6618
|
});
|
|
@@ -5691,9 +6620,10 @@ function connectTunnel(options) {
|
|
|
5691
6620
|
}
|
|
5692
6621
|
|
|
5693
6622
|
// src/lib/tunnel/runner-connection.ts
|
|
5694
|
-
var RunnerConnection = class {
|
|
6623
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5695
6624
|
opts;
|
|
5696
6625
|
sleep;
|
|
6626
|
+
now;
|
|
5697
6627
|
connection = null;
|
|
5698
6628
|
resolvedAgentId;
|
|
5699
6629
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5702,10 +6632,17 @@ var RunnerConnection = class {
|
|
|
5702
6632
|
reconnectPromise = null;
|
|
5703
6633
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5704
6634
|
reconnectAttempt = 0;
|
|
6635
|
+
generation = 0;
|
|
6636
|
+
disconnectedAtMs = null;
|
|
6637
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6638
|
+
"No pong received",
|
|
6639
|
+
"Orphaned socket"
|
|
6640
|
+
]);
|
|
5705
6641
|
constructor(opts) {
|
|
5706
6642
|
this.opts = opts;
|
|
5707
6643
|
this.resolvedAgentId = opts.agentId;
|
|
5708
6644
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6645
|
+
this.now = opts.now ?? Date.now;
|
|
5709
6646
|
}
|
|
5710
6647
|
get agentId() {
|
|
5711
6648
|
return this.resolvedAgentId;
|
|
@@ -5741,14 +6678,24 @@ var RunnerConnection = class {
|
|
|
5741
6678
|
port: this.opts.port,
|
|
5742
6679
|
openCodePassword: this.opts.openCodePassword,
|
|
5743
6680
|
onConnected: (agentId) => {
|
|
6681
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5744
6682
|
this.reconnectAttempt = 0;
|
|
5745
6683
|
this.reconnecting = false;
|
|
5746
6684
|
this.resolvedAgentId = agentId;
|
|
5747
|
-
|
|
6685
|
+
this.generation++;
|
|
6686
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6687
|
+
this.disconnectedAtMs = null;
|
|
6688
|
+
events.onConnected(agentId, isReconnect, {
|
|
6689
|
+
generation: this.generation,
|
|
6690
|
+
attempt,
|
|
6691
|
+
downtimeMs
|
|
6692
|
+
});
|
|
5748
6693
|
},
|
|
5749
6694
|
onDisconnected: (code, reason) => {
|
|
5750
|
-
|
|
5751
|
-
|
|
6695
|
+
this.disconnectedAtMs = this.now();
|
|
6696
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6697
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6698
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5752
6699
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5753
6700
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5754
6701
|
});
|
|
@@ -6087,16 +7034,111 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6087
7034
|
};
|
|
6088
7035
|
}
|
|
6089
7036
|
|
|
7037
|
+
// src/lib/resource-usage-low-disk.ts
|
|
7038
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
7039
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
7040
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
7041
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
7042
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
7043
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
7044
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
7045
|
+
var BAND_RANK = {
|
|
7046
|
+
low: 1,
|
|
7047
|
+
very_low: 2,
|
|
7048
|
+
critical: 3
|
|
7049
|
+
};
|
|
7050
|
+
function bandFor(freeBytes) {
|
|
7051
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
7052
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
7053
|
+
return "low";
|
|
7054
|
+
}
|
|
7055
|
+
function formatBytes(bytes) {
|
|
7056
|
+
if (bytes === null) return "unavailable";
|
|
7057
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
7058
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
7059
|
+
const value = bytes / 1024 ** unitIndex;
|
|
7060
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
7061
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
7062
|
+
}
|
|
7063
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
7064
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
7065
|
+
}
|
|
7066
|
+
function evaluateLowDiskWarning(input) {
|
|
7067
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
7068
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
7069
|
+
return { warning: null, state };
|
|
7070
|
+
}
|
|
7071
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
7072
|
+
if (recovered) {
|
|
7073
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
7074
|
+
}
|
|
7075
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
7076
|
+
if (!low) return { warning: null, state };
|
|
7077
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
7078
|
+
const currentRank = BAND_RANK[currentBand];
|
|
7079
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
7080
|
+
const worse = currentRank > previousRank;
|
|
7081
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
7082
|
+
const nextState = {
|
|
7083
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
7084
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
7085
|
+
};
|
|
7086
|
+
return {
|
|
7087
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
7088
|
+
state: nextState
|
|
7089
|
+
};
|
|
7090
|
+
}
|
|
7091
|
+
|
|
7092
|
+
// src/lib/opencode/session-store-health.ts
|
|
7093
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
7094
|
+
function createSessionStoreHealth(options = {}) {
|
|
7095
|
+
const now = options.now ?? (() => Date.now());
|
|
7096
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
7097
|
+
let lastUnwritableAt = null;
|
|
7098
|
+
let lastWritableAt = null;
|
|
7099
|
+
const causes = /* @__PURE__ */ new Map();
|
|
7100
|
+
return {
|
|
7101
|
+
observeUnwritable: () => {
|
|
7102
|
+
lastUnwritableAt = now();
|
|
7103
|
+
},
|
|
7104
|
+
observeWritable: () => {
|
|
7105
|
+
lastWritableAt = now();
|
|
7106
|
+
},
|
|
7107
|
+
isUnwritable: () => {
|
|
7108
|
+
if (lastUnwritableAt === null) return false;
|
|
7109
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
7110
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
7111
|
+
},
|
|
7112
|
+
observeCause: (sessionId, cause) => {
|
|
7113
|
+
if (sessionId === null) return;
|
|
7114
|
+
causes.delete(sessionId);
|
|
7115
|
+
causes.set(sessionId, { cause, observedAt: now() });
|
|
7116
|
+
while (causes.size > 256) {
|
|
7117
|
+
const oldest = causes.keys().next().value;
|
|
7118
|
+
if (typeof oldest !== "string") break;
|
|
7119
|
+
causes.delete(oldest);
|
|
7120
|
+
}
|
|
7121
|
+
},
|
|
7122
|
+
readCause: (sessionId) => {
|
|
7123
|
+
const observed = causes.get(sessionId);
|
|
7124
|
+
if (!observed) return null;
|
|
7125
|
+
causes.delete(sessionId);
|
|
7126
|
+
if (now() - observed.observedAt >= stalenessMs) return null;
|
|
7127
|
+
return observed.cause;
|
|
7128
|
+
}
|
|
7129
|
+
};
|
|
7130
|
+
}
|
|
7131
|
+
|
|
6090
7132
|
// src/lib/channels/driver.ts
|
|
6091
7133
|
import { homedir as homedir6 } from "node:os";
|
|
6092
7134
|
|
|
6093
7135
|
// src/lib/runner-file-sync.ts
|
|
6094
|
-
import { join as
|
|
7136
|
+
import { join as join10 } from "node:path";
|
|
6095
7137
|
|
|
6096
7138
|
// src/lib/file-push.ts
|
|
6097
7139
|
import { randomUUID } from "node:crypto";
|
|
6098
7140
|
import { chmod, mkdir, open as open3, realpath, rename, unlink } from "node:fs/promises";
|
|
6099
|
-
import { basename, dirname as
|
|
7141
|
+
import { basename as basename2, dirname as dirname6, isAbsolute as isAbsolute2, join as join9, relative as relative2, resolve as resolve3, sep as sep2 } from "node:path";
|
|
6100
7142
|
var FILE_MODE = 384;
|
|
6101
7143
|
var DIRECTORY_MODE = 448;
|
|
6102
7144
|
async function writePushedFile(request2) {
|
|
@@ -6127,9 +7169,9 @@ async function writePushedFile(request2) {
|
|
|
6127
7169
|
}
|
|
6128
7170
|
try {
|
|
6129
7171
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
6130
|
-
|
|
7172
|
+
dirname6(candidate)
|
|
6131
7173
|
);
|
|
6132
|
-
const realTarget =
|
|
7174
|
+
const realTarget = join9(existingAncestor, ...missingSegments, basename2(candidate));
|
|
6133
7175
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
6134
7176
|
if (allowedDirectory === null) {
|
|
6135
7177
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -6139,8 +7181,8 @@ async function writePushedFile(request2) {
|
|
|
6139
7181
|
}
|
|
6140
7182
|
if (missingSegments.length > 0) {
|
|
6141
7183
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
6142
|
-
const realParent = await realpath(
|
|
6143
|
-
if (realParent !==
|
|
7184
|
+
const realParent = await realpath(dirname6(realTarget));
|
|
7185
|
+
if (realParent !== dirname6(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
6144
7186
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
6145
7187
|
path: realTarget,
|
|
6146
7188
|
bytes,
|
|
@@ -6165,15 +7207,15 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
6165
7207
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
6166
7208
|
return null;
|
|
6167
7209
|
}
|
|
6168
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
7210
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join9(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
6169
7211
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
6170
7212
|
return null;
|
|
6171
7213
|
}
|
|
6172
|
-
if (!
|
|
7214
|
+
if (!isAbsolute2(expanded)) {
|
|
6173
7215
|
return null;
|
|
6174
7216
|
}
|
|
6175
|
-
const candidate =
|
|
6176
|
-
const name =
|
|
7217
|
+
const candidate = resolve3(expanded);
|
|
7218
|
+
const name = basename2(candidate);
|
|
6177
7219
|
return name === "" || name === "." || name === ".." ? null : candidate;
|
|
6178
7220
|
}
|
|
6179
7221
|
async function resolveNearestExistingAncestor(directory) {
|
|
@@ -6183,18 +7225,18 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
6183
7225
|
try {
|
|
6184
7226
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
6185
7227
|
} catch (err) {
|
|
6186
|
-
const parent =
|
|
7228
|
+
const parent = dirname6(current);
|
|
6187
7229
|
if (err.code !== "ENOENT" || parent === current) {
|
|
6188
7230
|
throw err;
|
|
6189
7231
|
}
|
|
6190
|
-
missingSegments.unshift(
|
|
7232
|
+
missingSegments.unshift(basename2(current));
|
|
6191
7233
|
current = parent;
|
|
6192
7234
|
}
|
|
6193
7235
|
}
|
|
6194
7236
|
}
|
|
6195
7237
|
async function findContainingAllowedDirectory(allowedDirectories, realTarget) {
|
|
6196
7238
|
for (const directory of allowedDirectories) {
|
|
6197
|
-
if (!
|
|
7239
|
+
if (!isAbsolute2(directory)) {
|
|
6198
7240
|
log("warn", "file_push_allowed_directory_skipped", { directory, reason: "not_absolute" });
|
|
6199
7241
|
continue;
|
|
6200
7242
|
}
|
|
@@ -6232,19 +7274,19 @@ async function realpathCreatingIfMissing(directory) {
|
|
|
6232
7274
|
}
|
|
6233
7275
|
}
|
|
6234
7276
|
function contains(realDirectory, realTarget) {
|
|
6235
|
-
const rel =
|
|
6236
|
-
return rel !== "" && rel !== ".." && !rel.startsWith(`..${
|
|
7277
|
+
const rel = relative2(realDirectory, realTarget);
|
|
7278
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep2}`) && !isAbsolute2(rel);
|
|
6237
7279
|
}
|
|
6238
7280
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
6239
7281
|
let current = existingAncestor;
|
|
6240
7282
|
for (const segment of missingSegments) {
|
|
6241
|
-
current =
|
|
7283
|
+
current = join9(current, segment);
|
|
6242
7284
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
6243
7285
|
await chmod(current, DIRECTORY_MODE);
|
|
6244
7286
|
}
|
|
6245
7287
|
}
|
|
6246
7288
|
async function writeAtomically(realTarget, content) {
|
|
6247
|
-
const temporaryPath =
|
|
7289
|
+
const temporaryPath = join9(dirname6(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
6248
7290
|
let handle;
|
|
6249
7291
|
try {
|
|
6250
7292
|
handle = await open3(temporaryPath, "wx", FILE_MODE);
|
|
@@ -6374,12 +7416,12 @@ var NOT_APPLIED = {
|
|
|
6374
7416
|
opencodeAuthApplied: false
|
|
6375
7417
|
};
|
|
6376
7418
|
function isClaudeCredentialPath(requestedPath, homeDir) {
|
|
6377
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6378
|
-
return expanded ===
|
|
7419
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7420
|
+
return expanded === join10(homeDir, ...CLAUDE_CREDENTIALS_SEGMENTS);
|
|
6379
7421
|
}
|
|
6380
7422
|
function isOpenCodeAuthPath(requestedPath, homeDir) {
|
|
6381
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
6382
|
-
return expanded ===
|
|
7423
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join10(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
7424
|
+
return expanded === join10(homeDir, ...OPENCODE_AUTH_SEGMENTS);
|
|
6383
7425
|
}
|
|
6384
7426
|
async function applyOne(options, file) {
|
|
6385
7427
|
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
@@ -6523,6 +7565,11 @@ function messageIdOf(m) {
|
|
|
6523
7565
|
const infoId = m.info?.id;
|
|
6524
7566
|
return typeof infoId === "string" ? infoId : void 0;
|
|
6525
7567
|
}
|
|
7568
|
+
function composeAbandonmentNote(finish, replyShape, outcome) {
|
|
7569
|
+
const base = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") ` + (outcome === "without-answer" ? "without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs." : "after writing only part of an answer, and confirmed it is no longer working on this conversation. Evident delivered what was written and is reporting the run as finished.");
|
|
7570
|
+
if (replyShape === null) return base;
|
|
7571
|
+
return `${base} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
|
|
7572
|
+
}
|
|
6526
7573
|
function cleanImageMime(contentType) {
|
|
6527
7574
|
if (!contentType) return null;
|
|
6528
7575
|
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
@@ -6543,8 +7590,13 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
|
|
|
6543
7590
|
var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
|
|
6544
7591
|
var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
6545
7592
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
7593
|
+
var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
|
|
7594
|
+
var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
7595
|
+
var SESSION_ERROR_RETAINED_DETAIL = "Runner-observed OpenCode session error: ";
|
|
7596
|
+
var SESSION_ERROR_OBSERVED_CAUSE = "Runner-observed OpenCode session-store cause: ";
|
|
6546
7597
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6547
7598
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7599
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6548
7600
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6549
7601
|
var HEARTBEAT_MS = 6e4;
|
|
6550
7602
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6606,6 +7658,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6606
7658
|
maxActiveSessions;
|
|
6607
7659
|
watcherStallMs;
|
|
6608
7660
|
wedgeWarningIntervalMs;
|
|
7661
|
+
sessionStoreHealth;
|
|
6609
7662
|
openCodeClient;
|
|
6610
7663
|
/** Cache of conversationId → opencode sessionId. */
|
|
6611
7664
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6698,6 +7751,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6698
7751
|
* with a short TTL so an unmatched session cannot retain an event indefinitely.
|
|
6699
7752
|
*/
|
|
6700
7753
|
bufferedSessionErrors = /* @__PURE__ */ new Map();
|
|
7754
|
+
retainedSessionErrors = /* @__PURE__ */ new Map();
|
|
6701
7755
|
/**
|
|
6702
7756
|
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
6703
7757
|
* dispatched and are still in-flight. A message in this set is never
|
|
@@ -6730,10 +7784,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6730
7784
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6731
7785
|
*/
|
|
6732
7786
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7787
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7788
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6733
7789
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6734
7790
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6735
7791
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6736
7792
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7793
|
+
/**
|
|
7794
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7795
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7796
|
+
* identical-text turn, so it never produces a correlation.
|
|
7797
|
+
*/
|
|
7798
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6737
7799
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6738
7800
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6739
7801
|
/**
|
|
@@ -6819,21 +7881,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6819
7881
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6820
7882
|
*/
|
|
6821
7883
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6822
|
-
/**
|
|
6823
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6824
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6825
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6826
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6827
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6828
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6829
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6830
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6831
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6832
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6833
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6834
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6835
|
-
*/
|
|
6836
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6837
7884
|
/**
|
|
6838
7885
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6839
7886
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6851,8 +7898,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6851
7898
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6852
7899
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6853
7900
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6854
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6855
|
-
*
|
|
7901
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7902
|
+
* the terminal-report fence and is never sent again).
|
|
6856
7903
|
*/
|
|
6857
7904
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6858
7905
|
/**
|
|
@@ -6904,6 +7951,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6904
7951
|
subagentInvocationPrefetches = /* @__PURE__ */ new Map();
|
|
6905
7952
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
6906
7953
|
draining = false;
|
|
7954
|
+
pendingProviderRegistryRepair = null;
|
|
6907
7955
|
/**
|
|
6908
7956
|
* Serialises runner-file syncs (#559) so the ~2s poll tick and a concurrent
|
|
6909
7957
|
* drain ping don't download, write and ack the same file twice.
|
|
@@ -6975,6 +8023,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6975
8023
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6976
8024
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6977
8025
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
8026
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6978
8027
|
}
|
|
6979
8028
|
get isV2() {
|
|
6980
8029
|
return this.openCodeClient.version === "v2";
|
|
@@ -6990,7 +8039,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6990
8039
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6991
8040
|
}
|
|
6992
8041
|
async listSessions() {
|
|
6993
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
8042
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
8043
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
8044
|
+
});
|
|
6994
8045
|
}
|
|
6995
8046
|
async sessionExists(sessionId) {
|
|
6996
8047
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7086,6 +8137,23 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7086
8137
|
async runDrain() {
|
|
7087
8138
|
let dispatched = 0;
|
|
7088
8139
|
try {
|
|
8140
|
+
const pendingRepair = this.pendingProviderRegistryRepair;
|
|
8141
|
+
if (pendingRepair && this.protectedSessionIds().size === 0) {
|
|
8142
|
+
try {
|
|
8143
|
+
await this.issueProviderRegistryReload(
|
|
8144
|
+
pendingRepair.conversationId,
|
|
8145
|
+
pendingRepair.messageId
|
|
8146
|
+
);
|
|
8147
|
+
} catch (err) {
|
|
8148
|
+
this.log({
|
|
8149
|
+
level: "error",
|
|
8150
|
+
message: `Provider registry repair adoption failed for message ${pendingRepair.messageId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
8151
|
+
conversation_id: pendingRepair.conversationId,
|
|
8152
|
+
message_id: pendingRepair.messageId
|
|
8153
|
+
});
|
|
8154
|
+
}
|
|
8155
|
+
return 0;
|
|
8156
|
+
}
|
|
7089
8157
|
this.pendingMessageIds.clear();
|
|
7090
8158
|
const conversations = await this.getPendingConversations();
|
|
7091
8159
|
if (this.recycleRequestedFlag) {
|
|
@@ -7167,6 +8235,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7167
8235
|
}
|
|
7168
8236
|
return ids;
|
|
7169
8237
|
}
|
|
8238
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
8239
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
8240
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
8241
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
8242
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
8243
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
8244
|
+
}
|
|
8245
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
8246
|
+
}
|
|
8247
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
8248
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
8249
|
+
this.awaitingCorrelation.delete(messageId);
|
|
8250
|
+
this.awaitingReadopt.delete(messageId);
|
|
8251
|
+
this.sessions.delete(conversationId);
|
|
8252
|
+
this.supersede(conversationId, sessionId);
|
|
8253
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
8254
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
8255
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
8256
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
8257
|
+
this.log({
|
|
8258
|
+
level: "error",
|
|
8259
|
+
message: errorMessage3,
|
|
8260
|
+
conversation_id: conversationId,
|
|
8261
|
+
message_id: messageId
|
|
8262
|
+
});
|
|
8263
|
+
const reportError = await this.reportUntrackableAck(
|
|
8264
|
+
conversationId,
|
|
8265
|
+
messageId,
|
|
8266
|
+
null,
|
|
8267
|
+
errorMessage3
|
|
8268
|
+
);
|
|
8269
|
+
if (reportError !== null) {
|
|
8270
|
+
this.log({
|
|
8271
|
+
level: "warn",
|
|
8272
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
8273
|
+
conversation_id: conversationId,
|
|
8274
|
+
message_id: messageId
|
|
8275
|
+
});
|
|
8276
|
+
}
|
|
8277
|
+
if (source === "readopt") {
|
|
8278
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
8279
|
+
}
|
|
8280
|
+
}
|
|
7170
8281
|
/**
|
|
7171
8282
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7172
8283
|
*
|
|
@@ -7346,6 +8457,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7346
8457
|
}
|
|
7347
8458
|
break;
|
|
7348
8459
|
}
|
|
8460
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
8461
|
+
if (correlation) {
|
|
8462
|
+
correlation.attempts += 1;
|
|
8463
|
+
let correlatedId = null;
|
|
8464
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
8465
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
8466
|
+
conv,
|
|
8467
|
+
message,
|
|
8468
|
+
correlation.sessionId
|
|
8469
|
+
);
|
|
8470
|
+
if (polled.ok)
|
|
8471
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
8472
|
+
} else if (correlation.knownUserIds === null) {
|
|
8473
|
+
this.log({
|
|
8474
|
+
level: "debug",
|
|
8475
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
8476
|
+
conversation_id: conv.id,
|
|
8477
|
+
message_id: message.id
|
|
8478
|
+
});
|
|
8479
|
+
}
|
|
8480
|
+
if (correlatedId !== null) {
|
|
8481
|
+
this.awaitingCorrelation.delete(message.id);
|
|
8482
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
8483
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
8484
|
+
this.dispatched.add(message.id);
|
|
8485
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
8486
|
+
dispatched += 1;
|
|
8487
|
+
this.log({
|
|
8488
|
+
level: "info",
|
|
8489
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
8490
|
+
conversation_id: conv.id,
|
|
8491
|
+
message_id: message.id
|
|
8492
|
+
});
|
|
8493
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
8494
|
+
opencode_message_id: correlatedId
|
|
8495
|
+
});
|
|
8496
|
+
continue;
|
|
8497
|
+
}
|
|
8498
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
8499
|
+
await this.handoffUntrackableCorrelation(
|
|
8500
|
+
conv.id,
|
|
8501
|
+
message.id,
|
|
8502
|
+
correlation.sessionId,
|
|
8503
|
+
correlation.attempts,
|
|
8504
|
+
"fresh"
|
|
8505
|
+
);
|
|
8506
|
+
break;
|
|
8507
|
+
}
|
|
8508
|
+
break;
|
|
8509
|
+
}
|
|
7349
8510
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7350
8511
|
if (effectiveOpencodeMessageId) {
|
|
7351
8512
|
const outcome = await this.resolveRedrive(
|
|
@@ -7367,6 +8528,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7367
8528
|
model: message.opencode_model ?? void 0
|
|
7368
8529
|
};
|
|
7369
8530
|
let opencodeMessageId;
|
|
8531
|
+
let preDispatchUserIds = null;
|
|
7370
8532
|
try {
|
|
7371
8533
|
this.log({
|
|
7372
8534
|
level: "info",
|
|
@@ -7396,7 +8558,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7396
8558
|
message.content,
|
|
7397
8559
|
options,
|
|
7398
8560
|
sendAttachments,
|
|
7399
|
-
this.openCodeClient
|
|
8561
|
+
this.openCodeClient,
|
|
8562
|
+
{
|
|
8563
|
+
onKnownUserIds: (ids) => {
|
|
8564
|
+
preDispatchUserIds = ids;
|
|
8565
|
+
},
|
|
8566
|
+
onKnownUserIdsError: (error2) => {
|
|
8567
|
+
this.log({
|
|
8568
|
+
level: "warn",
|
|
8569
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
8570
|
+
conversation_id: conv.id,
|
|
8571
|
+
message_id: message.id
|
|
8572
|
+
});
|
|
8573
|
+
}
|
|
8574
|
+
}
|
|
7400
8575
|
)
|
|
7401
8576
|
);
|
|
7402
8577
|
} catch (err) {
|
|
@@ -7404,6 +8579,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7404
8579
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7405
8580
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7406
8581
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
8582
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7407
8583
|
this.log({
|
|
7408
8584
|
level: "error",
|
|
7409
8585
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7479,39 +8655,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7479
8655
|
if (this.isV2) {
|
|
7480
8656
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7481
8657
|
}
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
|
|
7485
|
-
|
|
7486
|
-
|
|
7487
|
-
|
|
7488
|
-
message_id: message.id
|
|
7489
|
-
});
|
|
7490
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7491
|
-
continue;
|
|
7492
|
-
}
|
|
7493
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7494
|
-
this.sessions.delete(conv.id);
|
|
7495
|
-
this.supersede(conv.id, sessionId);
|
|
7496
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8658
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8659
|
+
sessionId,
|
|
8660
|
+
content: message.content,
|
|
8661
|
+
attempts: 0,
|
|
8662
|
+
knownUserIds: preDispatchUserIds
|
|
8663
|
+
});
|
|
7497
8664
|
this.log({
|
|
7498
|
-
level: "
|
|
7499
|
-
message:
|
|
8665
|
+
level: "warn",
|
|
8666
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7500
8667
|
conversation_id: conv.id,
|
|
7501
8668
|
message_id: message.id
|
|
7502
8669
|
});
|
|
7503
|
-
|
|
7504
|
-
this.log({
|
|
7505
|
-
level: "warn",
|
|
7506
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7507
|
-
conversation_id: conv.id,
|
|
7508
|
-
message_id: message.id
|
|
7509
|
-
});
|
|
7510
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7511
|
-
});
|
|
8670
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7512
8671
|
break;
|
|
7513
8672
|
}
|
|
7514
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7515
8673
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7516
8674
|
this.releasedOpencodeIds.delete(message.id);
|
|
7517
8675
|
this.dispatched.add(message.id);
|
|
@@ -8116,23 +9274,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8116
9274
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8117
9275
|
return 1;
|
|
8118
9276
|
}
|
|
8119
|
-
/**
|
|
8120
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8121
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8122
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8123
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8124
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8125
|
-
* count, since a new session is a genuinely different attempt.
|
|
8126
|
-
*/
|
|
8127
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8128
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8129
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8130
|
-
existing.count += 1;
|
|
8131
|
-
return existing.count;
|
|
8132
|
-
}
|
|
8133
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8134
|
-
return 1;
|
|
8135
|
-
}
|
|
8136
9277
|
/** Record one queued give-up toward the never-started bound. */
|
|
8137
9278
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8138
9279
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8444,8 +9585,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8444
9585
|
b2LastDescendantCheckMs: 0,
|
|
8445
9586
|
b2RootOngoingHeldLogged: false,
|
|
8446
9587
|
b2AbandonedSignalled: false,
|
|
9588
|
+
b2AbandonedWithoutAnswer: false,
|
|
8447
9589
|
ambiguousPinnedSinceMs: 0,
|
|
8448
9590
|
ambiguousResolved: false,
|
|
9591
|
+
sessionStoreFailurePending: false,
|
|
8449
9592
|
neverStartedResolved: false
|
|
8450
9593
|
});
|
|
8451
9594
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8467,6 +9610,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8467
9610
|
this.bufferedSessionErrors.delete(oldest);
|
|
8468
9611
|
}
|
|
8469
9612
|
}
|
|
9613
|
+
retainSessionError(event) {
|
|
9614
|
+
this.retainedSessionErrors.delete(event.sessionId);
|
|
9615
|
+
this.retainedSessionErrors.set(event.sessionId, {
|
|
9616
|
+
reason: event.reason,
|
|
9617
|
+
observedAt: this.now()
|
|
9618
|
+
});
|
|
9619
|
+
while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
|
|
9620
|
+
const oldest = this.retainedSessionErrors.keys().next().value;
|
|
9621
|
+
if (typeof oldest !== "string") break;
|
|
9622
|
+
this.retainedSessionErrors.delete(oldest);
|
|
9623
|
+
}
|
|
9624
|
+
}
|
|
9625
|
+
readRetainedSessionError(sessionId) {
|
|
9626
|
+
const retained = this.retainedSessionErrors.get(sessionId);
|
|
9627
|
+
if (!retained) return null;
|
|
9628
|
+
if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
|
|
9629
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9630
|
+
return null;
|
|
9631
|
+
}
|
|
9632
|
+
this.retainedSessionErrors.delete(sessionId);
|
|
9633
|
+
return retained.reason;
|
|
9634
|
+
}
|
|
8470
9635
|
/**
|
|
8471
9636
|
* Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
|
|
8472
9637
|
* EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
|
|
@@ -8562,8 +9727,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8562
9727
|
b2LastDescendantCheckMs: 0,
|
|
8563
9728
|
b2RootOngoingHeldLogged: false,
|
|
8564
9729
|
b2AbandonedSignalled: false,
|
|
9730
|
+
b2AbandonedWithoutAnswer: false,
|
|
8565
9731
|
ambiguousPinnedSinceMs: 0,
|
|
8566
9732
|
ambiguousResolved: false,
|
|
9733
|
+
sessionStoreFailurePending: false,
|
|
8567
9734
|
neverStartedResolved: false
|
|
8568
9735
|
});
|
|
8569
9736
|
}
|
|
@@ -8791,9 +9958,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8791
9958
|
return;
|
|
8792
9959
|
}
|
|
8793
9960
|
if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
|
|
9961
|
+
this.retainSessionError(event);
|
|
8794
9962
|
this.log({
|
|
8795
|
-
level: "
|
|
8796
|
-
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
|
|
9963
|
+
level: "warn",
|
|
9964
|
+
message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
|
|
8797
9965
|
conversation_id: watcher.conv.id
|
|
8798
9966
|
});
|
|
8799
9967
|
return;
|
|
@@ -8823,9 +9991,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8823
9991
|
const messages = await this.getSessionMessages(event.sessionId);
|
|
8824
9992
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
8825
9993
|
if (state !== "queued") {
|
|
9994
|
+
this.retainSessionError(event);
|
|
8826
9995
|
this.log({
|
|
8827
|
-
level: "
|
|
8828
|
-
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state};
|
|
9996
|
+
level: "warn",
|
|
9997
|
+
message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
|
|
8829
9998
|
conversation_id: watcher.conv.id,
|
|
8830
9999
|
message_id: inFlight.evidentMessageId
|
|
8831
10000
|
});
|
|
@@ -8837,14 +10006,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8837
10006
|
conversation_id: watcher.conv.id,
|
|
8838
10007
|
message_id: inFlight.evidentMessageId
|
|
8839
10008
|
});
|
|
8840
|
-
|
|
8841
|
-
|
|
8842
|
-
inFlight.
|
|
8843
|
-
|
|
8844
|
-
|
|
8845
|
-
|
|
10009
|
+
const isMissingProvider = isProviderModelNotFoundError(event.reason);
|
|
10010
|
+
if (isMissingProvider) {
|
|
10011
|
+
const dispatchedModel = inFlight.message.opencode_model;
|
|
10012
|
+
const separator = dispatchedModel?.indexOf("/") ?? -1;
|
|
10013
|
+
const hasProviderAndModel = dispatchedModel !== null && separator > 0 && separator < dispatchedModel.length - 1;
|
|
10014
|
+
await this.markFailed(
|
|
10015
|
+
watcher.conv.id,
|
|
10016
|
+
inFlight.evidentMessageId,
|
|
10017
|
+
event.sessionId,
|
|
10018
|
+
`OpenCode could not run this turn: ${event.reason}`,
|
|
10019
|
+
void 0,
|
|
10020
|
+
{
|
|
10021
|
+
classification: {
|
|
10022
|
+
kind: "model_auth",
|
|
10023
|
+
providerId: hasProviderAndModel ? dispatchedModel.slice(0, separator) : null,
|
|
10024
|
+
modelId: hasProviderAndModel ? dispatchedModel.slice(separator + 1) : null,
|
|
10025
|
+
reason: "missing"
|
|
10026
|
+
},
|
|
10027
|
+
errorShape: { name: null, statusCode: null }
|
|
10028
|
+
}
|
|
10029
|
+
);
|
|
10030
|
+
} else {
|
|
10031
|
+
await this.markFailed(
|
|
10032
|
+
watcher.conv.id,
|
|
10033
|
+
inFlight.evidentMessageId,
|
|
10034
|
+
event.sessionId,
|
|
10035
|
+
`OpenCode could not run this turn: ${event.reason}`
|
|
10036
|
+
);
|
|
10037
|
+
}
|
|
8846
10038
|
inFlight.done = true;
|
|
8847
10039
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10040
|
+
if (isMissingProvider) {
|
|
10041
|
+
await this.repairProviderRegistry(watcher.conv.id, inFlight.evidentMessageId);
|
|
10042
|
+
}
|
|
8848
10043
|
} catch (err) {
|
|
8849
10044
|
if (err instanceof ChannelAuthError) {
|
|
8850
10045
|
this.log({
|
|
@@ -8865,6 +10060,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8865
10060
|
this.sessionErrorHandled.delete(inFlight.evidentMessageId);
|
|
8866
10061
|
}
|
|
8867
10062
|
}
|
|
10063
|
+
async repairProviderRegistry(conversationId, messageId) {
|
|
10064
|
+
if (this.isV2) {
|
|
10065
|
+
void this.postSignal(conversationId, messageId, "provider_registry_reload_unsupported");
|
|
10066
|
+
return;
|
|
10067
|
+
}
|
|
10068
|
+
const activeSessions = this.protectedSessionIds().size;
|
|
10069
|
+
if (activeSessions > 0 || this.draining) {
|
|
10070
|
+
this.pendingProviderRegistryRepair = { conversationId, messageId };
|
|
10071
|
+
void this.postSignal(conversationId, messageId, "provider_registry_repair_deferred", {
|
|
10072
|
+
active_sessions: activeSessions
|
|
10073
|
+
});
|
|
10074
|
+
this.log({
|
|
10075
|
+
level: "warn",
|
|
10076
|
+
message: `Provider registry repair deferred until a lifecycle-safe point; ${activeSessions} active session(s) protected`,
|
|
10077
|
+
conversation_id: conversationId,
|
|
10078
|
+
message_id: messageId
|
|
10079
|
+
});
|
|
10080
|
+
return;
|
|
10081
|
+
}
|
|
10082
|
+
await this.issueProviderRegistryReload(conversationId, messageId);
|
|
10083
|
+
}
|
|
10084
|
+
async issueProviderRegistryReload(conversationId, messageId) {
|
|
10085
|
+
this.pendingProviderRegistryRepair = null;
|
|
10086
|
+
const result = await disposeAndReloadProviderConfig(this.port, this.openCodeClient);
|
|
10087
|
+
if (result.outcome === "failed") {
|
|
10088
|
+
const failure = result.status === null ? `transport failure${result.detail ? `: ${result.detail}` : ""}` : `HTTP ${result.status}`;
|
|
10089
|
+
this.log({
|
|
10090
|
+
level: "warn",
|
|
10091
|
+
message: `Provider registry reload failed (${failure})`,
|
|
10092
|
+
conversation_id: conversationId,
|
|
10093
|
+
message_id: messageId
|
|
10094
|
+
});
|
|
10095
|
+
}
|
|
10096
|
+
const repairOutcome = result.outcome === "reloaded" ? "reloaded" : "reload_failed";
|
|
10097
|
+
void this.postSignal(conversationId, messageId, "provider_registry_reload_issued", {
|
|
10098
|
+
repair_outcome: repairOutcome
|
|
10099
|
+
});
|
|
10100
|
+
}
|
|
8868
10101
|
/**
|
|
8869
10102
|
* The per-session polling loop (WI-3). Once per tick it:
|
|
8870
10103
|
* 1. polls `GET /session/:id/message` once and, per in-flight message,
|
|
@@ -8977,6 +10210,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8977
10210
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8978
10211
|
this.neverStartedAttempts.delete(id);
|
|
8979
10212
|
}
|
|
10213
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
10214
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10215
|
+
return;
|
|
10216
|
+
}
|
|
8980
10217
|
if (inFlight.neverStartedResolved) {
|
|
8981
10218
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8982
10219
|
inFlight.neverStartedResolved = false;
|
|
@@ -9050,7 +10287,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9050
10287
|
if (state === "failed") {
|
|
9051
10288
|
this.anchorDeliveryDeadline(inFlight);
|
|
9052
10289
|
if (!inFlight.done) {
|
|
9053
|
-
const
|
|
10290
|
+
const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
|
|
10291
|
+
const retainedReason = this.readRetainedSessionError(sessionId);
|
|
10292
|
+
const retainedError = retainedReason === null || messageErrorText !== null && messageErrorText.includes(retainedReason) ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
|
|
10293
|
+
const observedCause = this.sessionStoreHealth?.readCause(sessionId);
|
|
10294
|
+
const causeError = observedCause === null || observedCause === void 0 ? null : `${SESSION_ERROR_OBSERVED_CAUSE}${observedCause}`;
|
|
10295
|
+
const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
|
|
10296
|
+
const runnerDetails = [retainedError, causeError, storeError].filter(
|
|
10297
|
+
(detail) => detail !== null
|
|
10298
|
+
);
|
|
10299
|
+
const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
|
|
9054
10300
|
this.log({
|
|
9055
10301
|
level: "error",
|
|
9056
10302
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -9136,10 +10382,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9136
10382
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9137
10383
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9138
10384
|
inFlight.b2AbandonedSignalled = false;
|
|
10385
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9139
10386
|
}
|
|
9140
10387
|
} else {
|
|
9141
10388
|
if (inFlight.b2AbandonedSignalled) {
|
|
9142
|
-
|
|
10389
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10390
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10391
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
10392
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10393
|
+
sessionId,
|
|
10394
|
+
watcher,
|
|
10395
|
+
inFlight,
|
|
10396
|
+
messages,
|
|
10397
|
+
finish
|
|
10398
|
+
);
|
|
10399
|
+
} else {
|
|
10400
|
+
await this.settleMessageDone(
|
|
10401
|
+
sessionId,
|
|
10402
|
+
watcher,
|
|
10403
|
+
inFlight,
|
|
10404
|
+
messages,
|
|
10405
|
+
composeAbandonmentNote(
|
|
10406
|
+
finish,
|
|
10407
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10408
|
+
"with-answer"
|
|
10409
|
+
)
|
|
10410
|
+
);
|
|
10411
|
+
}
|
|
9143
10412
|
return;
|
|
9144
10413
|
}
|
|
9145
10414
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9156,22 +10425,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9156
10425
|
descendantOngoing,
|
|
9157
10426
|
rootOngoing
|
|
9158
10427
|
})) {
|
|
10428
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10429
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10430
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10431
|
+
watched_for_ms: pinnedForMs,
|
|
10432
|
+
run_state: "failed",
|
|
10433
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10434
|
+
});
|
|
10435
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10436
|
+
return;
|
|
10437
|
+
}
|
|
9159
10438
|
inFlight.b2AbandonedSignalled = true;
|
|
10439
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
10440
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
10441
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
10442
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
10443
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
10444
|
+
watched_for_ms: pinnedForMs,
|
|
10445
|
+
run_state: "failed",
|
|
10446
|
+
finish,
|
|
10447
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
10448
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
10449
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10450
|
+
});
|
|
10451
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
10452
|
+
sessionId,
|
|
10453
|
+
watcher,
|
|
10454
|
+
inFlight,
|
|
10455
|
+
messages,
|
|
10456
|
+
finish
|
|
10457
|
+
);
|
|
10458
|
+
return;
|
|
10459
|
+
}
|
|
9160
10460
|
this.log({
|
|
9161
10461
|
level: "warn",
|
|
9162
10462
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9163
10463
|
conversation_id: conv.id,
|
|
9164
10464
|
message_id: id
|
|
9165
10465
|
});
|
|
9166
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9167
10466
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9168
10467
|
watched_for_ms: pinnedForMs,
|
|
9169
|
-
finish
|
|
10468
|
+
finish,
|
|
9170
10469
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9171
10470
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9172
10471
|
opencode_message_id: inFlight.opencodeMessageId
|
|
9173
10472
|
});
|
|
9174
|
-
await this.settleMessageDone(
|
|
10473
|
+
await this.settleMessageDone(
|
|
10474
|
+
sessionId,
|
|
10475
|
+
watcher,
|
|
10476
|
+
inFlight,
|
|
10477
|
+
messages,
|
|
10478
|
+
composeAbandonmentNote(
|
|
10479
|
+
finish,
|
|
10480
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10481
|
+
"with-answer"
|
|
10482
|
+
)
|
|
10483
|
+
);
|
|
9175
10484
|
return;
|
|
9176
10485
|
}
|
|
9177
10486
|
if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
|
|
@@ -9214,6 +10523,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9214
10523
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9215
10524
|
sessionOngoing: ongoing
|
|
9216
10525
|
})) {
|
|
10526
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
10527
|
+
inFlight.sessionStoreFailurePending = true;
|
|
10528
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
10529
|
+
watched_for_ms: pinnedForMs,
|
|
10530
|
+
run_state: "failed",
|
|
10531
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
10532
|
+
});
|
|
10533
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
10534
|
+
return;
|
|
10535
|
+
}
|
|
9217
10536
|
inFlight.ambiguousResolved = true;
|
|
9218
10537
|
this.log({
|
|
9219
10538
|
level: "warn",
|
|
@@ -9361,15 +10680,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9361
10680
|
}
|
|
9362
10681
|
}
|
|
9363
10682
|
/**
|
|
9364
|
-
* Settle a message whose run-state has resolved `'done'
|
|
9365
|
-
*
|
|
9366
|
-
*
|
|
9367
|
-
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
9368
|
-
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
9369
|
-
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
9370
|
-
* and risking the two copies silently drifting apart.
|
|
10683
|
+
* Settle a message whose run-state has resolved `'done'`. Normal completion and
|
|
10684
|
+
* confirmed b2 abandonment share the same delivery, usage, and terminal handling.
|
|
10685
|
+
* Normal completions omit the optional terminal diagnostic.
|
|
9371
10686
|
*/
|
|
9372
|
-
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
10687
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages, note) {
|
|
9373
10688
|
const conv = watcher.conv;
|
|
9374
10689
|
this.anchorDeliveryDeadline(inFlight);
|
|
9375
10690
|
if (!inFlight.done) {
|
|
@@ -9396,7 +10711,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9396
10711
|
title,
|
|
9397
10712
|
usage,
|
|
9398
10713
|
usageAgentName,
|
|
9399
|
-
subagentInvocations
|
|
10714
|
+
subagentInvocations,
|
|
10715
|
+
note
|
|
9400
10716
|
);
|
|
9401
10717
|
} catch (err) {
|
|
9402
10718
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -9438,6 +10754,78 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9438
10754
|
}
|
|
9439
10755
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9440
10756
|
}
|
|
10757
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10758
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10759
|
+
const conv = watcher.conv;
|
|
10760
|
+
const error2 = composeAbandonmentNote(
|
|
10761
|
+
finish,
|
|
10762
|
+
describeAbandonedReplyShape(messages, inFlight.opencodeMessageId),
|
|
10763
|
+
"without-answer"
|
|
10764
|
+
);
|
|
10765
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10766
|
+
this.log({
|
|
10767
|
+
level: "warn",
|
|
10768
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10769
|
+
conversation_id: conv.id,
|
|
10770
|
+
message_id: inFlight.evidentMessageId
|
|
10771
|
+
});
|
|
10772
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10773
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10774
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10775
|
+
messages,
|
|
10776
|
+
inFlight.opencodeMessageId,
|
|
10777
|
+
inFlight.evidentMessageId
|
|
10778
|
+
);
|
|
10779
|
+
try {
|
|
10780
|
+
await this.markFailed(
|
|
10781
|
+
conv.id,
|
|
10782
|
+
inFlight.evidentMessageId,
|
|
10783
|
+
sessionId,
|
|
10784
|
+
error2,
|
|
10785
|
+
usage,
|
|
10786
|
+
void 0,
|
|
10787
|
+
usageAgentName,
|
|
10788
|
+
subagentInvocations
|
|
10789
|
+
);
|
|
10790
|
+
} catch (err) {
|
|
10791
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10792
|
+
if (err instanceof ChannelTerminalError) {
|
|
10793
|
+
this.log({
|
|
10794
|
+
level: "warn",
|
|
10795
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
10796
|
+
conversation_id: conv.id,
|
|
10797
|
+
message_id: inFlight.evidentMessageId
|
|
10798
|
+
});
|
|
10799
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10800
|
+
return;
|
|
10801
|
+
}
|
|
10802
|
+
if (this.now() >= inFlight.deadline) {
|
|
10803
|
+
this.log({
|
|
10804
|
+
level: "warn",
|
|
10805
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10806
|
+
conversation_id: conv.id,
|
|
10807
|
+
message_id: inFlight.evidentMessageId
|
|
10808
|
+
});
|
|
10809
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10810
|
+
return;
|
|
10811
|
+
}
|
|
10812
|
+
this.log({
|
|
10813
|
+
level: "warn",
|
|
10814
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10815
|
+
conversation_id: conv.id,
|
|
10816
|
+
message_id: inFlight.evidentMessageId
|
|
10817
|
+
});
|
|
10818
|
+
return;
|
|
10819
|
+
}
|
|
10820
|
+
inFlight.done = true;
|
|
10821
|
+
await this.reportSubagentAuthFailures(
|
|
10822
|
+
watcher.conv.id,
|
|
10823
|
+
inFlight.opencodeMessageId,
|
|
10824
|
+
inFlight.evidentMessageId,
|
|
10825
|
+
messages
|
|
10826
|
+
);
|
|
10827
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10828
|
+
}
|
|
9441
10829
|
/**
|
|
9442
10830
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9443
10831
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9493,6 +10881,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9493
10881
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9494
10882
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9495
10883
|
}
|
|
10884
|
+
/**
|
|
10885
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10886
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10887
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10888
|
+
*/
|
|
10889
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10890
|
+
const conv = watcher.conv;
|
|
10891
|
+
const error2 = SESSION_STORE_UNWRITABLE_ERROR;
|
|
10892
|
+
this.log({
|
|
10893
|
+
level: "error",
|
|
10894
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10895
|
+
conversation_id: conv.id,
|
|
10896
|
+
message_id: inFlight.evidentMessageId
|
|
10897
|
+
});
|
|
10898
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10899
|
+
try {
|
|
10900
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10901
|
+
} catch (err) {
|
|
10902
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10903
|
+
if (err instanceof ChannelTerminalError) {
|
|
10904
|
+
this.log({
|
|
10905
|
+
level: "warn",
|
|
10906
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10907
|
+
conversation_id: conv.id,
|
|
10908
|
+
message_id: inFlight.evidentMessageId
|
|
10909
|
+
});
|
|
10910
|
+
this.recordReleasedOpencodeId(
|
|
10911
|
+
inFlight.evidentMessageId,
|
|
10912
|
+
sessionId,
|
|
10913
|
+
inFlight.opencodeMessageId
|
|
10914
|
+
);
|
|
10915
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10916
|
+
return;
|
|
10917
|
+
}
|
|
10918
|
+
if (this.now() >= inFlight.deadline) {
|
|
10919
|
+
this.log({
|
|
10920
|
+
level: "warn",
|
|
10921
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10922
|
+
conversation_id: conv.id,
|
|
10923
|
+
message_id: inFlight.evidentMessageId
|
|
10924
|
+
});
|
|
10925
|
+
this.recordReleasedOpencodeId(
|
|
10926
|
+
inFlight.evidentMessageId,
|
|
10927
|
+
sessionId,
|
|
10928
|
+
inFlight.opencodeMessageId
|
|
10929
|
+
);
|
|
10930
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10931
|
+
return;
|
|
10932
|
+
}
|
|
10933
|
+
this.log({
|
|
10934
|
+
level: "warn",
|
|
10935
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10936
|
+
conversation_id: conv.id,
|
|
10937
|
+
message_id: inFlight.evidentMessageId
|
|
10938
|
+
});
|
|
10939
|
+
return;
|
|
10940
|
+
}
|
|
10941
|
+
inFlight.done = true;
|
|
10942
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10943
|
+
}
|
|
9496
10944
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9497
10945
|
/**
|
|
9498
10946
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9531,11 +10979,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9531
10979
|
}
|
|
9532
10980
|
}
|
|
9533
10981
|
}
|
|
9534
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10982
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9535
10983
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10984
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10985
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10986
|
+
this.awaitingCorrelation.delete(id);
|
|
10987
|
+
this.log({
|
|
10988
|
+
level: "debug",
|
|
10989
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10990
|
+
message_id: id
|
|
10991
|
+
});
|
|
10992
|
+
}
|
|
10993
|
+
}
|
|
9536
10994
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9537
10995
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9538
10996
|
this.untrackableAck.delete(id);
|
|
10997
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9539
10998
|
this.untrackableAckSignalled.delete(id);
|
|
9540
10999
|
this.untrackableAckWarned.delete(id);
|
|
9541
11000
|
this.log({
|
|
@@ -9575,7 +11034,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9575
11034
|
bySession.set(row.opencode_session_id, list);
|
|
9576
11035
|
}
|
|
9577
11036
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9578
|
-
let messages;
|
|
11037
|
+
let messages = null;
|
|
9579
11038
|
try {
|
|
9580
11039
|
if (this.isV2) {
|
|
9581
11040
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9584,9 +11043,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9584
11043
|
level: "warn",
|
|
9585
11044
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9586
11045
|
});
|
|
9587
|
-
|
|
11046
|
+
} else {
|
|
11047
|
+
messages = snapshot;
|
|
9588
11048
|
}
|
|
9589
|
-
messages = snapshot;
|
|
9590
11049
|
} else {
|
|
9591
11050
|
const polled = await pollSessionMessagesForRedrive(
|
|
9592
11051
|
this.port,
|
|
@@ -9599,15 +11058,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9599
11058
|
level: "warn",
|
|
9600
11059
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9601
11060
|
});
|
|
9602
|
-
|
|
11061
|
+
} else {
|
|
11062
|
+
messages = polled.messages;
|
|
9603
11063
|
}
|
|
9604
|
-
messages = polled.messages;
|
|
9605
11064
|
}
|
|
9606
11065
|
} catch (err) {
|
|
9607
11066
|
this.log({
|
|
9608
11067
|
level: "warn",
|
|
9609
11068
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9610
11069
|
});
|
|
11070
|
+
}
|
|
11071
|
+
if (messages === null) {
|
|
11072
|
+
for (const row of sessionRows) {
|
|
11073
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
11074
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
11075
|
+
}
|
|
11076
|
+
}
|
|
9611
11077
|
continue;
|
|
9612
11078
|
}
|
|
9613
11079
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9617,6 +11083,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9617
11083
|
}
|
|
9618
11084
|
}
|
|
9619
11085
|
}
|
|
11086
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
11087
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
11088
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
11089
|
+
if (!correlation) return false;
|
|
11090
|
+
correlation.attempts += 1;
|
|
11091
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
11092
|
+
if (correlatedId !== null) {
|
|
11093
|
+
this.awaitingCorrelation.delete(row.id);
|
|
11094
|
+
const conv = this.convForRow(sessionId, row);
|
|
11095
|
+
const message = this.queuedMessageForRow(row);
|
|
11096
|
+
const processedAtMs = this.processedAtMs(row);
|
|
11097
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
11098
|
+
this.dispatched.add(row.id);
|
|
11099
|
+
this.readopted.add(row.id);
|
|
11100
|
+
this.ensureWatcherRunning(sessionId);
|
|
11101
|
+
this.log({
|
|
11102
|
+
level: "info",
|
|
11103
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
11104
|
+
conversation_id: row.conversation_id,
|
|
11105
|
+
message_id: row.id
|
|
11106
|
+
});
|
|
11107
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
11108
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
11109
|
+
opencode_message_id: correlatedId
|
|
11110
|
+
});
|
|
11111
|
+
return true;
|
|
11112
|
+
}
|
|
11113
|
+
if (correlation.knownUserIds === null) {
|
|
11114
|
+
this.log({
|
|
11115
|
+
level: "debug",
|
|
11116
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
11117
|
+
conversation_id: row.conversation_id,
|
|
11118
|
+
message_id: row.id
|
|
11119
|
+
});
|
|
11120
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
11121
|
+
this.log({
|
|
11122
|
+
level: "warn",
|
|
11123
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
11124
|
+
conversation_id: row.conversation_id,
|
|
11125
|
+
message_id: row.id
|
|
11126
|
+
});
|
|
11127
|
+
}
|
|
11128
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
11129
|
+
await this.handoffUntrackableCorrelation(
|
|
11130
|
+
row.conversation_id,
|
|
11131
|
+
row.id,
|
|
11132
|
+
correlation.sessionId,
|
|
11133
|
+
correlation.attempts,
|
|
11134
|
+
"readopt"
|
|
11135
|
+
);
|
|
11136
|
+
}
|
|
11137
|
+
return true;
|
|
11138
|
+
}
|
|
9620
11139
|
/**
|
|
9621
11140
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9622
11141
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9624,8 +11143,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9624
11143
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9625
11144
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9626
11145
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9627
|
-
* before the restart)
|
|
9628
|
-
*
|
|
11146
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
11147
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
11148
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
11149
|
+
* `forceReadoptRun`):
|
|
9629
11150
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9630
11151
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9631
11152
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9634,7 +11155,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9634
11155
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9635
11156
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9636
11157
|
* tracking the stored id so the reply correlates by it;
|
|
9637
|
-
* - `unknown`/null id →
|
|
11158
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
11159
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9638
11160
|
*
|
|
9639
11161
|
* Only `ChannelAuthError` propagates.
|
|
9640
11162
|
*/
|
|
@@ -9648,6 +11170,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9648
11170
|
});
|
|
9649
11171
|
return;
|
|
9650
11172
|
}
|
|
11173
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9651
11174
|
const ocId = row.opencode_message_id;
|
|
9652
11175
|
const state = messageRunState(messages, ocId ?? "");
|
|
9653
11176
|
if (state === "done") {
|
|
@@ -9714,12 +11237,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9714
11237
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9715
11238
|
this.dontRedispatch.delete(row.id);
|
|
9716
11239
|
this.untrackableAck.delete(row.id);
|
|
11240
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9717
11241
|
this.untrackableAckSignalled.delete(row.id);
|
|
9718
11242
|
this.untrackableAckWarned.delete(row.id);
|
|
9719
11243
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9720
11244
|
return;
|
|
9721
11245
|
}
|
|
9722
|
-
if (this.
|
|
11246
|
+
if (this.untrackableAck.has(row.id)) {
|
|
11247
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
11248
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
11249
|
+
this.untrackableAckWarned.add(row.id);
|
|
11250
|
+
this.log({
|
|
11251
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
11252
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
11253
|
+
conversation_id: row.conversation_id,
|
|
11254
|
+
message_id: row.id
|
|
11255
|
+
});
|
|
11256
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
11257
|
+
const reportError = await this.reportUntrackableAck(
|
|
11258
|
+
row.conversation_id,
|
|
11259
|
+
row.id,
|
|
11260
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
11261
|
+
errorMessage3
|
|
11262
|
+
);
|
|
11263
|
+
if (reportError !== null) {
|
|
11264
|
+
this.log({
|
|
11265
|
+
level: "warn",
|
|
11266
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
11267
|
+
conversation_id: row.conversation_id,
|
|
11268
|
+
message_id: row.id
|
|
11269
|
+
});
|
|
11270
|
+
}
|
|
11271
|
+
} else {
|
|
11272
|
+
this.log({
|
|
11273
|
+
level: "debug",
|
|
11274
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
11275
|
+
conversation_id: row.conversation_id,
|
|
11276
|
+
message_id: row.id
|
|
11277
|
+
});
|
|
11278
|
+
}
|
|
11279
|
+
return;
|
|
11280
|
+
}
|
|
11281
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9723
11282
|
this.log({
|
|
9724
11283
|
level: "debug",
|
|
9725
11284
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9897,6 +11456,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9897
11456
|
}
|
|
9898
11457
|
this.dontRedispatch.delete(row.id);
|
|
9899
11458
|
this.untrackableAck.delete(row.id);
|
|
11459
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9900
11460
|
this.untrackableAckSignalled.delete(row.id);
|
|
9901
11461
|
this.untrackableAckWarned.delete(row.id);
|
|
9902
11462
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9916,8 +11476,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9916
11476
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9917
11477
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9918
11478
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9919
|
-
* re-entry)
|
|
9920
|
-
*
|
|
11479
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
11480
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
11481
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9921
11482
|
*
|
|
9922
11483
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9923
11484
|
* `processed_at` (Invariant 1).
|
|
@@ -9980,6 +11541,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9980
11541
|
);
|
|
9981
11542
|
}
|
|
9982
11543
|
let ocId;
|
|
11544
|
+
let preDispatchUserIds = null;
|
|
9983
11545
|
try {
|
|
9984
11546
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9985
11547
|
sessionId,
|
|
@@ -9989,7 +11551,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9989
11551
|
row.content,
|
|
9990
11552
|
options,
|
|
9991
11553
|
sendAttachments,
|
|
9992
|
-
this.openCodeClient
|
|
11554
|
+
this.openCodeClient,
|
|
11555
|
+
{
|
|
11556
|
+
onKnownUserIds: (ids) => {
|
|
11557
|
+
preDispatchUserIds = ids;
|
|
11558
|
+
},
|
|
11559
|
+
onKnownUserIdsError: (error2) => {
|
|
11560
|
+
this.log({
|
|
11561
|
+
level: "warn",
|
|
11562
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
11563
|
+
conversation_id: row.conversation_id,
|
|
11564
|
+
message_id: row.id
|
|
11565
|
+
});
|
|
11566
|
+
}
|
|
11567
|
+
}
|
|
9993
11568
|
)
|
|
9994
11569
|
);
|
|
9995
11570
|
} catch (err) {
|
|
@@ -10022,6 +11597,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10022
11597
|
}
|
|
10023
11598
|
} else {
|
|
10024
11599
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
11600
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
10025
11601
|
}
|
|
10026
11602
|
this.log({
|
|
10027
11603
|
level: "error",
|
|
@@ -10070,39 +11646,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10070
11646
|
}
|
|
10071
11647
|
if (ocId === null) {
|
|
10072
11648
|
this.awaitingReadopt.delete(row.id);
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
this.log({
|
|
10080
|
-
level: "error",
|
|
10081
|
-
message: errorMessage3,
|
|
10082
|
-
conversation_id: row.conversation_id,
|
|
10083
|
-
message_id: row.id
|
|
10084
|
-
});
|
|
10085
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
10086
|
-
this.log({
|
|
10087
|
-
level: "warn",
|
|
10088
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
10089
|
-
conversation_id: row.conversation_id,
|
|
10090
|
-
message_id: row.id
|
|
10091
|
-
});
|
|
10092
|
-
});
|
|
10093
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10094
|
-
return;
|
|
10095
|
-
}
|
|
11649
|
+
this.awaitingCorrelation.set(row.id, {
|
|
11650
|
+
sessionId,
|
|
11651
|
+
content: row.content,
|
|
11652
|
+
attempts: 0,
|
|
11653
|
+
knownUserIds: preDispatchUserIds
|
|
11654
|
+
});
|
|
10096
11655
|
this.log({
|
|
10097
11656
|
level: "warn",
|
|
10098
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
11657
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
10099
11658
|
conversation_id: row.conversation_id,
|
|
10100
11659
|
message_id: row.id
|
|
10101
11660
|
});
|
|
10102
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
10103
11661
|
return;
|
|
10104
11662
|
}
|
|
10105
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
10106
11663
|
const processedAtMs = this.processedAtMs(row);
|
|
10107
11664
|
this.registerReadopted(
|
|
10108
11665
|
readoptConv,
|
|
@@ -10441,14 +11998,14 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
10441
11998
|
const fetchChildWithoutBlocking = async (sessionId) => {
|
|
10442
11999
|
const pending = fetchChildMessages(sessionId);
|
|
10443
12000
|
let timer;
|
|
10444
|
-
const timeout = new Promise((
|
|
12001
|
+
const timeout = new Promise((resolve5) => {
|
|
10445
12002
|
timer = setTimeout(() => {
|
|
10446
12003
|
this.log({
|
|
10447
12004
|
level: "warn",
|
|
10448
12005
|
message: `Best-effort subagent session fetch for message ${messageId.slice(0, 8)} and child session ${sessionId.slice(0, 8)} was slow \u2014 omitting invocation telemetry without delaying completion`,
|
|
10449
12006
|
message_id: messageId
|
|
10450
12007
|
});
|
|
10451
|
-
|
|
12008
|
+
resolve5(null);
|
|
10452
12009
|
}, SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS);
|
|
10453
12010
|
});
|
|
10454
12011
|
try {
|
|
@@ -11008,7 +12565,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11008
12565
|
* watcher retries next tick within the
|
|
11009
12566
|
* deadline, Finding 4).
|
|
11010
12567
|
*/
|
|
11011
|
-
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations) {
|
|
12568
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title, usage, usageAgentName, subagentInvocations, error2) {
|
|
11012
12569
|
const res = await this.fetchImpl(
|
|
11013
12570
|
`${this.apiUrl}/runners/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
11014
12571
|
{
|
|
@@ -11026,7 +12583,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11026
12583
|
...title ? { title } : {},
|
|
11027
12584
|
...usage ? usage : {},
|
|
11028
12585
|
...usageAgentName ? { usage_agent_name: usageAgentName } : {},
|
|
11029
|
-
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {}
|
|
12586
|
+
...subagentInvocations && subagentInvocations.length > 0 ? { subagent_invocations: subagentInvocations } : {},
|
|
12587
|
+
...error2 ? { error: error2 } : {}
|
|
11030
12588
|
})
|
|
11031
12589
|
}
|
|
11032
12590
|
);
|
|
@@ -11086,11 +12644,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11086
12644
|
if (subagentInvocations && subagentInvocations.length > 0) {
|
|
11087
12645
|
body.subagent_invocations = subagentInvocations;
|
|
11088
12646
|
}
|
|
11089
|
-
if (failure) {
|
|
11090
|
-
body.
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
body.
|
|
12647
|
+
if (failure && failure.errorShape.name !== null) {
|
|
12648
|
+
body.failure_error_name = failure.errorShape.name;
|
|
12649
|
+
}
|
|
12650
|
+
if (failure && failure.errorShape.statusCode !== null) {
|
|
12651
|
+
body.failure_status_code = failure.errorShape.statusCode;
|
|
12652
|
+
}
|
|
12653
|
+
if (failure?.classification) {
|
|
12654
|
+
body.failure_kind = failure.classification.kind;
|
|
12655
|
+
body.failure_provider_id = failure.classification.providerId;
|
|
12656
|
+
body.failure_model_id = failure.classification.modelId;
|
|
12657
|
+
body.failure_reason = failure.classification.reason;
|
|
11094
12658
|
}
|
|
11095
12659
|
try {
|
|
11096
12660
|
await this.callWithRetry(
|
|
@@ -11111,27 +12675,28 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
11111
12675
|
this.clearSubagentInvocationCaches(messageId);
|
|
11112
12676
|
}
|
|
11113
12677
|
/**
|
|
11114
|
-
* Classify an errored turn
|
|
12678
|
+
* Classify an errored turn and extract its bounded error shape for `markFailed`.
|
|
11115
12679
|
*
|
|
11116
12680
|
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
11117
12681
|
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
11118
12682
|
* to the P1-2b zero-provider check — one extra loopback call to
|
|
11119
12683
|
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
11120
12684
|
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
11121
|
-
* (`null`/indeterminate) leaves the classification `null
|
|
11122
|
-
*
|
|
12685
|
+
* (`null`/indeterminate) leaves the classification `null`; the raw error shape
|
|
12686
|
+
* is still retained for diagnostics.
|
|
11123
12687
|
*/
|
|
11124
12688
|
async classifyModelAuthFailure(messages, userMessageId) {
|
|
11125
|
-
const classified = messageFailure(messages, userMessageId);
|
|
11126
|
-
if (classified != null) return classified;
|
|
11127
12689
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
11128
|
-
const
|
|
11129
|
-
|
|
12690
|
+
const errorShape = extractReplyErrorShape(reply);
|
|
12691
|
+
const classified = messageFailure(messages, userMessageId);
|
|
12692
|
+
if (classified != null) return { classification: classified, errorShape };
|
|
12693
|
+
const classification = applyZeroProviderFallback(
|
|
11130
12694
|
classified,
|
|
11131
|
-
|
|
12695
|
+
await this.hasAnyConfiguredProvider(),
|
|
11132
12696
|
reply?.info?.providerID ?? null,
|
|
11133
12697
|
reply?.info?.modelID ?? null
|
|
11134
12698
|
);
|
|
12699
|
+
return { classification, errorShape };
|
|
11135
12700
|
}
|
|
11136
12701
|
async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
|
|
11137
12702
|
const providerId = failure.providerId ?? "(unknown)";
|
|
@@ -11689,7 +13254,7 @@ function commandError2(result) {
|
|
|
11689
13254
|
return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
|
|
11690
13255
|
}
|
|
11691
13256
|
var runCommand2 = (command, args, opts) => {
|
|
11692
|
-
return new Promise((
|
|
13257
|
+
return new Promise((resolve5) => {
|
|
11693
13258
|
let child;
|
|
11694
13259
|
let stdout = "";
|
|
11695
13260
|
let stderr = "";
|
|
@@ -11699,7 +13264,7 @@ var runCommand2 = (command, args, opts) => {
|
|
|
11699
13264
|
if (settled) return;
|
|
11700
13265
|
settled = true;
|
|
11701
13266
|
if (timer.handle) clearTimeout(timer.handle);
|
|
11702
|
-
|
|
13267
|
+
resolve5(result);
|
|
11703
13268
|
};
|
|
11704
13269
|
try {
|
|
11705
13270
|
child = spawn5(command, args, { env: opts.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
@@ -11754,7 +13319,9 @@ async function installRunnerSecret({
|
|
|
11754
13319
|
}) {
|
|
11755
13320
|
const arn = env.RUNNER_SECRET_ARN?.trim();
|
|
11756
13321
|
if (!arn) {
|
|
11757
|
-
|
|
13322
|
+
if (!env.GH_TOKEN) {
|
|
13323
|
+
log3("runner secret is not configured; continuing without GitHub and MCP credentials");
|
|
13324
|
+
}
|
|
11758
13325
|
return false;
|
|
11759
13326
|
}
|
|
11760
13327
|
const result = await (commandRunner ?? runCommand2)(
|
|
@@ -11953,7 +13520,7 @@ async function configureGitHubAccess({ env, log: log3 }) {
|
|
|
11953
13520
|
// src/lib/opencode/config-overlay.ts
|
|
11954
13521
|
import { execFileSync as execFileSync2 } from "node:child_process";
|
|
11955
13522
|
import { copyFileSync, existsSync as existsSync2, statSync as statSync6 } from "node:fs";
|
|
11956
|
-
import { isAbsolute as
|
|
13523
|
+
import { isAbsolute as isAbsolute3, join as join11, resolve as resolve4 } from "node:path";
|
|
11957
13524
|
function isFile(filePath) {
|
|
11958
13525
|
return existsSync2(filePath) && statSync6(filePath).isFile();
|
|
11959
13526
|
}
|
|
@@ -11966,8 +13533,8 @@ function applyRunnerOpenCodeConfig({
|
|
|
11966
13533
|
log3("runner OpenCode config is not configured; using the baked project config", "debug");
|
|
11967
13534
|
return;
|
|
11968
13535
|
}
|
|
11969
|
-
const source =
|
|
11970
|
-
const target = isFile(
|
|
13536
|
+
const source = isAbsolute3(overlayPath) ? overlayPath : resolve4(cwd, overlayPath);
|
|
13537
|
+
const target = isFile(join11(cwd, "opencode.jsonc")) ? "opencode.jsonc" : "opencode.json";
|
|
11971
13538
|
if (!isFile(source)) {
|
|
11972
13539
|
log3(
|
|
11973
13540
|
`RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)`,
|
|
@@ -11975,7 +13542,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
11975
13542
|
);
|
|
11976
13543
|
return;
|
|
11977
13544
|
}
|
|
11978
|
-
copyFileSync(source,
|
|
13545
|
+
copyFileSync(source, join11(cwd, target));
|
|
11979
13546
|
try {
|
|
11980
13547
|
execFileSync2("git", ["-C", cwd, "update-index", "--skip-worktree", target], {
|
|
11981
13548
|
stdio: "ignore"
|
|
@@ -11984,7 +13551,7 @@ function applyRunnerOpenCodeConfig({
|
|
|
11984
13551
|
const detail = error2 instanceof Error ? error2.message : String(error2);
|
|
11985
13552
|
log3(`could not mark ${target} skip-worktree: ${detail}; it may show as a local change`, "warn");
|
|
11986
13553
|
}
|
|
11987
|
-
log3(`Applied runner OpenCode config ${source} to ${
|
|
13554
|
+
log3(`Applied runner OpenCode config ${source} to ${join11(cwd, target)}`);
|
|
11988
13555
|
}
|
|
11989
13556
|
|
|
11990
13557
|
// src/lib/credential-sync.ts
|
|
@@ -12002,14 +13569,14 @@ function errorMessage2(error2) {
|
|
|
12002
13569
|
return error2 instanceof Error ? error2.message : String(error2);
|
|
12003
13570
|
}
|
|
12004
13571
|
function waitForSettlement(promise, timeoutMs) {
|
|
12005
|
-
return new Promise((
|
|
13572
|
+
return new Promise((resolve5) => {
|
|
12006
13573
|
let settled = false;
|
|
12007
13574
|
const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
|
|
12008
13575
|
const finish = (value) => {
|
|
12009
13576
|
if (settled) return;
|
|
12010
13577
|
settled = true;
|
|
12011
13578
|
clearTimeout(timer);
|
|
12012
|
-
|
|
13579
|
+
resolve5(value);
|
|
12013
13580
|
};
|
|
12014
13581
|
promise.then(
|
|
12015
13582
|
() => finish(true),
|
|
@@ -12224,7 +13791,10 @@ function createCredentialSync({
|
|
|
12224
13791
|
|
|
12225
13792
|
// src/commands/run.ts
|
|
12226
13793
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
13794
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12227
13795
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
13796
|
+
var CREDENTIAL_ADOPTION_PROBE_ATTEMPTS = 3;
|
|
13797
|
+
var CREDENTIAL_ADOPTION_PROBE_INTERVAL_MS = 1e3;
|
|
12228
13798
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12229
13799
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
12230
13800
|
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
@@ -12259,8 +13829,8 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
12259
13829
|
if (trimmed === "") {
|
|
12260
13830
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
12261
13831
|
}
|
|
12262
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
12263
|
-
if (!
|
|
13832
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join12(homeDir, trimmed.slice(2)) : trimmed;
|
|
13833
|
+
if (!isAbsolute4(expanded)) {
|
|
12264
13834
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
12265
13835
|
}
|
|
12266
13836
|
const normalized = resolvePath2(expanded);
|
|
@@ -12360,8 +13930,7 @@ function resolveMaxActiveSessions(options, env = process.env) {
|
|
|
12360
13930
|
function meetsThreshold(state, level) {
|
|
12361
13931
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
12362
13932
|
}
|
|
12363
|
-
function
|
|
12364
|
-
if (!meetsThreshold(state, level)) return;
|
|
13933
|
+
function writeConsoleLog(state, message, level) {
|
|
12365
13934
|
if (state.json) {
|
|
12366
13935
|
console.log(
|
|
12367
13936
|
JSON.stringify({
|
|
@@ -12370,10 +13939,18 @@ function log2(state, message, level = "info") {
|
|
|
12370
13939
|
message
|
|
12371
13940
|
})
|
|
12372
13941
|
);
|
|
12373
|
-
|
|
12374
|
-
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
12375
|
-
console.log(`${prefix} ${message}`);
|
|
13942
|
+
return;
|
|
12376
13943
|
}
|
|
13944
|
+
const prefix = level === "error" ? chalk7.red("\u2717") : level === "warn" ? chalk7.yellow("!") : level === "debug" ? chalk7.dim("\xB7") : chalk7.green("\u2022");
|
|
13945
|
+
console.log(`${prefix} ${message}`);
|
|
13946
|
+
}
|
|
13947
|
+
function log2(state, message, level = "info") {
|
|
13948
|
+
if (!meetsThreshold(state, level)) return;
|
|
13949
|
+
if (state.json || !state.interactive) writeConsoleLog(state, message, level);
|
|
13950
|
+
}
|
|
13951
|
+
function logLocalOnly(state, message, level = "info") {
|
|
13952
|
+
if (!meetsThreshold(state, level)) return;
|
|
13953
|
+
writeConsoleLog(state, message, level);
|
|
12377
13954
|
}
|
|
12378
13955
|
function logActivity(state, entry) {
|
|
12379
13956
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
@@ -12442,7 +14019,9 @@ function displayStatus(state) {
|
|
|
12442
14019
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12443
14020
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12444
14021
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12445
|
-
const detail = last ? chalk7.dim(
|
|
14022
|
+
const detail = last ? chalk7.dim(
|
|
14023
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
14024
|
+
) : "";
|
|
12446
14025
|
const agent = state.agentName ?? state.agentId;
|
|
12447
14026
|
console.log(
|
|
12448
14027
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12461,7 +14040,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12461
14040
|
logActivity(state, {
|
|
12462
14041
|
type: "info",
|
|
12463
14042
|
level: "warn",
|
|
12464
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
14043
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12465
14044
|
});
|
|
12466
14045
|
}
|
|
12467
14046
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12563,6 +14142,125 @@ async function handleAuthError(state, error2) {
|
|
|
12563
14142
|
return { success: false };
|
|
12564
14143
|
}
|
|
12565
14144
|
}
|
|
14145
|
+
function logCredentialAdoptionActivity(state, activity, level) {
|
|
14146
|
+
const message = (() => {
|
|
14147
|
+
switch (activity.outcome) {
|
|
14148
|
+
case "deferred":
|
|
14149
|
+
return `CREDENTIAL-ADOPTION: deferred generation ${activity.generation} with ${activity.active_sessions} active session(s)`;
|
|
14150
|
+
case "reloaded_ready":
|
|
14151
|
+
return `CREDENTIAL-ADOPTION: adopted generation ${activity.generation} with ${activity.active_sessions} active session(s); readiness probe succeeded after ${activity.probe_attempts} attempt(s)`;
|
|
14152
|
+
case "reloaded_interrupted": {
|
|
14153
|
+
const response = activity.probe_status === null ? "without an HTTP response" : `with HTTP ${activity.probe_status} and ${activity.probe_body_empty ? "an empty" : "a non-empty"} body`;
|
|
14154
|
+
return `CREDENTIAL-ADOPTION: generation ${activity.generation} with ${activity.active_sessions} active session(s); replacement instance answered ${response} after ${activity.probe_attempts} probe attempt(s)`;
|
|
14155
|
+
}
|
|
14156
|
+
case "reload_failed":
|
|
14157
|
+
return `CREDENTIAL-ADOPTION: failed to adopt generation ${activity.generation} with ${activity.active_sessions} active session(s)`;
|
|
14158
|
+
case "skipped_v2":
|
|
14159
|
+
return `CREDENTIAL-ADOPTION: skipped generation ${activity.generation} with ${activity.active_sessions} active session(s) on OpenCode V2`;
|
|
14160
|
+
}
|
|
14161
|
+
})();
|
|
14162
|
+
logActivity(state, {
|
|
14163
|
+
type: level === "error" ? "error" : "info",
|
|
14164
|
+
level,
|
|
14165
|
+
...level === "error" ? { error: message } : { message },
|
|
14166
|
+
metadata: {
|
|
14167
|
+
...activity,
|
|
14168
|
+
opencode_version: state.opencodeVersion,
|
|
14169
|
+
port: state.port
|
|
14170
|
+
}
|
|
14171
|
+
});
|
|
14172
|
+
}
|
|
14173
|
+
async function adoptCredentialGeneration(state, generation, adoption, activeSessions, deferredForMs) {
|
|
14174
|
+
const reload = await disposeAndReloadProviderConfig(
|
|
14175
|
+
state.port,
|
|
14176
|
+
state.opencodeClient ?? void 0
|
|
14177
|
+
);
|
|
14178
|
+
if (reload.outcome === "failed") {
|
|
14179
|
+
logCredentialAdoptionActivity(
|
|
14180
|
+
state,
|
|
14181
|
+
{
|
|
14182
|
+
outcome: "reload_failed",
|
|
14183
|
+
generation,
|
|
14184
|
+
active_sessions: activeSessions,
|
|
14185
|
+
adoption,
|
|
14186
|
+
deferred_for_ms: deferredForMs,
|
|
14187
|
+
reload_status: reload.status,
|
|
14188
|
+
reload_detail: reload.detail,
|
|
14189
|
+
probe_attempts: null,
|
|
14190
|
+
probe_status: null,
|
|
14191
|
+
probe_body_empty: null,
|
|
14192
|
+
probe_detail: null
|
|
14193
|
+
},
|
|
14194
|
+
"error"
|
|
14195
|
+
);
|
|
14196
|
+
return;
|
|
14197
|
+
}
|
|
14198
|
+
if (reload.outcome === "skipped_v2") {
|
|
14199
|
+
logCredentialAdoptionActivity(
|
|
14200
|
+
state,
|
|
14201
|
+
{
|
|
14202
|
+
outcome: "skipped_v2",
|
|
14203
|
+
generation,
|
|
14204
|
+
active_sessions: activeSessions,
|
|
14205
|
+
adoption,
|
|
14206
|
+
deferred_for_ms: deferredForMs,
|
|
14207
|
+
reload_status: null,
|
|
14208
|
+
reload_detail: null,
|
|
14209
|
+
probe_attempts: null,
|
|
14210
|
+
probe_status: null,
|
|
14211
|
+
probe_body_empty: null,
|
|
14212
|
+
probe_detail: null
|
|
14213
|
+
},
|
|
14214
|
+
"debug"
|
|
14215
|
+
);
|
|
14216
|
+
return;
|
|
14217
|
+
}
|
|
14218
|
+
let lastProbe = null;
|
|
14219
|
+
let attempts = 0;
|
|
14220
|
+
for (attempts = 1; attempts <= CREDENTIAL_ADOPTION_PROBE_ATTEMPTS; attempts++) {
|
|
14221
|
+
lastProbe = await probeOpenCodeRequestReadiness(state.port, state.opencodeClient ?? void 0);
|
|
14222
|
+
if (lastProbe.ok) {
|
|
14223
|
+
logCredentialAdoptionActivity(
|
|
14224
|
+
state,
|
|
14225
|
+
{
|
|
14226
|
+
outcome: "reloaded_ready",
|
|
14227
|
+
generation,
|
|
14228
|
+
active_sessions: activeSessions,
|
|
14229
|
+
adoption,
|
|
14230
|
+
deferred_for_ms: deferredForMs,
|
|
14231
|
+
reload_status: null,
|
|
14232
|
+
reload_detail: null,
|
|
14233
|
+
probe_attempts: attempts,
|
|
14234
|
+
probe_status: lastProbe.status,
|
|
14235
|
+
probe_body_empty: lastProbe.bodyEmpty,
|
|
14236
|
+
probe_detail: lastProbe.detail
|
|
14237
|
+
},
|
|
14238
|
+
"warn"
|
|
14239
|
+
);
|
|
14240
|
+
return;
|
|
14241
|
+
}
|
|
14242
|
+
if (attempts < CREDENTIAL_ADOPTION_PROBE_ATTEMPTS) {
|
|
14243
|
+
await new Promise((resolve5) => setTimeout(resolve5, CREDENTIAL_ADOPTION_PROBE_INTERVAL_MS));
|
|
14244
|
+
}
|
|
14245
|
+
}
|
|
14246
|
+
logCredentialAdoptionActivity(
|
|
14247
|
+
state,
|
|
14248
|
+
{
|
|
14249
|
+
outcome: "reloaded_interrupted",
|
|
14250
|
+
generation,
|
|
14251
|
+
active_sessions: activeSessions,
|
|
14252
|
+
adoption,
|
|
14253
|
+
deferred_for_ms: deferredForMs,
|
|
14254
|
+
reload_status: null,
|
|
14255
|
+
reload_detail: null,
|
|
14256
|
+
probe_attempts: CREDENTIAL_ADOPTION_PROBE_ATTEMPTS,
|
|
14257
|
+
probe_status: lastProbe?.status ?? null,
|
|
14258
|
+
probe_body_empty: lastProbe?.bodyEmpty ?? null,
|
|
14259
|
+
probe_detail: lastProbe?.detail ?? null
|
|
14260
|
+
},
|
|
14261
|
+
"error"
|
|
14262
|
+
);
|
|
14263
|
+
}
|
|
12566
14264
|
async function driveChannels(state, driver) {
|
|
12567
14265
|
let idlePolls = 0;
|
|
12568
14266
|
let idleMs = 0;
|
|
@@ -12572,6 +14270,15 @@ async function driveChannels(state, driver) {
|
|
|
12572
14270
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
12573
14271
|
let lastSeenClaudeApplies = driver.fileSyncActivity().claudeCredentialApplies;
|
|
12574
14272
|
let lastSeenOpencodeAuthApplies = driver.fileSyncActivity().opencodeAuthApplyCount;
|
|
14273
|
+
const credentialAdoption = {
|
|
14274
|
+
generation: 0,
|
|
14275
|
+
pendingGeneration: null,
|
|
14276
|
+
pendingSinceMs: null,
|
|
14277
|
+
deferredGeneration: null,
|
|
14278
|
+
adoption: 0
|
|
14279
|
+
};
|
|
14280
|
+
const stayAwakeHolds = () => state.stayAwakeUntilMs !== null && Date.now() < state.stayAwakeUntilMs;
|
|
14281
|
+
let stayAwakeSuppressionLogged = false;
|
|
12575
14282
|
while (state.running) {
|
|
12576
14283
|
const cycleStartedAtMs = performance.now();
|
|
12577
14284
|
let idleThisCycle = false;
|
|
@@ -12613,12 +14320,30 @@ async function driveChannels(state, driver) {
|
|
|
12613
14320
|
lastSeenOpencodeAuthApplies = opencodeAuthApplies;
|
|
12614
14321
|
if (opencodeAuthApplied) state.openaiUsageRearm?.();
|
|
12615
14322
|
if (claudeCredentialApplied || opencodeAuthApplied) {
|
|
12616
|
-
|
|
12617
|
-
|
|
12618
|
-
|
|
12619
|
-
|
|
12620
|
-
|
|
12621
|
-
|
|
14323
|
+
credentialAdoption.generation++;
|
|
14324
|
+
if (state.opencodeApiVersion === "v2") {
|
|
14325
|
+
logCredentialAdoptionActivity(
|
|
14326
|
+
state,
|
|
14327
|
+
{
|
|
14328
|
+
outcome: "skipped_v2",
|
|
14329
|
+
generation: credentialAdoption.generation,
|
|
14330
|
+
active_sessions: driver.protectedSessionIds().size,
|
|
14331
|
+
adoption: null,
|
|
14332
|
+
deferred_for_ms: null,
|
|
14333
|
+
reload_status: null,
|
|
14334
|
+
reload_detail: null,
|
|
14335
|
+
probe_attempts: null,
|
|
14336
|
+
probe_status: null,
|
|
14337
|
+
probe_body_empty: null,
|
|
14338
|
+
probe_detail: null
|
|
14339
|
+
},
|
|
14340
|
+
"debug"
|
|
14341
|
+
);
|
|
14342
|
+
} else {
|
|
14343
|
+
credentialAdoption.pendingGeneration = credentialAdoption.generation;
|
|
14344
|
+
credentialAdoption.pendingSinceMs = Date.now();
|
|
14345
|
+
credentialAdoption.deferredGeneration = null;
|
|
14346
|
+
}
|
|
12622
14347
|
}
|
|
12623
14348
|
if (opencodeAuthApplied && state.opencodeApiVersion === "v2") {
|
|
12624
14349
|
void importOpenCodeV2CredentialsForState(state).catch(
|
|
@@ -12673,7 +14398,50 @@ async function driveChannels(state, driver) {
|
|
|
12673
14398
|
}
|
|
12674
14399
|
}
|
|
12675
14400
|
}
|
|
12676
|
-
|
|
14401
|
+
if (credentialAdoption.pendingGeneration !== null) {
|
|
14402
|
+
const activeSessions = driver.protectedSessionIds();
|
|
14403
|
+
const generation = credentialAdoption.pendingGeneration;
|
|
14404
|
+
if (activeSessions.size > 0) {
|
|
14405
|
+
if (credentialAdoption.deferredGeneration !== generation) {
|
|
14406
|
+
credentialAdoption.deferredGeneration = generation;
|
|
14407
|
+
logCredentialAdoptionActivity(
|
|
14408
|
+
state,
|
|
14409
|
+
{
|
|
14410
|
+
outcome: "deferred",
|
|
14411
|
+
generation,
|
|
14412
|
+
active_sessions: activeSessions.size,
|
|
14413
|
+
adoption: null,
|
|
14414
|
+
deferred_for_ms: null,
|
|
14415
|
+
reload_status: null,
|
|
14416
|
+
reload_detail: null,
|
|
14417
|
+
probe_attempts: null,
|
|
14418
|
+
probe_status: null,
|
|
14419
|
+
probe_body_empty: null,
|
|
14420
|
+
probe_detail: null
|
|
14421
|
+
},
|
|
14422
|
+
"warn"
|
|
14423
|
+
);
|
|
14424
|
+
}
|
|
14425
|
+
} else {
|
|
14426
|
+
const deferredForMs = credentialAdoption.deferredGeneration === generation && credentialAdoption.pendingSinceMs !== null ? Math.max(0, Date.now() - credentialAdoption.pendingSinceMs) : null;
|
|
14427
|
+
credentialAdoption.pendingGeneration = null;
|
|
14428
|
+
credentialAdoption.pendingSinceMs = null;
|
|
14429
|
+
credentialAdoption.adoption++;
|
|
14430
|
+
void adoptCredentialGeneration(
|
|
14431
|
+
state,
|
|
14432
|
+
generation,
|
|
14433
|
+
credentialAdoption.adoption,
|
|
14434
|
+
activeSessions.size,
|
|
14435
|
+
deferredForMs
|
|
14436
|
+
).catch(
|
|
14437
|
+
(error2) => logActivity(state, {
|
|
14438
|
+
type: "error",
|
|
14439
|
+
error: `CREDENTIAL-ADOPTION: adoption failed for generation ${generation} with ${activeSessions.size} active session(s) on port ${state.port}: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
14440
|
+
})
|
|
14441
|
+
);
|
|
14442
|
+
}
|
|
14443
|
+
}
|
|
14444
|
+
await new Promise((resolve5) => setTimeout(resolve5, CHANNEL_POLL_INTERVAL_MS));
|
|
12677
14445
|
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
12678
14446
|
if (idleThisCycle) idleMs += cycleMs;
|
|
12679
14447
|
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
@@ -12686,17 +14454,29 @@ async function driveChannels(state, driver) {
|
|
|
12686
14454
|
if (state.interactive) displayStatus(state);
|
|
12687
14455
|
break;
|
|
12688
14456
|
}
|
|
12689
|
-
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
|
|
14457
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && !stayAwakeHolds()) {
|
|
12690
14458
|
logActivity(state, { type: "info", message: "Idle timeout reached" });
|
|
12691
14459
|
if (state.interactive) displayStatus(state);
|
|
12692
14460
|
break;
|
|
12693
14461
|
}
|
|
14462
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3 && stayAwakeHolds()) {
|
|
14463
|
+
if (!stayAwakeSuppressionLogged) {
|
|
14464
|
+
stayAwakeSuppressionLogged = true;
|
|
14465
|
+
logActivity(state, {
|
|
14466
|
+
type: "info",
|
|
14467
|
+
message: "Idle, but staying awake until the requested minimum passes"
|
|
14468
|
+
});
|
|
14469
|
+
}
|
|
14470
|
+
}
|
|
12694
14471
|
}
|
|
12695
14472
|
}
|
|
12696
14473
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12697
|
-
var
|
|
14474
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
14475
|
+
var SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES = 2;
|
|
14476
|
+
var SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES = 2;
|
|
14477
|
+
var MAX_SESSION_DELETE_BODY_LENGTH = 200;
|
|
12698
14478
|
function shouldWarnForReclaimSkip(reason) {
|
|
12699
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
14479
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12700
14480
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12701
14481
|
if (!version2) return false;
|
|
12702
14482
|
const major = Number(version2[1]);
|
|
@@ -12705,7 +14485,7 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12705
14485
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12706
14486
|
}
|
|
12707
14487
|
function sessionDbPath() {
|
|
12708
|
-
return
|
|
14488
|
+
return join12(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
12709
14489
|
}
|
|
12710
14490
|
function logSessionCleanupActivity(state, phase, entry) {
|
|
12711
14491
|
logActivity(state, {
|
|
@@ -12753,61 +14533,146 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12753
14533
|
async function runSweep(state, driver, config) {
|
|
12754
14534
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12755
14535
|
try {
|
|
12756
|
-
const
|
|
12757
|
-
|
|
12758
|
-
|
|
14536
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
14537
|
+
const sessions = enumeration.sessions;
|
|
14538
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
14539
|
+
if (enumerationReason !== null) {
|
|
14540
|
+
logActivity(state, {
|
|
12759
14541
|
type: "info",
|
|
12760
|
-
|
|
12761
|
-
|
|
12762
|
-
outcome: "sweep-skipped",
|
|
12763
|
-
reason: "opencode-unreachable"
|
|
12764
|
-
}
|
|
14542
|
+
level: "warn",
|
|
14543
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12765
14544
|
});
|
|
12766
|
-
return;
|
|
12767
14545
|
}
|
|
12768
|
-
const toDelete = selectSessionsToDelete(
|
|
12769
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12770
|
-
{
|
|
12771
|
-
maxAgeMs: config.maxAgeMs,
|
|
12772
|
-
maxCount: config.maxCount,
|
|
12773
|
-
nowMs: Date.now(),
|
|
12774
|
-
protectedIds: driver.protectedSessionIds()
|
|
12775
|
-
}
|
|
12776
|
-
);
|
|
12777
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12778
14546
|
let deleted = 0;
|
|
12779
14547
|
let failed = 0;
|
|
14548
|
+
let failed4xx = 0;
|
|
14549
|
+
let failed5xx = 0;
|
|
14550
|
+
let failedOtherStatus = 0;
|
|
14551
|
+
let failedThrew = 0;
|
|
12780
14552
|
let skippedNewlyActive = 0;
|
|
12781
|
-
|
|
12782
|
-
|
|
12783
|
-
|
|
12784
|
-
|
|
14553
|
+
let remaining = 0;
|
|
14554
|
+
let protectedNow = driver.protectedSessionIds();
|
|
14555
|
+
if (sessions !== null) {
|
|
14556
|
+
const toDelete = selectSessionsToDelete(
|
|
14557
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
14558
|
+
{
|
|
14559
|
+
maxAgeMs: config.maxAgeMs,
|
|
14560
|
+
maxCount: config.maxCount,
|
|
14561
|
+
nowMs: Date.now(),
|
|
14562
|
+
protectedIds: protectedNow
|
|
14563
|
+
}
|
|
14564
|
+
);
|
|
14565
|
+
const eligibleToDelete = toDelete.filter(
|
|
14566
|
+
(id) => (state.sessionDeleteFailures.get(id)?.isolatedFailures ?? 0) < SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES
|
|
14567
|
+
);
|
|
14568
|
+
const deleteCandidates = eligibleToDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
14569
|
+
remaining = eligibleToDelete.length - deleteCandidates.length;
|
|
14570
|
+
const answeredFailures = [];
|
|
14571
|
+
protectedNow = driver.protectedSessionIds();
|
|
14572
|
+
for (const id of deleteCandidates) {
|
|
14573
|
+
if (protectedNow.has(id)) {
|
|
14574
|
+
skippedNewlyActive++;
|
|
14575
|
+
logActivity(state, {
|
|
14576
|
+
type: "info",
|
|
14577
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
14578
|
+
});
|
|
14579
|
+
continue;
|
|
14580
|
+
}
|
|
14581
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
14582
|
+
if (outcome.ok) {
|
|
14583
|
+
deleted++;
|
|
14584
|
+
} else if (outcome.status === "threw") {
|
|
14585
|
+
failed++;
|
|
14586
|
+
failedThrew++;
|
|
14587
|
+
} else {
|
|
14588
|
+
failed++;
|
|
14589
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
14590
|
+
if (statusClass === 4) failed4xx++;
|
|
14591
|
+
else if (statusClass === 5) failed5xx++;
|
|
14592
|
+
else failedOtherStatus++;
|
|
14593
|
+
answeredFailures.push({
|
|
14594
|
+
id,
|
|
14595
|
+
status: outcome.status,
|
|
14596
|
+
...outcome.body !== void 0 ? { body: outcome.body } : {}
|
|
14597
|
+
});
|
|
14598
|
+
}
|
|
14599
|
+
}
|
|
14600
|
+
const givenUpFailures = [];
|
|
14601
|
+
if (failed > SESSION_CLEANUP_MAX_ISOLATED_DELETE_FAILURES) {
|
|
14602
|
+
for (const { id } of answeredFailures) state.sessionDeleteFailures.delete(id);
|
|
14603
|
+
} else {
|
|
14604
|
+
for (const failure of answeredFailures) {
|
|
14605
|
+
const previous = state.sessionDeleteFailures.get(failure.id);
|
|
14606
|
+
const record = previous?.status === failure.status ? { status: failure.status, isolatedFailures: previous.isolatedFailures + 1 } : { status: failure.status, isolatedFailures: 1 };
|
|
14607
|
+
state.sessionDeleteFailures.set(failure.id, record);
|
|
14608
|
+
if (record.isolatedFailures >= SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES) {
|
|
14609
|
+
givenUpFailures.push(failure);
|
|
14610
|
+
}
|
|
14611
|
+
}
|
|
14612
|
+
}
|
|
14613
|
+
for (const failure of givenUpFailures) {
|
|
14614
|
+
const excerpt = redactDiagnosticText(failure.body ?? "(no response body)").slice(
|
|
14615
|
+
0,
|
|
14616
|
+
MAX_SESSION_DELETE_BODY_LENGTH
|
|
14617
|
+
);
|
|
14618
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12785
14619
|
type: "info",
|
|
12786
|
-
|
|
14620
|
+
level: "warn",
|
|
14621
|
+
message: `Session cleanup: gave up on session ${failure.id} after ${SESSION_CLEANUP_GIVE_UP_AFTER_FAILURES} isolated delete failures (status ${failure.status}): ${excerpt}`,
|
|
14622
|
+
metadata: {
|
|
14623
|
+
outcome: "session-delete-given-up",
|
|
14624
|
+
session_id: failure.id,
|
|
14625
|
+
status: failure.status,
|
|
14626
|
+
body_excerpt: excerpt,
|
|
14627
|
+
given_up_count: givenUpFailures.length
|
|
14628
|
+
}
|
|
12787
14629
|
});
|
|
12788
|
-
continue;
|
|
12789
14630
|
}
|
|
12790
|
-
|
|
12791
|
-
|
|
14631
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
14632
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
14633
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
14634
|
+
logActivity(state, {
|
|
14635
|
+
type: "info",
|
|
14636
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
14637
|
+
});
|
|
12792
14638
|
}
|
|
12793
|
-
const
|
|
12794
|
-
|
|
12795
|
-
|
|
12796
|
-
|
|
12797
|
-
|
|
12798
|
-
|
|
14639
|
+
const sessionMetadata = {
|
|
14640
|
+
sessions_inspected: sessions?.length ?? null,
|
|
14641
|
+
sessions_deleted: deleted,
|
|
14642
|
+
sessions_failed: failed,
|
|
14643
|
+
sessions_failed_4xx: failed4xx,
|
|
14644
|
+
sessions_failed_5xx: failed5xx,
|
|
14645
|
+
sessions_failed_other: failedOtherStatus,
|
|
14646
|
+
sessions_failed_threw: failedThrew,
|
|
14647
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
14648
|
+
sessions_remaining: remaining,
|
|
14649
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
14650
|
+
};
|
|
14651
|
+
const protectedForReclaim = driver.protectedSessionIds();
|
|
12799
14652
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12800
14653
|
dbPath: sessionDbPath(),
|
|
12801
|
-
|
|
12802
|
-
|
|
14654
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
14655
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
14656
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
14657
|
+
hasLiveTurn: protectedForReclaim.size > 0,
|
|
14658
|
+
hasProvenanceAnomaly: state.sessionDbProvenanceAnomaly
|
|
12803
14659
|
});
|
|
12804
14660
|
if (reclaimResult.ok) {
|
|
14661
|
+
state.sessionStoreHealth.observeWritable();
|
|
12805
14662
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12806
14663
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12807
14664
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
14665
|
+
const loopElapsedMs = Math.max(
|
|
14666
|
+
0,
|
|
14667
|
+
reclaimResult.elapsedMs - reclaimResult.checkpointElapsedMs
|
|
14668
|
+
);
|
|
14669
|
+
const reclaimIsSlow = reclaimResult.elapsedMs >= 4 * SESSION_DB_RECLAIM_TIME_BUDGET_MS;
|
|
14670
|
+
const perPageNote = reclaimResult.pagesReclaimed > 0 ? `, per-page ${(loopElapsedMs / reclaimResult.pagesReclaimed).toFixed(2)} ms/page` : "";
|
|
14671
|
+
const slowReclaimNote = reclaimIsSlow ? ` (slow reclaim: total elapsed ${reclaimResult.elapsedMs} ms, checkpoint ${reclaimResult.checkpointElapsedMs} ms, loop ${loopElapsedMs} ms, budget ${SESSION_DB_RECLAIM_TIME_BUDGET_MS} ms${perPageNote})` : "";
|
|
12808
14672
|
logSessionCleanupActivity(state, "sweep", {
|
|
12809
14673
|
type: "info",
|
|
12810
|
-
|
|
14674
|
+
level: reclaimIsSlow ? "warn" : "info",
|
|
14675
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote + slowReclaimNote,
|
|
12811
14676
|
metadata: {
|
|
12812
14677
|
outcome: "reclaimed",
|
|
12813
14678
|
mode: reclaimResult.mode,
|
|
@@ -12815,25 +14680,28 @@ async function runSweep(state, driver, config) {
|
|
|
12815
14680
|
after_bytes: reclaimResult.afterBytes,
|
|
12816
14681
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12817
14682
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12818
|
-
|
|
12819
|
-
|
|
12820
|
-
|
|
12821
|
-
|
|
14683
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
14684
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
14685
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
14686
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
14687
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
14688
|
+
checkpoint_elapsed_ms: reclaimResult.checkpointElapsedMs,
|
|
14689
|
+
...sessionMetadata
|
|
12822
14690
|
}
|
|
12823
14691
|
});
|
|
12824
14692
|
} else {
|
|
14693
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
14694
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
14695
|
+
}
|
|
12825
14696
|
logSessionCleanupActivity(state, "sweep", {
|
|
12826
14697
|
type: "info",
|
|
12827
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
14698
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12828
14699
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12829
14700
|
metadata: {
|
|
12830
14701
|
outcome: "reclaim-skipped",
|
|
12831
14702
|
reason: reclaimResult.skipped,
|
|
12832
14703
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12833
|
-
|
|
12834
|
-
sessions_deleted: deleted,
|
|
12835
|
-
sessions_failed: failed,
|
|
12836
|
-
sessions_skipped_active: skippedNewlyActive
|
|
14704
|
+
...sessionMetadata
|
|
12837
14705
|
}
|
|
12838
14706
|
});
|
|
12839
14707
|
}
|
|
@@ -12928,10 +14796,12 @@ function scheduleUsageReporting(state, params) {
|
|
|
12928
14796
|
let phase = "dormant";
|
|
12929
14797
|
let rearmRequested = false;
|
|
12930
14798
|
const armProbe = () => {
|
|
14799
|
+
if (!state.running) return;
|
|
12931
14800
|
phase = "probe-pending";
|
|
12932
14801
|
params.setTimer(setTimeout(() => void tick(true), params.firstDelayMs()));
|
|
12933
14802
|
};
|
|
12934
14803
|
const scheduleNextTick = () => {
|
|
14804
|
+
if (!state.running) return;
|
|
12935
14805
|
if (rearmRequested) {
|
|
12936
14806
|
rearmRequested = false;
|
|
12937
14807
|
armProbe();
|
|
@@ -12965,6 +14835,7 @@ function scheduleUsageReporting(state, params) {
|
|
|
12965
14835
|
phase = "tick-in-flight";
|
|
12966
14836
|
try {
|
|
12967
14837
|
const usage = await params.fetchUsage();
|
|
14838
|
+
if (!state.running) return;
|
|
12968
14839
|
const result = await params.report(usage);
|
|
12969
14840
|
if (result.ok) {
|
|
12970
14841
|
if (consecutiveFailures > 0) {
|
|
@@ -13090,6 +14961,20 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13090
14961
|
message: `Resource usage collection: ${warning2}`
|
|
13091
14962
|
});
|
|
13092
14963
|
}
|
|
14964
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
14965
|
+
...usage,
|
|
14966
|
+
nowMs: Date.now(),
|
|
14967
|
+
state: state.lowDiskWarningState
|
|
14968
|
+
});
|
|
14969
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
14970
|
+
if (lowDiskDecision.warning !== null) {
|
|
14971
|
+
logActivity(state, {
|
|
14972
|
+
type: "info",
|
|
14973
|
+
level: "warn",
|
|
14974
|
+
message: lowDiskDecision.warning
|
|
14975
|
+
});
|
|
14976
|
+
}
|
|
14977
|
+
if (!state.running) return;
|
|
13093
14978
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13094
14979
|
if (result.ok) {
|
|
13095
14980
|
if (consecutiveFailures > 0) {
|
|
@@ -13122,7 +15007,9 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13122
15007
|
message: `Resource usage reporting failed: ${message}${failureStreakSuffix(consecutiveFailures)}`
|
|
13123
15008
|
});
|
|
13124
15009
|
} finally {
|
|
13125
|
-
state.
|
|
15010
|
+
if (state.running) {
|
|
15011
|
+
state.resourceUsageTimer = setTimeout(() => void tick(), usageReportDelayMs());
|
|
15012
|
+
}
|
|
13126
15013
|
}
|
|
13127
15014
|
};
|
|
13128
15015
|
state.resourceUsageTimer = setTimeout(() => void tick(), firstReportDelayMs());
|
|
@@ -13284,8 +15171,8 @@ async function beginGracefulShutdown(state, trigger) {
|
|
|
13284
15171
|
return true;
|
|
13285
15172
|
}
|
|
13286
15173
|
);
|
|
13287
|
-
const timedOut = new Promise((
|
|
13288
|
-
timer = setTimeout(() =>
|
|
15174
|
+
const timedOut = new Promise((resolve5) => {
|
|
15175
|
+
timer = setTimeout(() => resolve5(false), telemetryBudgetMs);
|
|
13289
15176
|
});
|
|
13290
15177
|
if (!await Promise.race([flushed, timedOut])) {
|
|
13291
15178
|
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
@@ -13325,6 +15212,8 @@ async function run(options) {
|
|
|
13325
15212
|
port: options.port ?? 4096,
|
|
13326
15213
|
conversationFilter: options.conversation ?? null,
|
|
13327
15214
|
idleTimeout: options.idleTimeout ?? null,
|
|
15215
|
+
stayAwakeSeconds: null,
|
|
15216
|
+
stayAwakeUntilMs: null,
|
|
13328
15217
|
json: options.json ?? false,
|
|
13329
15218
|
interactive,
|
|
13330
15219
|
logLevel,
|
|
@@ -13336,6 +15225,7 @@ async function run(options) {
|
|
|
13336
15225
|
opencodeClient: null,
|
|
13337
15226
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13338
15227
|
sessionDbProvenanceAnomaly: false,
|
|
15228
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13339
15229
|
opencodeProcess: null,
|
|
13340
15230
|
stopOpenCodeLogTail: null,
|
|
13341
15231
|
litestreamProcess: null,
|
|
@@ -13347,12 +15237,14 @@ async function run(options) {
|
|
|
13347
15237
|
messageCount: 0,
|
|
13348
15238
|
lastProxiedActivityAt: null,
|
|
13349
15239
|
sessionCleanupTimers: [],
|
|
15240
|
+
sessionDeleteFailures: /* @__PURE__ */ new Map(),
|
|
13350
15241
|
claudeUsageTimer: null,
|
|
13351
15242
|
claudeUsageRearm: null,
|
|
13352
15243
|
openaiUsageTimer: null,
|
|
13353
15244
|
openaiUsageRearm: null,
|
|
13354
15245
|
resourceUsageTimer: null,
|
|
13355
15246
|
stopResourceUsageSampling: null,
|
|
15247
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13356
15248
|
credentialSync: null,
|
|
13357
15249
|
authHeader: ""
|
|
13358
15250
|
};
|
|
@@ -13511,6 +15403,17 @@ async function run(options) {
|
|
|
13511
15403
|
}
|
|
13512
15404
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
13513
15405
|
state.agentName = validation.agent.name;
|
|
15406
|
+
const requestedStayAwakeSeconds = validation.agent?.wake_stay_awake_seconds;
|
|
15407
|
+
if (typeof requestedStayAwakeSeconds === "number" && Number.isFinite(requestedStayAwakeSeconds) && requestedStayAwakeSeconds > 0) {
|
|
15408
|
+
state.stayAwakeSeconds = requestedStayAwakeSeconds;
|
|
15409
|
+
} else if (requestedStayAwakeSeconds !== void 0 && requestedStayAwakeSeconds !== null) {
|
|
15410
|
+
state.stayAwakeSeconds = null;
|
|
15411
|
+
log2(
|
|
15412
|
+
state,
|
|
15413
|
+
`Ignoring invalid wake_stay_awake_seconds (type: ${typeof requestedStayAwakeSeconds})`,
|
|
15414
|
+
"warn"
|
|
15415
|
+
);
|
|
15416
|
+
}
|
|
13514
15417
|
const microvmId = process.env.MICROVM_ID?.trim();
|
|
13515
15418
|
if (microvmId) {
|
|
13516
15419
|
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
@@ -13526,27 +15429,74 @@ async function run(options) {
|
|
|
13526
15429
|
} else {
|
|
13527
15430
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13528
15431
|
}
|
|
13529
|
-
if (options.restoreRunnerCredentials) {
|
|
13530
|
-
|
|
13531
|
-
|
|
13532
|
-
|
|
13533
|
-
|
|
13534
|
-
if (level === "error") {
|
|
13535
|
-
logActivity(state, { type: "error", error: message });
|
|
13536
|
-
} else {
|
|
13537
|
-
logActivity(state, { type: "info", level, message });
|
|
13538
|
-
}
|
|
15432
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
15433
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
15434
|
+
log2(state, message, level);
|
|
15435
|
+
if (state.interactive && !state.json) {
|
|
15436
|
+
logActivity(state, { type: "info", level, message });
|
|
13539
15437
|
}
|
|
13540
15438
|
};
|
|
13541
|
-
const
|
|
15439
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
15440
|
+
if (vaultResolution.error) {
|
|
15441
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15442
|
+
logCredentialVaultResolution(
|
|
15443
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
15444
|
+
"warn"
|
|
15445
|
+
);
|
|
15446
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
15447
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
15448
|
+
logCredentialVaultResolution(
|
|
15449
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
15450
|
+
"info"
|
|
15451
|
+
);
|
|
15452
|
+
} else {
|
|
15453
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
15454
|
+
logCredentialVaultResolution(
|
|
15455
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
15456
|
+
"debug"
|
|
15457
|
+
);
|
|
15458
|
+
}
|
|
15459
|
+
}
|
|
15460
|
+
const credentialContext = {
|
|
15461
|
+
env: process.env,
|
|
15462
|
+
log: (message, level = "info") => {
|
|
15463
|
+
if (level === "error") {
|
|
15464
|
+
logActivity(state, { type: "error", error: message });
|
|
15465
|
+
} else {
|
|
15466
|
+
logActivity(state, { type: "info", level, message });
|
|
15467
|
+
}
|
|
15468
|
+
}
|
|
15469
|
+
};
|
|
15470
|
+
const githubTokenPopulated = await installRunnerSecret(credentialContext);
|
|
15471
|
+
if (options.restoreRunnerCredentials) {
|
|
13542
15472
|
await restoreCredentialStores(credentialContext);
|
|
13543
|
-
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13544
15473
|
}
|
|
15474
|
+
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13545
15475
|
state.credentialSync?.arm();
|
|
13546
|
-
const
|
|
13547
|
-
|
|
13548
|
-
|
|
13549
|
-
|
|
15476
|
+
const dependencyInstallDiagnosticReporter = createDependencyInstallDiagnosticReporter({
|
|
15477
|
+
context: {
|
|
15478
|
+
homeDir: homedir7(),
|
|
15479
|
+
cwd: process.cwd(),
|
|
15480
|
+
microvmId: process.env.MICROVM_ID?.trim(),
|
|
15481
|
+
euid: process.getuid?.(),
|
|
15482
|
+
egid: process.getgid?.()
|
|
15483
|
+
},
|
|
15484
|
+
emitEvent: (message) => logActivity(state, { type: "info", level: "warn", message }),
|
|
15485
|
+
emitLocal: (message) => logLocalOnly(state, message, "warn")
|
|
15486
|
+
});
|
|
15487
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
15488
|
+
() => ({
|
|
15489
|
+
agentId: state.agentId,
|
|
15490
|
+
authHeader: state.authHeader
|
|
15491
|
+
}),
|
|
15492
|
+
({ isDiskFailure, sessionId, cause }) => {
|
|
15493
|
+
if (isDiskFailure) state.sessionStoreHealth.observeUnwritable();
|
|
15494
|
+
if (cause !== null && sessionId !== null) {
|
|
15495
|
+
state.sessionStoreHealth.observeCause(sessionId, cause);
|
|
15496
|
+
}
|
|
15497
|
+
},
|
|
15498
|
+
dependencyInstallDiagnosticReporter
|
|
15499
|
+
);
|
|
13550
15500
|
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13551
15501
|
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13552
15502
|
openCodeActivityForwarder
|
|
@@ -13556,22 +15506,27 @@ async function run(options) {
|
|
|
13556
15506
|
openCodeLogTail.stop();
|
|
13557
15507
|
};
|
|
13558
15508
|
let sessionDbVerifyFatal = false;
|
|
15509
|
+
let sessionDbVerifyFatalReason;
|
|
13559
15510
|
if (!options.restoreSessionDb) {
|
|
13560
|
-
|
|
15511
|
+
logActivity(state, {
|
|
15512
|
+
type: "info",
|
|
15513
|
+
level: "warn",
|
|
15514
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=flag-absent \u2014 Skipping session-DB restore: --restore-session-db was not passed"
|
|
15515
|
+
});
|
|
13561
15516
|
} else {
|
|
13562
15517
|
const health = await checkOpenCodeHealth(state.port);
|
|
13563
15518
|
if (health.healthy) {
|
|
13564
|
-
|
|
13565
|
-
|
|
13566
|
-
|
|
13567
|
-
"
|
|
13568
|
-
);
|
|
15519
|
+
logActivity(state, {
|
|
15520
|
+
type: "info",
|
|
15521
|
+
level: "warn",
|
|
15522
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-serving \u2014 Skipping session-DB restore: OpenCode is already serving this database"
|
|
15523
|
+
});
|
|
13569
15524
|
} else if (health.authFailed) {
|
|
13570
|
-
|
|
13571
|
-
|
|
13572
|
-
|
|
13573
|
-
"
|
|
13574
|
-
);
|
|
15525
|
+
logActivity(state, {
|
|
15526
|
+
type: "info",
|
|
15527
|
+
level: "warn",
|
|
15528
|
+
message: "SESSION-DB-VERIFY-SKIPPED: reason=opencode-auth-challenge \u2014 Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use"
|
|
15529
|
+
});
|
|
13575
15530
|
} else {
|
|
13576
15531
|
const result = await restoreAndVerifySessionDb({
|
|
13577
15532
|
dbPath: sessionDbPath(),
|
|
@@ -13588,12 +15543,13 @@ async function run(options) {
|
|
|
13588
15543
|
reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
|
|
13589
15544
|
});
|
|
13590
15545
|
sessionDbVerifyFatal = result.verifyFatal;
|
|
15546
|
+
sessionDbVerifyFatalReason = result.verifyFatalReason;
|
|
13591
15547
|
}
|
|
13592
15548
|
}
|
|
13593
15549
|
reportSessionDbRecovery(state);
|
|
13594
15550
|
if (sessionDbVerifyFatal) {
|
|
13595
15551
|
throw new Error(
|
|
13596
|
-
"SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
|
|
15552
|
+
sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
|
|
13597
15553
|
);
|
|
13598
15554
|
}
|
|
13599
15555
|
applyRunnerOpenCodeConfig({
|
|
@@ -13701,7 +15657,7 @@ async function run(options) {
|
|
|
13701
15657
|
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
13702
15658
|
}
|
|
13703
15659
|
}
|
|
13704
|
-
await
|
|
15660
|
+
await disposeAndReloadProviderConfig(state.port, state.opencodeClient ?? void 0);
|
|
13705
15661
|
if (state.opencodeApiVersion === "v2" && state.opencodeClient !== null) {
|
|
13706
15662
|
await importOpenCodeV2CredentialsForState(state);
|
|
13707
15663
|
}
|
|
@@ -13838,6 +15794,7 @@ async function run(options) {
|
|
|
13838
15794
|
getAuthHeader: () => state.authHeader,
|
|
13839
15795
|
conversationFilter: state.conversationFilter,
|
|
13840
15796
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
15797
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13841
15798
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13842
15799
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13843
15800
|
fileSyncDirectories,
|
|
@@ -13863,13 +15820,21 @@ async function run(options) {
|
|
|
13863
15820
|
openCodePassword: state.opencodePassword,
|
|
13864
15821
|
isRunning: () => state.running,
|
|
13865
15822
|
events: {
|
|
13866
|
-
onConnected: (agentId, isReconnect) => {
|
|
15823
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13867
15824
|
state.connected = true;
|
|
13868
15825
|
state.agentId = agentId;
|
|
15826
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13869
15827
|
logActivity(state, {
|
|
13870
15828
|
type: "info",
|
|
13871
|
-
message:
|
|
15829
|
+
message: connectionMessage
|
|
13872
15830
|
});
|
|
15831
|
+
if (!isReconnect && state.stayAwakeSeconds !== null && state.stayAwakeUntilMs === null) {
|
|
15832
|
+
state.stayAwakeUntilMs = Date.now() + state.stayAwakeSeconds * 1e3;
|
|
15833
|
+
logActivity(state, {
|
|
15834
|
+
type: "info",
|
|
15835
|
+
message: `Staying awake for at least ${state.stayAwakeSeconds}s as requested by the wake`
|
|
15836
|
+
});
|
|
15837
|
+
}
|
|
13873
15838
|
if (options.tunnelReadyFile) {
|
|
13874
15839
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
13875
15840
|
if (marker.ok) {
|
|
@@ -13889,7 +15854,12 @@ async function run(options) {
|
|
|
13889
15854
|
version: state.opencodeVersion,
|
|
13890
15855
|
major: state.opencodeApiVersion,
|
|
13891
15856
|
connected: state.opencodeConnected
|
|
13892
|
-
})
|
|
15857
|
+
}),
|
|
15858
|
+
connection_generation: transition.generation,
|
|
15859
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
15860
|
+
reconnect_attempt: transition.attempt,
|
|
15861
|
+
downtime_ms: transition.downtimeMs
|
|
15862
|
+
} : {}
|
|
13893
15863
|
});
|
|
13894
15864
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13895
15865
|
if (state.interactive) displayStatus(state);
|
|
@@ -13911,13 +15881,18 @@ async function run(options) {
|
|
|
13911
15881
|
if (state.interactive) displayStatus(state);
|
|
13912
15882
|
});
|
|
13913
15883
|
},
|
|
13914
|
-
onDisconnected: (code, reason) => {
|
|
15884
|
+
onDisconnected: (code, reason, transition) => {
|
|
13915
15885
|
state.connected = false;
|
|
15886
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13916
15887
|
logActivity(state, {
|
|
13917
15888
|
type: "info",
|
|
13918
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
15889
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
15890
|
+
});
|
|
15891
|
+
emitAgentDisconnected(state.agentId, {
|
|
15892
|
+
code,
|
|
15893
|
+
reason: safeReason2,
|
|
15894
|
+
connection_generation: transition.generation
|
|
13919
15895
|
});
|
|
13920
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13921
15896
|
if (state.interactive) displayStatus(state);
|
|
13922
15897
|
},
|
|
13923
15898
|
onError: (error2) => {
|
|
@@ -13994,6 +15969,7 @@ async function run(options) {
|
|
|
13994
15969
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
13995
15970
|
throw error2;
|
|
13996
15971
|
}
|
|
15972
|
+
if (!state.running) return;
|
|
13997
15973
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
13998
15974
|
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
13999
15975
|
state.openaiUsageRearm = scheduleUsageReporting(state, {
|
|
@@ -14017,7 +15993,7 @@ async function run(options) {
|
|
|
14017
15993
|
},
|
|
14018
15994
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
14019
15995
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
14020
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
15996
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
14021
15997
|
firstDelayMs: firstReportDelayMs,
|
|
14022
15998
|
nextDelayMs: usageReportDelayMs,
|
|
14023
15999
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -14060,7 +16036,7 @@ async function run(options) {
|
|
|
14060
16036
|
}
|
|
14061
16037
|
|
|
14062
16038
|
// src/index.ts
|
|
14063
|
-
var
|
|
16039
|
+
var version = getCliVersion();
|
|
14064
16040
|
var program = new Command();
|
|
14065
16041
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
14066
16042
|
"--endpoint <url>",
|
|
@@ -14078,6 +16054,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
14078
16054
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
14079
16055
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
14080
16056
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
16057
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
14081
16058
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
14082
16059
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
14083
16060
|
"-a, --agent [id]",
|